Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-8322] Compact MCP workflow status responses and fix wr lookup (#5084)
Marc Kelechava965fac68c72f
diff --git a/pyproject.toml b/pyproject.tomlindex cf4525667..7f9ca99f5 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -223,7 +223,7 @@ skip = ["webeye/actions/__init__.py", "forge/sdk/__init__.py"]plugins = "sqlalchemy.ext.mypy.plugin"[project.scripts]-skyvern = "skyvern.cli.commands:cli_app"+skyvern = "skyvern.__main__:main"[tool.pytest.ini_options]norecursedirs = ["eval", "tests/sdk"]diff --git a/skyvern/__main__.py b/skyvern/__main__.pyindex d9410e40e..127657ce4 100644--- a/skyvern/__main__.py+++ b/skyvern/__main__.py@@ -1,4 +1,11 @@-from skyvern.cli.commands import cli_app+def main() -> None:+ from skyvern._cli_bootstrap import configure_cli_bootstrap_logging # noqa: PLC0415++ configure_cli_bootstrap_logging()+ from skyvern.cli.commands import cli_app # noqa: PLC0415-if __name__ == "__main__":cli_app() # type: ignore+++if __name__ == "__main__":+ main()diff --git a/skyvern/_cli_bootstrap.py b/skyvern/_cli_bootstrap.pynew file mode 100644index 000000000..2a238f826--- /dev/null+++ b/skyvern/_cli_bootstrap.py@@ -0,0 +1,32 @@+import logging+from collections.abc import Set++_DEFAULT_CLI_LOG_LEVEL = "WARNING"++_QUIET_CLI_LOGGERS = ("skyvern", "httpx", "litellm", "playwright", "httpcore")+++def _resolve_cli_log_level_name(settings: object) -> str:+ """Honor explicit settings while keeping CLI defaults quiet."""++ fields_set: Set[str] = getattr(settings, "model_fields_set", set())+ configured_level = str(getattr(settings, "LOG_LEVEL", _DEFAULT_CLI_LOG_LEVEL)).upper()+ if "LOG_LEVEL" in fields_set:+ return configured_level+ return _DEFAULT_CLI_LOG_LEVEL+++def configure_cli_bootstrap_logging() -> None:+ """Clamp CLI process logging before importing the command tree."""+ from skyvern.config import settings # noqa: PLC0415+ from skyvern.forge.sdk.forge_log import setup_logger # noqa: PLC0415++ log_level_name = _resolve_cli_log_level_name(settings)+ settings.LOG_LEVEL = log_level_name+ setup_logger()+ log_level = logging.getLevelName(log_level_name)+ if not isinstance(log_level, int):+ log_level = logging.WARNING+ logging.getLogger().setLevel(log_level)+ for logger_name in _QUIET_CLI_LOGGERS:+ logging.getLogger(logger_name).setLevel(log_level)diff --git a/skyvern/cli/commands/__init__.py b/skyvern/cli/commands/__init__.pyindex 2771c1e3f..221bf587a 100644--- a/skyvern/cli/commands/__init__.py+++ b/skyvern/cli/commands/__init__.py@@ -1,9 +1,7 @@-import logging-import typerfrom dotenv import load_dotenv-from skyvern.forge.sdk.forge_log import setup_logger as _setup_logger+from skyvern._cli_bootstrap import configure_cli_bootstrap_logging as _configure_cli_bootstrap_loggingfrom skyvern.utils.env_paths import resolve_backend_env_pathfrom ..auth_command import login as login_command@@ -34,10 +32,8 @@ def configure_cli_logging() -> None:return_cli_logging_configured = True- # Suppress noisy SDK/third-party logs for CLI execution only.- for logger_name in ("skyvern", "httpx", "litellm", "playwright", "httpcore"):- logging.getLogger(logger_name).setLevel(logging.WARNING)- _setup_logger()+ # Keep callback-time execution aligned with the entrypoint bootstrap.+ _configure_cli_bootstrap_logging()cli_app = typer.Typer(diff --git a/skyvern/cli/commands/__main__.py b/skyvern/cli/commands/__main__.pyindex 3ab570f15..ca79ac561 100644--- a/skyvern/cli/commands/__main__.py+++ b/skyvern/cli/commands/__main__.py@@ -1,9 +1,16 @@from dotenv import load_dotenv+from skyvern._cli_bootstrap import configure_cli_bootstrap_loggingfrom skyvern.utils.env_paths import resolve_backend_env_path-from . import cli_app-if __name__ == "__main__": # pragma: no cover - manual CLI invocation+def main() -> None:+ configure_cli_bootstrap_logging()+ from . import cli_app # noqa: PLC0415+load_dotenv(resolve_backend_env_path())cli_app()+++if __name__ == "__main__": # pragma: no cover - manual CLI invocation+ main()diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.pyindex 3ccc710ab..203c06e53 100644--- a/skyvern/cli/mcp_tools/workflow.py+++ b/skyvern/cli/mcp_tools/workflow.py@@ -9,8 +9,9 @@ from __future__ import annotationsimport asyncioimport json+from datetime import datetimefrom enum import Enum-from typing import Annotated, Any+from typing import Annotated, Any, Literalimport structlogimport yaml@@ -26,6 +27,13 @@ from ._session import get_skyvernfrom ._validation import validate_folder_idLOG = structlog.get_logger()+_SUMMARY_TOP_LEVEL_KEY_LIMIT = 8+_SUMMARY_SCALAR_PREVIEW_LIMIT = 3+_SUMMARY_ARTIFACT_PREVIEW_LIMIT = 4+_SUMMARY_STRING_PREVIEW_LIMIT = 120+_SUMMARY_RECURSION_LIMIT = 10+_SCREENSHOT_LIST_KEYS = frozenset({"task_screenshots", "workflow_screenshots", "screenshot_urls"})+_SCREENSHOT_ARTIFACT_ID_KEYS = frozenset({"task_screenshot_artifact_ids", "workflow_screenshot_artifact_ids"})# ---------------------------------------------------------------------------# Helpers@@ -98,6 +106,268 @@ def _serialize_run(run: Any) -> dict[str, Any]:return data+def _get_value(obj: Any, key: str, default: Any = None) -> Any:+ if isinstance(obj, dict):+ return obj.get(key, default)+ return getattr(obj, key, default)+++def _get_run_id(run: Any) -> str | None:+ return _get_value(run, "run_id") or _get_value(run, "workflow_run_id")+++def _jsonable(value: Any) -> Any:+ if hasattr(value, "model_dump"):+ return value.model_dump(mode="json")+ if isinstance(value, datetime):+ return value.isoformat()+ if isinstance(value, list):+ return [_jsonable(item) for item in value]+ if isinstance(value, tuple):+ return [_jsonable(item) for item in value]+ if isinstance(value, dict):+ return {k: _jsonable(v) for k, v in value.items()}+ return value+++def _truncate_preview(value: Any) -> Any:+ value = _jsonable(value)+ if isinstance(value, str) and len(value) > _SUMMARY_STRING_PREVIEW_LIMIT:+ return f"{value[: _SUMMARY_STRING_PREVIEW_LIMIT - 3]}..."+ return value+++def _is_scalarish(value: Any) -> bool:+ return value is None or isinstance(value, str | int | float | bool)+++def _init_output_stats() -> dict[str, Any]:+ return {+ "has_extracted_information": False,+ "nested_screenshot_count": 0,+ "artifact_id_count": 0,+ "artifact_ids_preview": [],+ }+++def _note_artifact_ids(stats: dict[str, Any], values: list[Any]) -> None:+ stats["artifact_id_count"] += len(values)+ preview = stats["artifact_ids_preview"]+ for value in values:+ if len(preview) >= _SUMMARY_ARTIFACT_PREVIEW_LIMIT:+ break+ value_str = str(value)+ if value_str not in preview:+ preview.append(value_str)+++def _scan_output_value(value: Any, stats: dict[str, Any], depth: int = 0) -> None:+ if depth > _SUMMARY_RECURSION_LIMIT:+ return++ value = _jsonable(value)++ if isinstance(value, dict):+ for key, nested_value in value.items():+ if key == "extracted_information" and nested_value is not None:+ stats["has_extracted_information"] = True+ if key in _SCREENSHOT_LIST_KEYS and isinstance(nested_value, list):+ stats["nested_screenshot_count"] += len(nested_value)+ if key in _SCREENSHOT_ARTIFACT_ID_KEYS and isinstance(nested_value, list):+ _note_artifact_ids(stats, nested_value)+ _scan_output_value(nested_value, stats, depth + 1)+ return++ if isinstance(value, list):+ for item in value:+ _scan_output_value(item, stats, depth + 1)+++def _summarize_output_value(output_value: Any) -> tuple[dict[str, Any], dict[str, Any]]:+ if output_value is None:+ return {"present": False}, _init_output_stats()++ output_value = _jsonable(output_value)+ stats = _init_output_stats()+ _scan_output_value(output_value, stats)++ summary: dict[str, Any] = {"present": True}++ if isinstance(output_value, dict):+ top_level_keys = list(output_value.keys())+ summary["top_level_keys"] = top_level_keys[:_SUMMARY_TOP_LEVEL_KEY_LIMIT]+ if len(top_level_keys) > _SUMMARY_TOP_LEVEL_KEY_LIMIT:+ summary["top_level_key_count"] = len(top_level_keys)+ summary["block_output_count"] = len([key for key in top_level_keys if key != "extracted_information"])++ scalar_preview: dict[str, Any] = {}+ for key, value in output_value.items():+ if key == "extracted_information":+ continue+ if _is_scalarish(value):+ scalar_preview[key] = _truncate_preview(value)+ elif (+ isinstance(value, list)+ and value+ and len(value) <= _SUMMARY_SCALAR_PREVIEW_LIMIT+ and all(_is_scalarish(item) for item in value)+ ):+ scalar_preview[key] = [_truncate_preview(item) for item in value]+ if len(scalar_preview) >= _SUMMARY_SCALAR_PREVIEW_LIMIT:+ break+ if scalar_preview:+ summary["scalar_preview"] = scalar_preview+ elif isinstance(output_value, list):+ summary["item_count"] = len(output_value)+ if (+ output_value+ and len(output_value) <= _SUMMARY_SCALAR_PREVIEW_LIMIT+ and all(_is_scalarish(item) for item in output_value)+ ):+ summary["scalar_preview"] = [_truncate_preview(item) for item in output_value]+ else:+ summary["scalar_preview"] = _truncate_preview(output_value)++ summary["has_extracted_information"] = stats["has_extracted_information"]+ summary["nested_screenshot_count"] = stats["nested_screenshot_count"]+ summary["artifact_id_count"] = stats["artifact_id_count"]++ return summary, stats+++def _summarize_artifacts(run: Any, output_stats: dict[str, Any]) -> dict[str, Any]:+ downloaded_files = _jsonable(_get_value(run, "downloaded_files")) or []+ screenshot_urls = _jsonable(_get_value(run, "screenshot_urls")) or []++ summary: dict[str, Any] = {+ "recording_available": bool(_get_value(run, "recording_url")),+ "workflow_screenshot_count": len(screenshot_urls),+ "downloaded_file_count": len(downloaded_files),+ "artifact_id_count": output_stats["artifact_id_count"],+ }++ filenames = [+ filename+ for filename in (_get_value(file_info, "filename") for file_info in downloaded_files)+ if isinstance(filename, str) and filename+ ]+ if filenames:+ summary["downloaded_file_names"] = filenames[:_SUMMARY_SCALAR_PREVIEW_LIMIT]++ if output_stats["artifact_ids_preview"]:+ summary["artifact_ids_preview"] = output_stats["artifact_ids_preview"]++ return summary+++def _serialize_run_summary(run: Any) -> dict[str, Any]:+ run_id = _get_run_id(run)+ run_type = _get_value(run, "run_type")+ if run_type is None and _get_value(run, "workflow_run_id"):+ run_type = "workflow_run"++ output_value = _get_value(run, "output")+ if output_value is None and _get_value(run, "outputs") is not None:+ output_value = _get_value(run, "outputs")++ output_summary, output_stats = _summarize_output_value(output_value)++ summary: dict[str, Any] = {+ "run_id": run_id,+ "status": str(_get_value(run, "status")) if _get_value(run, "status") is not None else None,+ "run_type": str(run_type) if run_type is not None else None,+ "artifact_summary": _summarize_artifacts(run, output_stats),+ "output_summary": output_summary,+ }++ failure_reason = _get_value(run, "failure_reason")+ if failure_reason:+ summary["failure_reason"] = failure_reason++ run_with = _get_value(run, "run_with")+ if run_with:+ summary["run_with"] = run_with++ workflow_title = _get_value(run, "workflow_title")+ if workflow_title:+ summary["workflow_title"] = workflow_title++ step_count = _get_value(run, "step_count")+ total_steps = _get_value(run, "total_steps")+ if step_count is not None:+ summary["step_count"] = step_count+ elif total_steps is not None:+ summary["total_steps"] = total_steps++ return {key: value for key, value in summary.items() if value is not None}+++def _serialize_run_full(run: Any) -> dict[str, Any]:+ if not isinstance(run, dict):+ return _serialize_run(run)++ data: dict[str, Any] = {+ "run_id": _get_run_id(run),+ "status": str(_get_value(run, "status")) if _get_value(run, "status") is not None else None,+ "run_type": "workflow_run" if _get_value(run, "workflow_run_id") else _get_value(run, "run_type"),+ }++ for field in (+ "workflow_id",+ "workflow_title",+ "failure_reason",+ "recording_url",+ "screenshot_urls",+ "downloaded_files",+ "downloaded_file_urls",+ "parameters",+ "errors",+ "browser_session_id",+ "browser_profile_id",+ "run_with",+ "total_steps",+ ):+ value = _get_value(run, field)+ if value is not None:+ data[field] = _jsonable(value)++ outputs = _get_value(run, "outputs")+ if outputs is not None:+ data["output"] = _jsonable(outputs)++ for ts_field in ("created_at", "modified_at", "started_at", "finished_at", "queued_at"):+ value = _get_value(run, ts_field)+ if value is not None:+ data[ts_field] = _jsonable(value)++ return {key: value for key, value in data.items() if value is not None}+++async def _get_workflow_run_status(+ workflow_run_id: str,+ *,+ include_output_details: bool,+) -> dict[str, Any]:+ skyvern = get_skyvern()+ # The generated SDK only exposes get_run() for /v1/runs/{run_id}; wr_... IDs+ # require the workflow-run detail route until a public SDK helper exists.+ response = await skyvern._client_wrapper.httpx_client.request(+ f"api/v1/workflows/runs/{workflow_run_id}",+ method="GET",+ params={"include_output_details": include_output_details},+ )+ if response.status_code == 404:+ raise NotFoundError(body={"detail": f"Workflow run {workflow_run_id!r} not found"})+ if response.status_code >= 400:+ detail = ""+ try:+ detail = response.json().get("detail", response.text)+ except Exception:+ detail = response.text+ raise RuntimeError(f"HTTP {response.status_code}: {detail}")+ return response.json()++def _validate_workflow_id(workflow_id: str, action: str) -> dict[str, Any] | None:"""Validate workflow_id format. Returns a make_result error dict or None if valid."""if "/" in workflow_id or "\\" in workflow_id:@@ -911,17 +1181,36 @@ async def skyvern_workflow_run(async def skyvern_workflow_status(run_id: Annotated[str, "Run ID to check (wr_... for workflow runs, tsk_v2_... for task runs)"],+ verbosity: Annotated[+ Literal["summary", "full"],+ Field(description="`summary` returns a compact status payload. `full` includes outputs, timestamps, and URLs."),+ ] = "summary",) -> dict[str, Any]:"""Check the status and progress of a workflow or task run. Use when you need to monitora running workflow, check if it completed, or retrieve its output."""if err := _validate_run_id(run_id, "skyvern_workflow_status"):return err-- skyvern = get_skyvern()+ if verbosity not in {"summary", "full"}:+ return make_result(+ "skyvern_workflow_status",+ ok=False,+ error=make_error(+ ErrorCode.INVALID_INPUT,+ f"Invalid verbosity: {verbosity!r}",+ "Use verbosity='summary' for compact status or verbosity='full' for full detail.",+ ),+ )with Timer() as timer:try:- run = await skyvern.get_run(run_id)+ if run_id.startswith("wr_"):+ run = await _get_workflow_run_status(+ run_id,+ include_output_details=verbosity == "full",+ )+ else:+ skyvern = get_skyvern()+ run = await skyvern.get_run(run_id)timer.mark("sdk")except NotFoundError:return make_result(@@ -942,8 +1231,14 @@ async def skyvern_workflow_status(error=make_error(ErrorCode.API_ERROR, str(e), "Check the run ID and your API key"),)- data = _serialize_run(run)- data["sdk_equivalent"] = f"await skyvern.get_run({run_id!r})"+ data = _serialize_run_full(run) if verbosity == "full" else _serialize_run_summary(run)+ if run_id.startswith("wr_"):+ data["sdk_equivalent"] = f"await skyvern_workflow_status(run_id={run_id!r}, verbosity={verbosity!r})"+ else:+ verbosity_arg = "" if verbosity == "summary" else f", verbosity={verbosity!r}"+ data["sdk_equivalent"] = (+ f"await skyvern.get_run({run_id!r}) # or skyvern_workflow_status(run_id={run_id!r}{verbosity_arg})"+ )return make_result("skyvern_workflow_status", data=data, timing_ms=timer.timing_ms)diff --git a/skyvern/cli/workflow.py b/skyvern/cli/workflow.pyindex edd8295db..faf0c4c1b 100644--- a/skyvern/cli/workflow.py+++ b/skyvern/cli/workflow.py@@ -253,7 +253,7 @@ def workflow_status("""Get workflow run status."""async def _run() -> dict[str, Any]:- return await tool_workflow_status(run_id=run_id)+ return await tool_workflow_status(run_id=run_id, verbosity="full")_run_tool(_run, json_output=json_output, hint_on_exception="Check the run ID and API key.")diff --git a/skyvern/config.py b/skyvern/config.pyindex ebc461c64..040d64514 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -28,6 +28,7 @@ class Settings(BaseSettings):# settings for experimentationENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE: bool = False+ ENABLE_DOM_PARSER_V2: bool = FalseADDITIONAL_MODULES: list[str] = []diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 22fce7ea1..fca0798a8 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -1,6 +1,7 @@import asyncioimport copyimport hashlib+from collections import dequefrom datetime import timedeltafrom typing import Dict, List@@ -81,10 +82,8 @@ def _get_shape_cache_key(hash: str) -> str:return f"skyvern:shape:{hash}"-def _remove_skyvern_attributes(element: Dict) -> Dict:- """- To get the original HTML element without skyvern attributes- """+def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:+ """Legacy: deep-copy based attribute removal (pre-V2)."""element_copied = copy.deepcopy(element)for attr in ELEMENT_NODE_ATTRIBUTES:if element_copied.get(attr):@@ -102,12 +101,36 @@ def _remove_skyvern_attributes(element: Dict) -> Dict:trimmed_children = []for child in children:- trimmed_children.append(_remove_skyvern_attributes(child))+ trimmed_children.append(_remove_skyvern_attributes_legacy(child))element_copied["children"] = trimmed_childrenreturn element_copied+def _remove_skyvern_attributes(element: Dict) -> Dict:+ """+ To get the original HTML element without skyvern attributes.+ V2 uses shallow copy + filtered dicts instead of double deepcopy for performance.+ """+ if not settings.ENABLE_DOM_PARSER_V2:+ return _remove_skyvern_attributes_legacy(element)++ element_copied = dict(element)+ for attr in ELEMENT_NODE_ATTRIBUTES:+ element_copied.pop(attr, None)++ if "attributes" in element_copied:+ element_copied["attributes"] = {+ k: v for k, v in element_copied["attributes"].items() if k not in USELESS_SHAPE_ATTRIBUTE+ }++ children: List[Dict] | None = element_copied.get("children")+ if children is not None:+ element_copied["children"] = [_remove_skyvern_attributes(child) for child in children]++ return element_copied++def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:context = skyvern_context.ensure_context()if hashed_key:@@ -519,7 +542,7 @@ class AgentFunction:skyvern_frame = await SkyvernFrame.create_instance(frame=frame)current_frame_index = context.frame_index_map.get(frame, 0)- queue = []+ queue: deque = deque()element_cnt = 0eligible_svgs = [] # List to store eligible SVGs and their frames@@ -527,7 +550,7 @@ class AgentFunction:queue.append(element)while queue:- queue_ele = queue.pop(0)+ queue_ele = queue.popleft()element_cnt += 1if element_cnt == MAX_ELEMENT_CNT:diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.jsindex af412daf4..8b1ff6e0b 100644--- a/skyvern/webeye/scraper/domUtils.js+++ b/skyvern/webeye/scraper/domUtils.js@@ -157,7 +157,8 @@ class DomUtils {// this.let computedStyle;if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {- for (const child of Array.from(element.children)) {+ for (let ci = 0; ci < element.children.length; ci++) {+ const child = element.children[ci];computedStyle = getElementComputedStyle(child, null);if (!computedStyle) {continue;@@ -360,11 +361,27 @@ class QuadTreeNode {}}-// from playwright+// from playwright — with per-tree-build cache for non-pseudo styles+const _computedStyleCache = new WeakMap();+function getElementComputedStyle(element, pseudo) {- return element.ownerDocument && element.ownerDocument.defaultView- ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)- : undefined;+ if (!element.ownerDocument || !element.ownerDocument.defaultView) {+ return undefined;+ }+ // Cache only non-pseudo styles (the hot path called 3-5× per element)+ if (pseudo == null) {+ const cached = _computedStyleCache.get(element);+ if (cached !== undefined) {+ return cached;+ }+ const style = element.ownerDocument.defaultView.getComputedStyle(+ element,+ null,+ );+ _computedStyleCache.set(element, style);+ return style;+ }+ return element.ownerDocument.defaultView.getComputedStyle(element, pseudo);}// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98@@ -411,8 +428,14 @@ function hasASPClientControl() {return typeof ASPxClientControl !== "undefined";}+// Cache for isHoverOnlyElement — called up to 3× per element+// (isElementVisible, isInteractable, buildElementObject)+const _hoverOnlyCache = new WeakMap();+// Check if element is only visible on hover (e.g., hover-only buttons)function isHoverOnlyElement(element) {+ const cached = _hoverOnlyCache.get(element);+ if (cached !== undefined) return cached;// Check for common hover-only patterns in class namesconst className = element.className?.toString() ?? "";const parentClassName = element.parentElement?.className?.toString() ?? "";@@ -424,6 +447,7 @@ function isHoverOnlyElement(element) {parentClassName.includes("hover-") ||parentClassName.includes("-hover")) {+ _hoverOnlyCache.set(element, true);return true;}@@ -440,12 +464,14 @@ function isHoverOnlyElement(element) {parentClass.includes("item")) {// This element might be revealed on parent hover+ _hoverOnlyCache.set(element, true);return true;}parent = parent.parentElement;depth += 1;}+ _hoverOnlyCache.set(element, false);return false;}@@ -454,9 +480,10 @@ function isHoverOnlyElement(element) {function isElementVisible(element) {// TODO: This is a hack to not check visibility for option elements// because they are not visible by default. We check their parent instead for visibility.+ const elTagName = element.tagName.toLowerCase();if (- element.tagName.toLowerCase() === "option" ||- (element.tagName.toLowerCase() === "input" &&+ elTagName === "option" ||+ (elTagName === "input" &&(element.type === "radio" || element.type === "checkbox")))return element.parentElement && isElementVisible(element.parentElement);@@ -622,11 +649,10 @@ function expectHitTarget(hitPoint, targetElement) {}function getChildElements(element) {- if (element.childElementCount !== 0) {- return Array.from(element.children);- } else {- return [];- }+ // Return the live HTMLCollection directly — callers iterate by index,+ // and processElement only modifies attributes (not child structure),+ // so the live collection remains stable during iteration.+ return element.children;}function isParent(parent, child) {@@ -661,9 +687,10 @@ function isHidden(element) {return true;}if (element.hidden) {+ const tn = element.tagName.toLowerCase();if (style?.cursor === "pointer" &&- element.tagName.toLowerCase() === "input" &&+ tn === "input" &&(element.type === "submit" || element.type === "button")) {// there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.@@ -723,6 +750,25 @@ function hasAngularClickBinding(element) {);}+// Pre-computed Set for widget roles — avoids recreating array on every hasWidgetRole call+const WIDGET_ROLES = new Set([+ "button",+ "link",+ "checkbox",+ "menuitem",+ "menuitemcheckbox",+ "menuitemradio",+ "radio",+ "tab",+ "combobox",+ "searchbox",+ "slider",+ "spinbutton",+ "switch",+ "gridcell",+ "option",+]);+function hasWidgetRole(element) {const role = element.getAttribute("role");if (!role) {@@ -730,44 +776,15 @@ function hasWidgetRole(element) {}// https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles// Not all roles make sense for the time being so we only check for the ones that do- if (role.toLowerCase().trim() === "textbox") {+ const normalizedRole = role.toLowerCase().trim();+ if (normalizedRole === "textbox") {return !isReadonlyElement(element);}-- const widgetRoles = [- "button",- "link",- "checkbox",- "menuitem",- "menuitemcheckbox",- "menuitemradio",- "radio",- "tab",- "combobox",- "searchbox",- "slider",- "spinbutton",- "switch",- "gridcell",- "option",- ];- return widgetRoles.includes(role.toLowerCase().trim());+ return WIDGET_ROLES.has(normalizedRole);}function isTableRelatedElement(element) {- const tagName = element.tagName.toLowerCase();- return [- "table",- "caption",- "thead",- "tbody",- "tfoot",- "tr",- "th",- "td",- "colgroup",- "col",- ].includes(tagName);+ return TABLE_RELATED_TAGS.has(element.tagName.toLowerCase());}function isDOMNodeRepresentDiv(element) {@@ -844,17 +861,74 @@ function isValidCSSSelector(selector) {}}-function isInteractable(element, hoverStylesMap) {- if (!isElementVisible(element)) {- return false;- }+// Pre-computed tag sets for O(1) lookup in isInteractable hot path+const ALWAYS_INTERACTABLE_TAGS = new Set([+ "button",+ "select",+ "option",+ "textarea",+]);+const POINTER_CHECK_TAGS = new Set([+ "div",+ "img",+ "span",+ "a",+ "i",+ "li",+ "p",+ "td",+ "svg",+ "strong",+ "h1",+ "h2",+ "h3",+ "h4",+]);+const SKIP_TAGS = new Set(["html", "iframe", "frameset", "frame"]);+// Attributes that should be normalized to boolean true/false in buildElementObject+const BOOLEAN_ATTRS = new Set([+ "required",+ "aria-required",+ "checked",+ "aria-checked",+ "selected",+ "aria-selected",+ "readonly",+ "aria-readonly",+ "disabled",+ "aria-disabled",+]);+const TABLE_RELATED_TAGS = new Set([+ "table",+ "caption",+ "thead",+ "tbody",+ "tfoot",+ "tr",+ "th",+ "td",+ "colgroup",+ "col",+]);++function isInteractable(+ element,+ hoverStylesMap,+ skipVisibilityChecks = false,+ pageFlags = {},+) {+ if (!skipVisibilityChecks) {+ if (!isElementVisible(element)) {+ return false;+ }- if (isHidden(element)) {- return false;- }+ if (isHidden(element)) {+ return false;+ }- if (isScriptOrStyle(element)) {- return false;+ if (isScriptOrStyle(element)) {+ return false;+ }}if (hasWidgetRole(element)) {@@ -879,19 +953,7 @@ function isInteractable(element, hoverStylesMap) {}const tagName = element.tagName.toLowerCase();- if (tagName === "html") {- return false;- }-- if (tagName === "iframe") {- return false;- }-- if (tagName === "frameset") {- return false;- }-- if (tagName === "frame") {+ if (SKIP_TAGS.has(tagName)) {return false;}@@ -904,12 +966,7 @@ function isInteractable(element, hoverStylesMap) {return false;}- if (- tagName === "button" ||- tagName === "select" ||- tagName === "option" ||- tagName === "textarea"- ) {+ if (ALWAYS_INTERACTABLE_TAGS.has(tagName)) {return true;}@@ -943,18 +1000,11 @@ function isInteractable(element, hoverStylesMap) {// support listbox and options underneath it// div element should be checked here before the css pointer- if (- (tagName === "ul" || tagName === "div") &&- element.hasAttribute("role") &&- element.getAttribute("role").toLowerCase() === "listbox"- ) {+ const roleAttr = element.getAttribute("role")?.toLowerCase();+ if ((tagName === "ul" || tagName === "div") && roleAttr === "listbox") {return true;}- if (- (tagName === "li" || tagName === "div") &&- element.hasAttribute("role") &&- element.getAttribute("role").toLowerCase() === "option"- ) {+ if ((tagName === "li" || tagName === "div") && roleAttr === "option") {return true;}@@ -974,6 +1024,7 @@ function isInteractable(element, hoverStylesMap) {if (tagName === "div" &&className.includes("pac-item") &&+ pageFlags.hasPacContainers !== false &&element.closest('div[class*="pac-container"]')) {return true;@@ -981,33 +1032,23 @@ function isInteractable(element, hoverStylesMap) {if (tagName === "div" &&- element.hasAttribute("aria-disabled") &&- element.getAttribute("aria-disabled").toLowerCase() === "false"+ element.getAttribute("aria-disabled")?.toLowerCase() === "false") {return true;}- if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {+ if (+ tagName === "span" &&+ pageFlags.hasDropdownContainers !== false &&+ element.closest('div[id*="dropdown-container"]')+ ) {return true;}// FIXME: maybe we need to enable the pointer check for all elements?if (- tagName === "div" ||- tagName === "img" ||- tagName === "span" ||- tagName === "a" ||- tagName === "i" ||- tagName === "li" ||- tagName === "p" ||- tagName === "td" ||- tagName === "svg" ||- tagName === "strong" ||- tagName === "h1" ||- tagName === "h2" ||- tagName === "h3" ||- tagName === "h4" ||- // sometime it's a customized element like <my-login-button>, we should check pointer style+ POINTER_CHECK_TAGS.has(tagName) ||+ // customized elements like <my-login-button>, check pointer styletagName.includes("button") ||tagName.includes("select") ||tagName.includes("option") ||@@ -1046,11 +1087,7 @@ function isInteractable(element, hoverStylesMap) {}// consider <div tabindex="0"> as interactable- if (- tagName.toLowerCase() === "div" &&- element.hasAttribute("tabindex") &&- element.getAttribute("tabindex").toLowerCase() === "0"- ) {+ if (tagName === "div" && element.getAttribute("tabindex") === "0") {return true;}@@ -1099,15 +1136,10 @@ const isComboboxDropdown = (element) => {if (element.tagName.toLowerCase() !== "input") {return false;}- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";- const readonly =- element.getAttribute("readonly") &&- element.getAttribute("readonly").toLowerCase() !== "false";+ const role = element.getAttribute("role")?.toLowerCase() ?? "";+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";+ const readonlyVal = element.getAttribute("readonly");+ const readonly = readonlyVal && readonlyVal.toLowerCase() !== "false";const controls = element.hasAttribute("aria-controls");return role && haspopup && controls && readonly;};@@ -1117,24 +1149,16 @@ const isDivComboboxDropdown = (element) => {if (tagName !== "div") {return false;}- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";+ const role = element.getAttribute("role")?.toLowerCase() ?? "";+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";const controls = element.hasAttribute("aria-controls");return role === "combobox" && controls && haspopup;};const isDropdownButton = (element) => {const tagName = element.tagName.toLowerCase();- const type = element.getAttribute("type")- ? element.getAttribute("type").toLowerCase()- : "";- const haspopup = element.getAttribute("aria-haspopup")- ? element.getAttribute("aria-haspopup").toLowerCase()- : "";+ const type = element.getAttribute("type")?.toLowerCase() ?? "";+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";const hasExpanded = element.hasAttribute("aria-expanded");return (tagName === "button" &&@@ -1146,9 +1170,7 @@ const isDropdownButton = (element) => {const isSelect2Dropdown = (element) => {const tagName = element.tagName.toLowerCase();const className = element.className.toString();- const role = element.getAttribute("role")- ? element.getAttribute("role").toLowerCase()- : "";+ const role = element.getAttribute("role")?.toLowerCase() ?? "";if (tagName === "a") {return className.includes("select2-choice");@@ -1249,9 +1271,7 @@ const isAngularDropdown = (element) => {const tagName = element.tagName.toLowerCase();if (tagName === "input" || tagName === "span") {- const ariaLabel = element.hasAttribute("aria-label")- ? element.getAttribute("aria-label").toLowerCase()- : "";+ const ariaLabel = element.getAttribute("aria-label")?.toLowerCase() ?? "";return ariaLabel.includes("select") || ariaLabel.includes("choose");}@@ -1272,21 +1292,34 @@ const isAngularMaterialDatePicker = (element) => {);};+// Cache for pseudo-element content — called in hasBeforeOrAfterPseudoContent+// and again in buildElementObject for the same element+const _pseudoContentCache = new WeakMap();+function getPseudoContent(element, pseudo) {- const pseudoStyle = getElementComputedStyle(element, pseudo);- if (!pseudoStyle) {- return null;+ let elementCache = _pseudoContentCache.get(element);+ if (elementCache && pseudo in elementCache) {+ return elementCache[pseudo];}- const content = pseudoStyle- .getPropertyValue("content")- .replace(/"/g, "")- .trim();- if (content === "none" || !content) {- return null;+ const pseudoStyle = getElementComputedStyle(element, pseudo);+ let result = null;+ if (pseudoStyle) {+ const content = pseudoStyle+ .getPropertyValue("content")+ .replace(/"/g, "")+ .trim();+ if (content !== "none" && content) {+ result = content;+ }}- return content;+ if (!elementCache) {+ elementCache = {};+ _pseudoContentCache.set(element, elementCache);+ }+ elementCache[pseudo] = result;+ return result;}function hasBeforeOrAfterPseudoContent(element) {@@ -1312,12 +1345,8 @@ function removeMultipleSpaces(str) {return str;}- // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement- if (- str.indexOf(" ") === -1 &&- str.indexOf("\t") === -1 &&- str.indexOf("\n") === -1- ) {+ // Optimization: check if contains whitespace worth collapsing before running regex+ if (!/\s{2}|[\t\n]/.test(str)) {return str;}@@ -1346,10 +1375,9 @@ function cleanupText(text) {}const checkStringIncludeRequire = (str) => {+ const lower = str.toLowerCase();return (- str.toLowerCase().includes("*") ||- str.toLowerCase().includes("✱") ||- str.toLowerCase().includes("require")+ lower.includes("*") || lower.includes("✱") || lower.includes("require"));};@@ -1434,10 +1462,10 @@ function getElementText(element) {}function getSelectOptions(element) {- const options = Array.from(element.options);const selectOptions = [];- for (const option of options) {+ for (let i = 0; i < element.options.length; i++) {+ const option = element.options[i];selectOptions.push({optionIndex: option.index,text: removeMultipleSpaces(option.textContent),@@ -1518,6 +1546,7 @@ async function buildElementObject(element,interactable,purgeable = false,+ cachedIsInSvg = null,) {var element_id = element.getAttribute("unique_id") ?? (await uniqueId());var elementTagNameLower = element.tagName.toLowerCase();@@ -1527,23 +1556,9 @@ async function buildElementObject(if (element.attributes[Symbol.iterator]) {for (const attr of element.attributes) {var attrValue = attr.value;- if (- attr.name === "required" ||- attr.name === "aria-required" ||- attr.name === "checked" ||- attr.name === "aria-checked" ||- attr.name === "selected" ||- attr.name === "aria-selected" ||- attr.name === "readonly" ||- attr.name === "aria-readonly" ||- attr.name === "disabled" ||- attr.name === "aria-disabled"- ) {- if (attrValue && attrValue.toLowerCase() === "false") {- attrValue = false;- } else {- attrValue = true;- }+ if (BOOLEAN_ATTRS.has(attr.name)) {+ attrValue =+ attrValue && attrValue.toLowerCase() === "false" ? false : true;}attrs[attr.name] = attrValue;}@@ -1637,7 +1652,10 @@ async function buildElementObject(purgeable: purgeable,// don't trim any attr of this element if keepAllAttr=TruekeepAllAttr:- elementTagNameLower === "svg" || element.closest("svg") !== null,+ elementTagNameLower === "svg" ||+ (cachedIsInSvg !== null+ ? cachedIsInSvg+ : element.closest("svg") !== null),isSelectable:elementTagNameLower === "select" ||isDatePickerSelector(element) ||@@ -1729,6 +1747,15 @@ async function buildElementTree(var elements = [];var resultArray = [];+ // O(1) parent lookup map — replaces the O(n) elements.find() that caused O(n²) tree building+ const elementIdMap = new Map();+ // Page-level check: does the page have any dropdown-container divs?+ // If not, skip expensive .closest() for every span element.+ const hasDropdownContainers =+ document.querySelector('div[id*="dropdown-container"]') !== null;+ // Page-level check: does the page have any Google Maps pac-container?+ const hasPacContainers =+ document.querySelector('div[class*="pac-container"]') !== null;async function processElement(element,@@ -1784,7 +1811,12 @@ async function buildElementTree(}const isVisible = isElementVisible(element);if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {- let interactable = isInteractable(element, hoverStylesMap);+ // Pass skipVisibilityChecks=true since we already verified visible+not hidden+not script/style.+ // Pass pageFlags so .closest() calls are skipped when the page lacks those containers.+ let interactable = isInteractable(element, hoverStylesMap, true, {+ hasDropdownContainers,+ hasPacContainers,+ });let elementObj = null;let isParentSVG = null;if (must_included_tags.includes(tagName)) {@@ -1807,65 +1839,76 @@ async function buildElementTree(} else if (hasBeforeOrAfterPseudoContent(element)) {elementObj = await buildElementObject(frame, element, interactable);} else if (tagName === "svg") {- elementObj = await buildElementObject(frame, element, interactable);+ elementObj = await buildElementObject(+ frame,+ element,+ interactable,+ false,+ true,+ );} else if ((isParentSVG = element.closest("svg")) &&isParentSVG.getAttribute("unique_id")) {// if element is the children of the <svg> with an unique_id- elementObj = await buildElementObject(frame, element, interactable);- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {- elementObj = await buildElementObject(frame, element, interactable);- } else if (- tagName === "embed" &&- element.getAttribute("type")?.toLowerCase() === "application/pdf"- ) {elementObj = await buildElementObject(frame,element,interactable,+ false,true,);+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {+ elementObj = await buildElementObject(frame, element, interactable);} else if (- getElementText(element).length > 0 &&- getElementText(element).length <= 5000+ tagName === "embed" &&+ element.getAttribute("type")?.toLowerCase() === "application/pdf") {- if (window.GlobalEnableAllTextualElements) {- // force all textual elements to be interactable- interactable = true;- }- elementObj = await buildElementObject(frame, element, interactable);- } else if (full_tree) {- // when building full tree, we only get text from element itself- // elements without text are purgeableelementObj = await buildElementObject(frame,element,interactable,true,);- if (elementObj.text.length > 0) {- elementObj.purgeable = false;+ } else {+ // Cache getElementText() to avoid calling it twice+ const cachedText = getElementText(element);+ if (cachedText.length > 0 && cachedText.length <= 5000) {+ if (window.GlobalEnableAllTextualElements) {+ // force all textual elements to be interactable+ interactable = true;+ }+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (full_tree) {+ // when building full tree, we only get text from element itself+ // elements without text are purgeable+ elementObj = await buildElementObject(+ frame,+ element,+ interactable,+ true,+ );+ if (elementObj.text.length > 0) {+ elementObj.purgeable = false;+ }}}if (elementObj) {elementObj.xpath = current_xpath;elements.push(elementObj);- // If the element is interactable but has no interactable parent,+ elementIdMap.set(elementObj.id, elementObj);+ // If the element has no parent in the tree,// then it starts a new tree, so add it to the result array- // and set its id as the interactable parent id for the next elements- // under itif (parentId === null) {resultArray.push(elementObj);}- // If the element is interactable and has an interactable parent,- // then add it to the children of the parent+ // Otherwise add it to the children of the parent via O(1) map lookupelse {- // TODO: use dict/object so that we access these in O(1) instead- elements- .find((element) => element.id === parentId)- .children.push(elementObj);+ const parentObj = elementIdMap.get(parentId);+ if (parentObj) {+ parentObj.children.push(elementObj);+ }}parentId = elementObj.id;}@@ -1909,24 +1952,51 @@ async function buildElementTree(return;}+ let text = element.text;+ if (!text) {+ // Still recurse into children+ for (let i = 0; i < element.children.length; i++) {+ trimDuplicatedText(element.children[i]);+ }+ return;+ }++ // Collect all substrings to remove, then remove them in one pass+ const textsToRemove = [];+// if the element has options, text will be duplicated with the option textif (element.options) {- element.options.forEach((option) => {- element.text = element.text.replace(option.text, "");- });+ for (let i = 0; i < element.options.length; i++) {+ if (element.options[i].text) {+ textsToRemove.push(element.options[i].text);+ }+ }}- // BFS to delete duplicated text- element.children.forEach((child) => {- // delete duplicated text in the tree- element.text = element.text.replace(child.text, "");- trimDuplicatedText(child);- });+ for (let i = 0; i < element.children.length; i++) {+ if (element.children[i].text) {+ textsToRemove.push(element.children[i].text);+ }+ }- // trim multiple ";"- element.text = element.text.replace(/;+/g, ";");- // trimleft and trimright ";"- element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");+ // Remove all collected substrings from parent text+ for (let i = 0; i < textsToRemove.length; i++) {+ const idx = text.indexOf(textsToRemove[i]);+ if (idx !== -1) {+ text =+ text.substring(0, idx) ++ text.substring(idx + textsToRemove[i].length);+ }+ }++ // Clean up semicolons in one pass using a single regex+ text = text.replace(/;+/g, ";").replace(/^;+|;+$/g, "");+ element.text = text;++ // Recurse into children+ for (let i = 0; i < element.children.length; i++) {+ trimDuplicatedText(element.children[i]);+ }};// some elements without children nodes should be removed out, such as <label>@@ -2068,14 +2138,19 @@ function groupElementsVisually(elements) {return groups;}-// Helper functions+// Helper functions — single-pass min/max avoids 4 intermediate arrays + spreadfunction calculateBounds(elements) {- const rects = elements.map((el) => el.rect);- const left = Math.min(...rects.map((r) => r.left));- const top = Math.min(...rects.map((r) => r.top));- const right = Math.max(...rects.map((r) => r.right));- const bottom = Math.max(...rects.map((r) => r.bottom));-+ let left = Infinity,+ top = Infinity,+ right = -Infinity,+ bottom = -Infinity;+ for (let i = 0; i < elements.length; i++) {+ const r = elements[i].rect;+ if (r.left < left) left = r.left;+ if (r.top < top) top = r.top;+ if (r.right > right) right = r.right;+ if (r.bottom > bottom) bottom = r.bottom;+ }return {x: left,y: top,@@ -2110,11 +2185,18 @@ function findOverlappingElements(element, allElements, quadTree, processed) {}function createRectangleForGroup(group) {- const rects = group.elements.map((element) => element.rect);- const top = Math.min(...rects.map((rect) => rect.top));- const left = Math.min(...rects.map((rect) => rect.left));- const bottom = Math.max(...rects.map((rect) => rect.bottom));- const right = Math.max(...rects.map((rect) => rect.right));+ const elems = group.elements;+ let top = Infinity,+ left = Infinity,+ bottom = -Infinity,+ right = -Infinity;+ for (let i = 0; i < elems.length; i++) {+ const r = elems[i].rect;+ if (r.top < top) top = r.top;+ if (r.left < left) left = r.left;+ if (r.bottom > bottom) bottom = r.bottom;+ if (r.right > right) right = r.right;+ }return Rect.create(left, top, right, bottom);}@@ -2427,12 +2509,16 @@ function scrollToElementTop(element) {*/async function getHoverStylesMap() {const hoverMap = new Map();- const sheets = [...document.styleSheets];+ const sheets = Array.from(document.styleSheets);const parseCssSheet = (sheet) => {const rules = sheet.cssRules || sheet.rules;for (const rule of rules) {if (rule.type === 1 && rule.selectorText) {+ // Fast-path: skip rules that don't contain :hover at all+ if (!rule.selectorText.includes(":hover")) {+ continue;+ }// Split multiple selectors (e.g., "a:hover, button:hover")const selectors = rule.selectorText.split(",").map((s) => s.trim());@@ -2454,8 +2540,17 @@ async function getHoverStylesMap() {continue;}+ // Early check: only keep rules that set cursor (or already have it from prior rules)+ const existingStyles = hoverMap.get(baseSelector);+ const hasCursorInRule = rule.style.cursor;+ const hasCursorInExisting =+ existingStyles && "cursor" in existingStyles;+ if (!hasCursorInRule && !hasCursorInExisting) {+ continue;+ }+// Get or create styles object for this selector- let styles = hoverMap.get(baseSelector) || {};+ let styles = existingStyles || {};// Add all style propertiesfor (const prop of rule.style) {@@ -2467,18 +2562,16 @@ async function getHoverStylesMap() {if (parts.length > 1) {const fullSelector = selector;styles["__nested__"] = styles["__nested__"] || [];+ const nestedStyles = {};+ for (const prop of rule.style) {+ nestedStyles[prop] = rule.style[prop];+ }styles["__nested__"].push({selector: fullSelector,- styles: Object.fromEntries(- [...rule.style].map((prop) => [prop, rule.style[prop]]),- ),+ styles: nestedStyles,});}- // only need the style which includes the cursor attribute.- if (!("cursor" in styles)) {- continue;- }hoverMap.set(baseSelector, styles);}}@@ -2508,15 +2601,35 @@ async function getHoverStylesMap() {url.searchParams.set("v", Date.now());newLink.href = url.toString();newLink.crossOrigin = "anonymous";- // until the new link loaded, removing the old onedocument.head.append(newLink);- // wait for a while until the sheet is fully loaded- await asyncSleepFor(1500);- const newSheets = [...document.styleSheets];- const refreshedSheet = newSheets.find(- (s) => s.href === newLink.href,- );+ // Wait for the stylesheet to load via event instead of fixed 1500ms sleep+ await new Promise((resolve) => {+ const timeout = setTimeout(() => {+ _jsConsoleWarn(+ "Stylesheet load timed out after 3s:",+ newLink.href,+ );+ resolve();+ }, 3000);+ newLink.addEventListener("load", () => {+ clearTimeout(timeout);+ resolve();+ });+ newLink.addEventListener("error", () => {+ clearTimeout(timeout);+ resolve();+ });+ });++ const allSheets = document.styleSheets;+ let refreshedSheet = null;+ for (let si = 0; si < allSheets.length; si++) {+ if (allSheets[si].href === newLink.href) {+ refreshedSheet = allSheets[si];+ break;+ }+ }if (!refreshedSheet) {newLink.remove();return;@@ -2578,19 +2691,18 @@ if (window.globalDomDepthMap === undefined) {function isClassNameIncludesHidden(className) {// some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`+ const lower = className.toLowerCase();return (- className.toLowerCase().includes("hide") ||- className.toLowerCase().includes("invisible") ||- className.toLowerCase().includes("closed")+ lower.includes("hide") ||+ lower.includes("invisible") ||+ lower.includes("closed"));}function isClassNameIncludesActivatedStatus(className) {// some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click- return (- className.toLowerCase().includes("open") ||- className.toLowerCase().includes("active")- );+ const lower = className.toLowerCase();+ return lower.includes("open") || lower.includes("active");}function waitForNextFrame() {@@ -2926,21 +3038,25 @@ function isAnimationFinished() {* This includes elements in the main document and shadow DOM.*/function removeAllUniqueIds() {- // Function to recursively remove unique_id from an element and its children+ // Function to recursively remove unique_id from an element and its children.+ // We iterate element.children directly (live HTMLCollection) since+ // removeAttribute only modifies attributes, not child structure.const removeUniqueIdFromElement = (element) => {if (element.hasAttribute("unique_id")) {element.removeAttribute("unique_id");}// Process children in the main DOM- for (const child of Array.from(element.children)) {- removeUniqueIdFromElement(child);+ const children = element.children;+ for (let i = 0; i < children.length; i++) {+ removeUniqueIdFromElement(children[i]);}// Process elements in shadow DOM if presentif (element.shadowRoot) {- for (const shadowChild of Array.from(element.shadowRoot.children)) {- removeUniqueIdFromElement(shadowChild);+ const shadowChildren = element.shadowRoot.children;+ for (let i = 0; i < shadowChildren.length; i++) {+ removeUniqueIdFromElement(shadowChildren[i]);}}};diff --git a/skyvern/webeye/scraper/domUtils_legacy.js b/skyvern/webeye/scraper/domUtils_legacy.jsnew file mode 100644index 000000000..7cd6804c1--- /dev/null+++ b/skyvern/webeye/scraper/domUtils_legacy.js@@ -0,0 +1,2972 @@+// LEGACY FROZEN BASELINE — do not modify. See domUtils.js for the optimized version.+// we only use chromium browser for now+let browserNameForWorkarounds = "chromium";++let _jsConsoleLog = console?.log ?? function () {}; // prevent no console.log error+let _jsConsoleError = console?.error ?? _jsConsoleLog;+let _jsConsoleWarn = console?.warn ?? _jsConsoleLog;++class SafeCounter {+ constructor() {+ this.value = 0;+ this.lock = Promise.resolve();+ }++ async add() {+ await this.lock;+ this.lock = new Promise((resolve) => {+ this.value += 1;+ resolve();+ });+ return this.value;+ }++ async get() {+ await this.lock;+ return this.value;+ }+}++// Commands for manipulating rects.+// Want to debug this? Run chromium, go to sources, and create a new snippet with the code in domUtils.js+class Rect {+ // Create a rect given the top left and bottom right corners.+ static create(x1, y1, x2, y2) {+ return {+ bottom: y2,+ top: y1,+ left: x1,+ right: x2,+ width: x2 - x1,+ height: y2 - y1,+ };+ }++ static copy(rect) {+ return {+ bottom: rect.bottom,+ top: rect.top,+ left: rect.left,+ right: rect.right,+ width: rect.width,+ height: rect.height,+ };+ }++ // Translate a rect by x horizontally and y vertically.+ static translate(rect, x, y) {+ if (x == null) x = 0;+ if (y == null) y = 0;+ return {+ bottom: rect.bottom + y,+ top: rect.top + y,+ left: rect.left + x,+ right: rect.right + x,+ width: rect.width,+ height: rect.height,+ };+ }++ // Determine whether two rects overlap.+ static intersects(rect1, rect2) {+ return (+ rect1.right > rect2.left &&+ rect1.left < rect2.right &&+ rect1.bottom > rect2.top &&+ rect1.top < rect2.bottom+ );+ }++ static equals(rect1, rect2) {+ for (const property of [+ "top",+ "bottom",+ "left",+ "right",+ "width",+ "height",+ ]) {+ if (rect1[property] !== rect2[property]) return false;+ }+ return true;+ }+}++class DomUtils {+ static elementListCache = [];+ static visibleClientRectCache = new WeakMap();+ //+ // Bounds the rect by the current viewport dimensions. If the rect is offscreen or has a height or+ // width < 3 then null is returned instead of a rect.+ //+ static cropRectToVisible(rect) {+ const boundedRect = Rect.create(+ Math.max(rect.left, 0),+ Math.max(rect.top, 0),+ rect.right,+ rect.bottom,+ );+ if (+ boundedRect.top >= window.innerHeight - 4 ||+ boundedRect.left >= window.innerWidth - 4+ ) {+ return null;+ } else {+ return boundedRect;+ }+ }++ // add cache to optimize performance+ static getVisibleClientRect(element, testChildren = false) {+ // check cache+ const cacheKey = `${testChildren}`;+ if (DomUtils.visibleClientRectCache.has(element)) {+ const elementCache = DomUtils.visibleClientRectCache.get(element);+ if (elementCache.has(cacheKey)) {+ _jsConsoleLog("hit cache to get the rect of element");+ return elementCache.get(cacheKey);+ }+ }++ // Note: this call will be expensive if we modify the DOM in between calls.+ let clientRect;+ const clientRects = (() => {+ const result = [];+ for (clientRect of element.getClientRects()) {+ result.push(Rect.copy(clientRect));+ }+ return result;+ })();++ // Inline elements with font-size: 0px; will declare a height of zero, even if a child with+ // non-zero font-size contains text.+ let isInlineZeroHeight = function () {+ const elementComputedStyle = getElementComputedStyle(element, null);+ const isInlineZeroFontSize =+ 0 ===+ elementComputedStyle?.getPropertyValue("display").indexOf("inline") &&+ elementComputedStyle?.getPropertyValue("font-size") === "0px";+ // Override the function to return this value for the rest of this context.+ isInlineZeroHeight = () => isInlineZeroFontSize;+ return isInlineZeroFontSize;+ };++ let result = null;++ for (clientRect of clientRects) {+ // If the link has zero dimensions, it may be wrapping visible but floated elements. Check for+ // this.+ let computedStyle;+ if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {+ for (const child of Array.from(element.children)) {+ computedStyle = getElementComputedStyle(child, null);+ if (!computedStyle) {+ continue;+ }+ // Ignore child elements which are not floated and not absolutely positioned for parent+ // elements with zero width/height, as long as the case described at isInlineZeroHeight+ // does not apply.+ // NOTE(mrmr1993): This ignores floated/absolutely positioned descendants nested within+ // inline children.+ const position = computedStyle.getPropertyValue("position");+ if (+ computedStyle.getPropertyValue("float") === "none" &&+ !["absolute", "fixed"].includes(position) &&+ !(+ clientRect.height === 0 &&+ isInlineZeroHeight() &&+ 0 === computedStyle.getPropertyValue("display").indexOf("inline")+ )+ ) {+ continue;+ }+ const childClientRect = this.getVisibleClientRect(child, true);+ if (+ childClientRect === null ||+ childClientRect.width < 3 ||+ childClientRect.height < 3+ )+ continue;+ result = childClientRect;+ break;+ }+ if (result) break;+ } else {+ clientRect = this.cropRectToVisible(clientRect);++ if (+ clientRect === null ||+ clientRect.width < 3 ||+ clientRect.height < 3+ )+ continue;++ // eliminate invisible elements (see test_harnesses/visibility_test.html)+ computedStyle = getElementComputedStyle(element, null);+ if (!computedStyle) {+ continue;+ }+ if (computedStyle.getPropertyValue("visibility") !== "visible")+ continue;++ result = clientRect;+ break;+ }+ }++ // cache result+ if (!DomUtils.visibleClientRectCache.has(element)) {+ DomUtils.visibleClientRectCache.set(element, new Map());+ }+ DomUtils.visibleClientRectCache.get(element).set(cacheKey, result);++ return result;+ }++ // clear cache+ static clearVisibleClientRectCache() {+ DomUtils.visibleClientRectCache = new WeakMap();+ }++ static getViewportTopLeft() {+ const box = document.documentElement;+ const style = getComputedStyle(box);+ const rect = box.getBoundingClientRect();+ if (+ style &&+ style.position === "static" &&+ !/content|paint|strict/.test(style.contain || "")+ ) {+ // The margin is included in the client rect, so we need to subtract it back out.+ const marginTop = parseInt(style.marginTop);+ const marginLeft = parseInt(style.marginLeft);+ return {+ top: -rect.top + marginTop,+ left: -rect.left + marginLeft,+ };+ } else {+ const { clientTop, clientLeft } = box;+ return {+ top: -rect.top - clientTop,+ left: -rect.left - clientLeft,+ };+ }+ }+}++class QuadTreeNode {+ constructor(bounds, maxElements = 10, maxDepth = 4) {+ this.bounds = bounds; // {x, y, width, height}+ this.maxElements = maxElements;+ this.maxDepth = maxDepth;+ this.elements = [];+ this.children = null;+ this.depth = 0;+ }++ insert(element) {+ if (!this.contains(element.rect)) {+ return false;+ }++ if (this.children === null && this.elements.length < this.maxElements) {+ this.elements.push(element);+ return true;+ }++ if (this.children === null) {+ this.subdivide();+ }++ for (const child of this.children) {+ if (child.insert(element)) {+ return true;+ }+ }++ this.elements.push(element);+ return true;+ }++ subdivide() {+ const x = this.bounds.x;+ const y = this.bounds.y;+ const w = this.bounds.width / 2;+ const h = this.bounds.height / 2;++ this.children = [+ new QuadTreeNode(+ { x, y, width: w, height: h },+ this.maxElements,+ this.maxDepth,+ ),+ new QuadTreeNode(+ { x: x + w, y, width: w, height: h },+ this.maxElements,+ this.maxDepth,+ ),+ new QuadTreeNode(+ { x, y: y + h, width: w, height: h },+ this.maxElements,+ this.maxDepth,+ ),+ new QuadTreeNode(+ { x: x + w, y: y + h, width: w, height: h },+ this.maxElements,+ this.maxDepth,+ ),+ ];++ for (const child of this.children) {+ child.depth = this.depth + 1;+ }+ }++ contains(rect) {+ return (+ rect.left >= this.bounds.x &&+ rect.right <= this.bounds.x + this.bounds.width &&+ rect.top >= this.bounds.y &&+ rect.bottom <= this.bounds.y + this.bounds.height+ );+ }++ query(rect) {+ const result = [];+ this.queryRecursive(rect, result);+ return result;+ }++ queryRecursive(rect, result) {+ if (!this.intersects(rect)) {+ return;+ }++ result.push(...this.elements);++ if (this.children) {+ for (const child of this.children) {+ child.queryRecursive(rect, result);+ }+ }+ }++ intersects(rect) {+ return (+ rect.left < this.bounds.x + this.bounds.width &&+ rect.right > this.bounds.x &&+ rect.top < this.bounds.y + this.bounds.height &&+ rect.bottom > this.bounds.y+ );+ }+}++// from playwright+function getElementComputedStyle(element, pseudo) {+ return element.ownerDocument && element.ownerDocument.defaultView+ ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)+ : undefined;+}++// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98+function isElementStyleVisibilityVisible(element, style) {+ style = style ?? getElementComputedStyle(element);+ if (!style) return true;+ // Element.checkVisibility checks for content-visibility and also looks at+ // styles up the flat tree including user-agent ShadowRoots, such as the+ // details element for example.+ // All the browser implement it, but WebKit has a bug which prevents us from using it:+ // https://bugs.webkit.org/show_bug.cgi?id=264733+ // @ts-ignore+ if (+ Element.prototype.checkVisibility &&+ browserNameForWorkarounds !== "webkit"+ ) {+ if (!element.checkVisibility()) return false;+ } else {+ // Manual workaround for WebKit that does not have checkVisibility.+ const detailsOrSummary = element.closest("details,summary");+ if (+ detailsOrSummary !== element &&+ detailsOrSummary?.nodeName === "DETAILS" &&+ !detailsOrSummary.open+ )+ return false;+ }+ if (style.visibility !== "visible") return false;++ // TODO: support style.clipPath and style.clipRule?+ // if element is clipped with rect(0px, 0px, 0px, 0px), it means it's invisible on the page+ // FIXME: need a better algorithm to calculate the visible rect area, using (right-left)*(bottom-top) from rect(top, right, bottom, left)+ if (+ style.clip === "rect(0px, 0px, 0px, 0px)" ||+ style.clip === "rect(1px, 1px, 1px, 1px)"+ ) {+ return false;+ }++ return true;+}++function hasASPClientControl() {+ return typeof ASPxClientControl !== "undefined";+}++// Check if element is only visible on hover (e.g., hover-only buttons)+function isHoverOnlyElement(element) {+ // Check for common hover-only patterns in class names+ const className = element.className?.toString() ?? "";+ const parentClassName = element.parentElement?.className?.toString() ?? "";++ // Common hover-only class patterns+ if (+ className.includes("hover-") ||+ className.includes("-hover") ||+ parentClassName.includes("hover-") ||+ parentClassName.includes("-hover")+ ) {+ return true;+ }++ // Check if parent has hover-related attributes or classes that might reveal this element+ let parent = element.parentElement;+ let depth = 0;+ // Cap recursion to avoid walking the entire tree and bloating prompts+ const maxDepth = 5;+ while (parent && parent !== document.body && depth < maxDepth) {+ const parentClass = parent.className?.toString() ?? "";+ if (+ parentClass.includes("hover") ||+ parentClass.includes("card") ||+ parentClass.includes("item")+ ) {+ // This element might be revealed on parent hover+ return true;+ }+ parent = parent.parentElement;+ depth += 1;+ }++ return false;+}++// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119+// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.+function isElementVisible(element) {+ // TODO: This is a hack to not check visibility for option elements+ // because they are not visible by default. We check their parent instead for visibility.+ if (+ element.tagName.toLowerCase() === "option" ||+ (element.tagName.toLowerCase() === "input" &&+ (element.type === "radio" || element.type === "checkbox"))+ )+ return element.parentElement && isElementVisible(element.parentElement);++ const className = element.className ? element.className.toString() : "";+ if (+ className.includes("select2-offscreen") ||+ className.includes("select2-hidden") ||+ className.includes("ui-select-offscreen")+ ) {+ return false;+ }++ const style = getElementComputedStyle(element);+ if (!style) return true;+ if (style.display === "contents") {+ // display:contents is not rendered itself, but its child nodes are.+ for (let child = element.firstChild; child; child = child.nextSibling) {+ if (+ child.nodeType === 1 /* Node.ELEMENT_NODE */ &&+ isElementVisible(child)+ )+ return true;+ if (child.nodeType === 3 /* Node.TEXT_NODE */ && isVisibleTextNode(child))+ return true;+ }+ return false;+ }+ if (!isElementStyleVisibilityVisible(element, style)) return false;+ const rect = element.getBoundingClientRect();+ if (rect.width <= 0 || rect.height <= 0) {+ // Check if this element might be visible on hover before marking as invisible+ if (isHoverOnlyElement(element)) {+ return true;+ }+ return false;+ }++ // if the center point of the element is not in the page, we tag it as an non-interactable element+ // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X+ const center_x = (rect.left + rect.width) / 2 + window.scrollX;+ if (center_x < 0) {+ return false;+ }+ // const center_y = (rect.top + rect.height) / 2 + window.scrollY;+ // if (center_x < 0 || center_y < 0) {+ // return false;+ // }++ return true;+}++// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L121-L127+function isVisibleTextNode(node) {+ // https://stackoverflow.com/questions/1461059/is-there-an-equivalent-to-getboundingclientrect-for-text-nodes+ const range = node.ownerDocument.createRange();+ range.selectNode(node);+ const rect = range.getBoundingClientRect();+ if (rect.width <= 0 || rect.height <= 0) {+ return false;+ }++ // if the center point of the element is not in the page, we tag it as an non-interactable element+ // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X+ const center_x = (rect.left + rect.width) / 2 + window.scrollX;+ if (center_x < 0) {+ return false;+ }+ // const center_y = (rect.top + rect.height) / 2 + window.scrollY;+ // if (center_x < 0 || center_y < 0) {+ // return false;+ // }+ return true;+}++// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L37-L44+function parentElementOrShadowHost(element) {+ if (element.parentElement) return element.parentElement;+ if (!element.parentNode) return;+ if (+ element.parentNode.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ &&+ element.parentNode.host+ )+ return element.parentNode.host;+}++// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L46-L52+function enclosingShadowRootOrDocument(element) {+ let node = element;+ while (node.parentNode) node = node.parentNode;+ if (+ node.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ ||+ node.nodeType === 9 /* Node.DOCUMENT_NODE */+ )+ return node;+}++// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/injectedScript.ts#L799-L859+function expectHitTarget(hitPoint, targetElement) {+ const roots = [];++ // Get all component roots leading to the target element.+ // Go from the bottom to the top to make it work with closed shadow roots.+ let parentElement = targetElement;+ while (parentElement) {+ const root = enclosingShadowRootOrDocument(parentElement);+ if (!root) break;+ roots.push(root);+ if (root.nodeType === 9 /* Node.DOCUMENT_NODE */) break;+ parentElement = root.host;+ }++ // Hit target in each component root should point to the next component root.+ // Hit target in the last component root should point to the target or its descendant.+ let hitElement;+ for (let index = roots.length - 1; index >= 0; index--) {+ const root = roots[index];+ // All browsers have different behavior around elementFromPoint and elementsFromPoint.+ // https://github.com/w3c/csswg-drafts/issues/556+ // http://crbug.com/1188919+ const elements = root.elementsFromPoint(hitPoint.x, hitPoint.y);+ const singleElement = root.elementFromPoint(hitPoint.x, hitPoint.y);+ if (+ singleElement &&+ elements[0] &&+ parentElementOrShadowHost(singleElement) === elements[0]+ ) {+ const style = getElementComputedStyle(singleElement);+ if (style?.display === "contents") {+ // Workaround a case where elementsFromPoint misses the inner-most element with display:contents.+ // https://bugs.chromium.org/p/chromium/issues/detail?id=1342092+ elements.unshift(singleElement);+ }+ }+ if (+ elements[0] &&+ elements[0].shadowRoot === root &&+ elements[1] === singleElement+ ) {+ // Workaround webkit but where first two elements are swapped:+ // <host>+ // #shadow root+ // <target>+ // elementsFromPoint produces [<host>, <target>], while it should be [<target>, <host>]+ // In this case, just ignore <host>.+ elements.shift();+ }+ const innerElement = elements[0];+ if (!innerElement) break;+ hitElement = innerElement;+ if (index && innerElement !== roots[index - 1].host) break;+ }++ // Check whether hit target is the target or its descendant.+ const hitParents = [];+ while (hitElement && hitElement !== targetElement) {+ hitParents.push(hitElement);+ hitElement = parentElementOrShadowHost(hitElement);+ }+ if (hitElement === targetElement) return null;++ return hitParents[0] || document.documentElement;+}++function getChildElements(element) {+ if (element.childElementCount !== 0) {+ return Array.from(element.children);+ } else {+ return [];+ }+}++function isParent(parent, child) {+ return parent.contains(child);+}++function isSibling(el1, el2) {+ return el1.parentElement === el2.parentElement;+}++function getBlockElementUniqueID(element) {+ const rect = element.getBoundingClientRect();++ const hitElement = expectHitTarget(+ {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ },+ element,+ );++ if (!hitElement) {+ return ["", false];+ }++ return [hitElement.getAttribute("unique_id") ?? "", true];+}++function isHidden(element) {+ const style = getElementComputedStyle(element);+ if (style?.display === "none") {+ return true;+ }+ if (element.hidden) {+ if (+ style?.cursor === "pointer" &&+ element.tagName.toLowerCase() === "input" &&+ (element.type === "submit" || element.type === "button")+ ) {+ // there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.+ // such an element is not really hidden+ return false;+ }+ return true;+ }+ return false;+}++function isHiddenOrDisabled(element) {+ return isHidden(element) || element.disabled;+}++function isScriptOrStyle(element) {+ const tagName = element.tagName.toLowerCase();+ return tagName === "script" || tagName === "style";+}++function isReadonlyElement(element) {+ if (element.readOnly) {+ return true;+ }++ if (element.hasAttribute("readonly")) {+ return true;+ }++ if (element.hasAttribute("aria-readonly")) {+ // only aria-readonly="false" should be considered as "not readonly"+ return (+ element.getAttribute("aria-readonly").toLowerCase().trim() !== "false"+ );+ }++ return false;+}++function isDropdownRelatedElement(element) {+ const tagName = element.tagName?.toLowerCase();+ if (tagName === "select") {+ return true;+ }++ const role = element.getAttribute("role")?.toLowerCase();+ if (role === "option" || role === "listbox") {+ return true;+ }++ return false;+}++function hasAngularClickBinding(element) {+ return (+ element.hasAttribute("ng-click") || element.hasAttribute("data-ng-click")+ );+}++function hasWidgetRole(element) {+ const role = element.getAttribute("role");+ if (!role) {+ return false;+ }+ // https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles+ // Not all roles make sense for the time being so we only check for the ones that do+ if (role.toLowerCase().trim() === "textbox") {+ return !isReadonlyElement(element);+ }++ const widgetRoles = [+ "button",+ "link",+ "checkbox",+ "menuitem",+ "menuitemcheckbox",+ "menuitemradio",+ "radio",+ "tab",+ "combobox",+ "searchbox",+ "slider",+ "spinbutton",+ "switch",+ "gridcell",+ "option",+ ];+ return widgetRoles.includes(role.toLowerCase().trim());+}++function isTableRelatedElement(element) {+ const tagName = element.tagName.toLowerCase();+ return [+ "table",+ "caption",+ "thead",+ "tbody",+ "tfoot",+ "tr",+ "th",+ "td",+ "colgroup",+ "col",+ ].includes(tagName);+}++function isDOMNodeRepresentDiv(element) {+ if (element?.tagName?.toLowerCase() !== "div") {+ return false;+ }+ const style = getElementComputedStyle(element);+ const children = getChildElements(element);+ // flex usually means there are multiple elements in the div as a line or a column+ // if the children elements are not just one, we should keep it in the HTML tree to represent a tree structure+ if (style?.display === "flex" && children.length > 1) {+ return true;+ }+ return false;+}++function isHoverPointerElement(element, hoverStylesMap) {+ const tagName = element.tagName.toLowerCase();+ const elementClassName = element.className.toString();+ const elementCursor = getElementComputedStyle(element)?.cursor;+ if (elementCursor === "pointer") {+ return true;+ }++ // Check if element has hover styles that change cursor to pointer+ // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover+ if (elementCursor === "auto" || elementCursor === "default") {+ // TODO: we need a better algorithm to match the selector with better performance+ for (const [selector, styles] of hoverStylesMap) {+ let shouldMatch = false;+ for (const className of element.classList) {+ if (selector.includes(className)) {+ shouldMatch = true;+ break;+ }+ }+ if (shouldMatch || selector.includes(tagName)) {+ if (element.matches(selector) && styles.cursor === "pointer") {+ return true;+ }+ }+ }+ }++ // FIXME: hardcode to fix the bug about hover style now+ if (elementClassName.includes("hover:cursor-pointer")) {+ return true;+ }++ return false;+}++function isInteractableInput(element, hoverStylesMap) {+ const tagName = element.tagName.toLowerCase();+ if (tagName !== "input") {+ // let other checks decide+ return false;+ }+ // Browsers default to "text" when the type is not set or is invalid+ // Here's the list of valid types: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input#input_types+ // Examples of unrecognized types that we've seen and caused issues because we didn't mark them interactable:+ // "city", "state", "zip", "country"+ // That's the reason I (Kerem) removed the valid input types check+ var type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";+ return isHoverPointerElement(element, hoverStylesMap) || type !== "hidden";+}++function isValidCSSSelector(selector) {+ try {+ document.querySelector(selector);+ return true;+ } catch (e) {+ return false;+ }+}++function isInteractable(element, hoverStylesMap) {+ if (!isElementVisible(element)) {+ return false;+ }++ if (isHidden(element)) {+ return false;+ }++ if (isScriptOrStyle(element)) {+ return false;+ }++ if (hasWidgetRole(element)) {+ return true;+ }++ // element with pointer-events: none should not be considered as interactable+ // but for elements which are disabled, we should not use this logic to test the interactable+ // https://developer.mozilla.org/en-US/docs/Web/CSS/pointer-events#none+ const elementPointerEvent = getElementComputedStyle(element)?.pointerEvents;+ if (elementPointerEvent === "none" && !element.disabled) {+ // Some CTAs stay hidden until the parent is hovered+ // When we can infer that the element is revealed on hover, keep it interactable so the agent+ // has a chance to hover the parent before clicking.+ if (!isHoverOnlyElement(element)) {+ return false;+ }+ }++ if (isInteractableInput(element, hoverStylesMap)) {+ return true;+ }++ const tagName = element.tagName.toLowerCase();+ if (tagName === "html") {+ return false;+ }++ if (tagName === "iframe") {+ return false;+ }++ if (tagName === "frameset") {+ return false;+ }++ if (tagName === "frame") {+ return false;+ }++ if (tagName === "a" && element.href) {+ return true;+ }++ // Check if the option's parent (select) is hidden or disabled+ if (tagName === "option" && isHiddenOrDisabled(element.parentElement)) {+ return false;+ }++ if (+ tagName === "button" ||+ tagName === "select" ||+ tagName === "option" ||+ tagName === "textarea"+ ) {+ return true;+ }++ if (tagName === "label" && element.control && !element.control.disabled) {+ return true;+ }++ if (+ element.hasAttribute("onclick") ||+ element.isContentEditable ||+ element.hasAttribute("jsaction")+ ) {+ return true;+ }++ const className = element.className?.toString() ?? "";++ if (tagName === "div" || tagName === "span") {+ if (hasAngularClickBinding(element)) {+ return true;+ }+ if (className.includes("blinking-cursor")) {+ return true;+ }+ // https://www.oxygenxml.com/dita/1.3/specs/langRef/technicalContent/svg-container.html+ // svg-container is usually used for clickable elements that wrap SVGs+ if (className.includes("svg-container")) {+ return true;+ }+ }++ // support listbox and options underneath it+ // div element should be checked here before the css pointer+ if (+ (tagName === "ul" || tagName === "div") &&+ element.hasAttribute("role") &&+ element.getAttribute("role").toLowerCase() === "listbox"+ ) {+ return true;+ }+ if (+ (tagName === "li" || tagName === "div") &&+ element.hasAttribute("role") &&+ element.getAttribute("role").toLowerCase() === "option"+ ) {+ return true;+ }++ if (+ tagName === "li" &&+ (className.includes("ui-menu-item") ||+ className.includes("dropdown-item") ||+ className === "option" ||+ className.includes("rddlItem"))+ ) {+ return true;+ }++ // google map address auto complete+ // https://developers.google.com/maps/documentation/javascript/place-autocomplete#style-autocomplete+ // demo: https://developers.google.com/maps/documentation/javascript/examples/places-autocomplete-addressform+ if (+ tagName === "div" &&+ className.includes("pac-item") &&+ element.closest('div[class*="pac-container"]')+ ) {+ return true;+ }++ if (+ tagName === "div" &&+ element.hasAttribute("aria-disabled") &&+ element.getAttribute("aria-disabled").toLowerCase() === "false"+ ) {+ return true;+ }++ if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {+ return true;+ }++ // FIXME: maybe we need to enable the pointer check for all elements?+ if (+ tagName === "div" ||+ tagName === "img" ||+ tagName === "span" ||+ tagName === "a" ||+ tagName === "i" ||+ tagName === "li" ||+ tagName === "p" ||+ tagName === "td" ||+ tagName === "svg" ||+ tagName === "strong" ||+ tagName === "h1" ||+ tagName === "h2" ||+ tagName === "h3" ||+ tagName === "h4" ||+ // sometime it's a customized element like <my-login-button>, we should check pointer style+ tagName.includes("button") ||+ tagName.includes("select") ||+ tagName.includes("option") ||+ tagName.includes("textarea")+ ) {+ if (isHoverPointerElement(element, hoverStylesMap)) {+ return true;+ }+ }++ if (hasASPClientControl() && tagName === "tr") {+ return true;+ }++ if (tagName === "div" && element.hasAttribute("data-selectable")) {+ return true;+ }++ try {+ if (window.jQuery && window.jQuery._data) {+ const events = window.jQuery._data(element, "events");+ if (events && "click" in events) {+ return true;+ }+ }+ } catch (e) {+ _jsConsoleError("Error getting jQuery click events:", e);+ }++ try {+ if (hasAngularClickEvent(element)) {+ return true;+ }+ } catch (e) {+ _jsConsoleError("Error checking angular click event:", e);+ }++ // consider <div tabindex="0"> as interactable+ if (+ tagName.toLowerCase() === "div" &&+ element.hasAttribute("tabindex") &&+ element.getAttribute("tabindex").toLowerCase() === "0"+ ) {+ return true;+ }++ return false;+}++function isScrollable(element) {+ const scrollHeight = element.scrollHeight || 0;+ const clientHeight = element.clientHeight || 0;+ const scrollWidth = element.scrollWidth || 0;+ const clientWidth = element.clientWidth || 0;++ const hasScrollableContent =+ scrollHeight > clientHeight || scrollWidth > clientWidth;+ const hasScrollableOverflow = isScrollableOverflow(element);+ return hasScrollableContent && hasScrollableOverflow;+}++function isScrollableOverflow(element) {+ const style = getElementComputedStyle(element);+ if (!style) {+ return false;+ }+ return (+ style.overflow === "auto" ||+ style.overflow === "scroll" ||+ style.overflowX === "auto" ||+ style.overflowX === "scroll" ||+ style.overflowY === "auto" ||+ style.overflowY === "scroll"+ );+}++function isDatePickerSelector(element) {+ const tagName = element.tagName.toLowerCase();+ if (+ tagName === "button" &&+ element.getAttribute("data-testid")?.includes("date")+ ) {+ return true;+ }+ return false;+}++const isComboboxDropdown = (element) => {+ if (element.tagName.toLowerCase() !== "input") {+ return false;+ }+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";+ const readonly =+ element.getAttribute("readonly") &&+ element.getAttribute("readonly").toLowerCase() !== "false";+ const controls = element.hasAttribute("aria-controls");+ return role && haspopup && controls && readonly;+};++const isDivComboboxDropdown = (element) => {+ const tagName = element.tagName.toLowerCase();+ if (tagName !== "div") {+ return false;+ }+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";+ const controls = element.hasAttribute("aria-controls");+ return role === "combobox" && controls && haspopup;+};++const isDropdownButton = (element) => {+ const tagName = element.tagName.toLowerCase();+ const type = element.getAttribute("type")+ ? element.getAttribute("type").toLowerCase()+ : "";+ const haspopup = element.getAttribute("aria-haspopup")+ ? element.getAttribute("aria-haspopup").toLowerCase()+ : "";+ const hasExpanded = element.hasAttribute("aria-expanded");+ return (+ tagName === "button" &&+ type === "button" &&+ (hasExpanded || haspopup === "listbox")+ );+};++const isSelect2Dropdown = (element) => {+ const tagName = element.tagName.toLowerCase();+ const className = element.className.toString();+ const role = element.getAttribute("role")+ ? element.getAttribute("role").toLowerCase()+ : "";++ if (tagName === "a") {+ return className.includes("select2-choice");+ }++ if (tagName === "span") {+ return className.includes("select2-selection") && role === "combobox";+ }++ return false;+};++const isSelect2MultiChoice = (element) => {+ return (+ element.tagName.toLowerCase() === "input" &&+ element.className.toString().includes("select2-input")+ );+};++const isReactSelectDropdown = (element) => {+ return (+ element.tagName.toLowerCase() === "input" &&+ element.className.toString().includes("select__input") &&+ element.getAttribute("role") === "combobox"+ );+};++function isReadonlyInputDropdown(element) {+ const className = element.className?.toString() ?? "";+ return (+ element.tagName.toLowerCase() === "input" &&+ className.includes("custom-select") &&+ isReadonlyElement(element)+ );+}++function hasNgAttribute(element) {+ if (!element.attributes[Symbol.iterator]) {+ return false;+ }++ for (let attr of element.attributes) {+ if (attr.name.startsWith("ng-")) {+ return true;+ }+ }+ return false;+}++// TODO: it's a hack, should continue to optimize it+function hasAngularClickEvent(element) {+ const ctx = element.__ngContext__;+ const tView = ctx && ctx[1];+ if (!tView || !Array.isArray(tView.data)) {+ return false;+ }++ const tagName = element.tagName.toLowerCase();+ if (!tagName) {+ _jsConsoleLog("Element has no tag name: ", element);+ return false;+ }++ for (const tNode of tView.data) {+ if (!tNode || typeof tNode !== "object") continue;+ if (tNode.type !== 0 && tNode.type !== 2) continue; // 0: Element, 2: Container+ if (tNode.value && tagName !== tNode.value.toLowerCase()) continue;+ if (!Array.isArray(tNode.attrs)) continue;+ if (tNode.attrs.includes("click")) {+ return true;+ }+ }++ return false;+}++function isAngularMaterial(element) {+ if (!element.attributes[Symbol.iterator]) {+ return false;+ }++ for (let attr of element.attributes) {+ if (attr.name.startsWith("mat")) {+ return true;+ }+ }+ return false;+}++const isAngularDropdown = (element) => {+ if (!hasNgAttribute(element)) {+ return false;+ }++ if (element.type?.toLowerCase() === "search") {+ return false;+ }++ const tagName = element.tagName.toLowerCase();+ if (tagName === "input" || tagName === "span") {+ const ariaLabel = element.hasAttribute("aria-label")+ ? element.getAttribute("aria-label").toLowerCase()+ : "";+ return ariaLabel.includes("select") || ariaLabel.includes("choose");+ }++ return false;+};++const isAngularMaterialDatePicker = (element) => {+ if (!isAngularMaterial(element)) {+ return false;+ }++ const tagName = element.tagName.toLowerCase();+ if (tagName !== "input") return false;++ return (+ (element.closest("mat-datepicker") ||+ element.closest("mat-formio-date")) !== null+ );+};++function getPseudoContent(element, pseudo) {+ const pseudoStyle = getElementComputedStyle(element, pseudo);+ if (!pseudoStyle) {+ return null;+ }+ const content = pseudoStyle+ .getPropertyValue("content")+ .replace(/"/g, "")+ .trim();++ if (content === "none" || !content) {+ return null;+ }++ return content;+}++function hasBeforeOrAfterPseudoContent(element) {+ return (+ getPseudoContent(element, "::before") != null ||+ getPseudoContent(element, "::after") != null+ );+}++const checkParentClass = (className) => {+ const targetParentClasses = ["field", "entry"];+ for (let i = 0; i < targetParentClasses.length; i++) {+ if (className.includes(targetParentClasses[i])) {+ return true;+ }+ }+ return false;+};++function removeMultipleSpaces(str) {+ // Optimization: check for empty values early+ if (!str || str.length === 0) {+ return str;+ }++ // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement+ if (+ str.indexOf(" ") === -1 &&+ str.indexOf("\t") === -1 &&+ str.indexOf("\n") === -1+ ) {+ return str;+ }++ return str.replace(/\s+/g, " ");+}++function cleanupText(text) {+ // Optimization: check for empty values early to avoid unnecessary processing+ if (!text || text.length === 0) {+ return "";+ }++ // Optimization: use more efficient string replacement+ let cleanedText = text;++ // Remove specific SVG error message+ if (cleanedText.includes("SVGs not supported by this browser.")) {+ cleanedText = cleanedText.replace(+ "SVGs not supported by this browser.",+ "",+ );+ }++ // Optimization: combine space processing and trim operations+ return removeMultipleSpaces(cleanedText).trim();+}++const checkStringIncludeRequire = (str) => {+ return (+ str.toLowerCase().includes("*") ||+ str.toLowerCase().includes("✱") ||+ str.toLowerCase().includes("require")+ );+};++const checkRequiredFromStyle = (element) => {+ const afterCustomStyle = getElementComputedStyle(element, "::after");+ if (afterCustomStyle) {+ const afterCustom = afterCustomStyle+ .getPropertyValue("content")+ .replace(/"/g, "");+ if (checkStringIncludeRequire(afterCustom)) {+ return true;+ }+ }++ if (!element.className || typeof element.className !== "string") {+ return false;+ }++ return element.className.toLowerCase().includes("require");+};++function checkDisabledFromStyle(element) {+ const className = element.className.toString().toLowerCase();+ if (className.includes("react-datepicker__day--disabled")) {+ return true;+ }+ return false;+}++function getVisibleText(element) {+ let visibleText = [];++ function collectVisibleText(node) {+ if (+ node.nodeType === Node.TEXT_NODE &&+ isElementVisible(node.parentElement)+ ) {+ const trimmedText = node.data.trim();+ if (trimmedText.length > 0) {+ visibleText.push(trimmedText);+ }+ } else if (node.nodeType === Node.ELEMENT_NODE && isElementVisible(node)) {+ for (let child of node.childNodes) {+ collectVisibleText(child);+ }+ }+ }++ collectVisibleText(element);+ return visibleText.join(" ");+}++// only get text from element itself+function getElementText(element) {+ if (element.nodeType === Node.TEXT_NODE) {+ return element.data.trim();+ }++ const childNodes = element.childNodes;+ const childNodesLength = childNodes.length;++ // If no child nodes, return empty string directly+ if (childNodesLength === 0) {+ return "";+ }++ const visibleText = [];+ let hasText = false;++ for (let i = 0; i < childNodesLength; i++) {+ const node = childNodes[i];+ if (node.nodeType === Node.TEXT_NODE) {+ const nodeText = node.data.trim();+ if (nodeText.length > 0) {+ visibleText.push(nodeText);+ hasText = true;+ }+ }+ }++ return hasText ? visibleText.join(";") : "";+}++function getSelectOptions(element) {+ const options = Array.from(element.options);+ const selectOptions = [];++ for (const option of options) {+ selectOptions.push({+ optionIndex: option.index,+ text: removeMultipleSpaces(option.textContent),+ value: removeMultipleSpaces(option.value),+ });+ }++ const selectedOption = element.querySelector("option:checked");+ if (!selectedOption) {+ return [selectOptions, ""];+ }++ return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];+}++function getDOMElementBySkyvenElement(elementObj) {+ // if element has shadowHost set, we need to find the shadowHost element first then find the element+ if (elementObj.shadowHost) {+ let shadowHostEle = document.querySelector(+ `[unique_id="${elementObj.shadowHost}"]`,+ );+ if (!shadowHostEle) {+ _jsConsoleLog(+ "Could not find shadowHost element with unique_id: ",+ elementObj.shadowHost,+ );+ return null;+ }+ return shadowHostEle.shadowRoot.querySelector(+ `[unique_id="${elementObj.id}"]`,+ );+ }++ return document.querySelector(`[unique_id="${elementObj.id}"]`);+}++if (window.elementIdCounter === undefined) {+ window.elementIdCounter = new SafeCounter();+}++// generate a unique id for the element+// length is 4, the first character is from the frame index, the last 3 characters are from the counter,+async function uniqueId() {+ const characters =+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";+ const base = characters.length;++ const extraCharacters = "~!@#$%^&*()-_+=";+ const extraBase = extraCharacters.length;++ let result = "";++ if (+ window.GlobalSkyvernFrameIndex === undefined ||+ window.GlobalSkyvernFrameIndex < 0+ ) {+ const randomIndex = Math.floor(Math.random() * extraBase);+ result += extraCharacters[randomIndex];+ } else {+ const c1 = window.GlobalSkyvernFrameIndex % base;+ result += characters[c1];+ }++ const countPart =+ (await window.elementIdCounter.add()) % (base * base * base);+ const c2 = Math.floor(countPart / (base * base));+ result += characters[c2];+ const c3 = Math.floor(countPart / base) % base;+ result += characters[c3];+ const c4 = countPart % base;+ result += characters[c4];++ return result;+}++async function buildElementObject(+ frame,+ element,+ interactable,+ purgeable = false,+) {+ var element_id = element.getAttribute("unique_id") ?? (await uniqueId());+ var elementTagNameLower = element.tagName.toLowerCase();+ element.setAttribute("unique_id", element_id);++ const attrs = {};+ if (element.attributes[Symbol.iterator]) {+ for (const attr of element.attributes) {+ var attrValue = attr.value;+ if (+ attr.name === "required" ||+ attr.name === "aria-required" ||+ attr.name === "checked" ||+ attr.name === "aria-checked" ||+ attr.name === "selected" ||+ attr.name === "aria-selected" ||+ attr.name === "readonly" ||+ attr.name === "aria-readonly" ||+ attr.name === "disabled" ||+ attr.name === "aria-disabled"+ ) {+ if (attrValue && attrValue.toLowerCase() === "false") {+ attrValue = false;+ } else {+ attrValue = true;+ }+ }+ attrs[attr.name] = attrValue;+ }+ } else {+ _jsConsoleWarn(+ "element.attributes is not iterable. element_id=" + element_id,+ );+ }++ if (+ checkDisabledFromStyle(element) &&+ !attrs["disabled"] &&+ !attrs["aria-disabled"]+ ) {+ attrs["disabled"] = true;+ }++ if (+ checkRequiredFromStyle(element) &&+ !attrs["required"] &&+ !attrs["aria-required"]+ ) {+ attrs["required"] = true;+ }++ // check DOM property of required/checked/selected/readonly/disabled+ // the value from the DOM property should be the top priority+ if (element.required !== undefined) {+ delete attrs["required"];+ delete attrs["aria-required"];+ if (element.required) {+ attrs["required"] = true;+ }+ }+ if (element.checked !== undefined) {+ delete attrs["checked"];+ delete attrs["aria-checked"];+ if (element.checked) {+ attrs["checked"] = true;+ } else if (+ elementTagNameLower === "input" &&+ (element.type === "checkbox" || element.type === "radio")+ ) {+ // checked property always exists for checkbox and radio elements+ attrs["checked"] = false;+ }+ }+ if (element.selected !== undefined) {+ delete attrs["selected"];+ delete attrs["aria-selected"];+ if (element.selected) {+ attrs["selected"] = true;+ }+ }+ if (element.readOnly !== undefined) {+ delete attrs["readonly"];+ delete attrs["aria-readonly"];+ if (element.readOnly) {+ attrs["readonly"] = true;+ }+ }+ if (element.disabled !== undefined) {+ delete attrs["disabled"];+ delete attrs["aria-disabled"];+ if (element.disabled) {+ attrs["disabled"] = true;+ }+ }++ if (elementTagNameLower === "input" || elementTagNameLower === "textarea") {+ if (element.type === "password") {+ attrs["value"] = element.value ? "*".repeat(element.value.length) : "";+ } else {+ attrs["value"] = element.value;+ }+ }++ let elementObj = {+ id: element_id,+ frame: frame,+ frame_index: window.GlobalSkyvernFrameIndex,+ interactable: interactable,+ hoverOnly: isHoverOnlyElement(element),+ tagName: elementTagNameLower,+ attributes: attrs,+ beforePseudoText: getPseudoContent(element, "::before"),+ text: getElementText(element),+ afterPseudoText: getPseudoContent(element, "::after"),+ children: [],+ // if purgeable is True, which means this element is only used for building the tree relationship+ purgeable: purgeable,+ // don't trim any attr of this element if keepAllAttr=True+ keepAllAttr:+ elementTagNameLower === "svg" || element.closest("svg") !== null,+ isSelectable:+ elementTagNameLower === "select" ||+ isDatePickerSelector(element) ||+ isDivComboboxDropdown(element) ||+ isDropdownButton(element) ||+ isAngularDropdown(element) ||+ isAngularMaterialDatePicker(element) ||+ isSelect2Dropdown(element) ||+ isSelect2MultiChoice(element) ||+ isReadonlyInputDropdown(element),+ };++ let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;+ if (isInShadowRoot) {+ let shadowHostEle = element.getRootNode().host;+ let shadowHostId = shadowHostEle.getAttribute("unique_id");+ // assign shadowHostId to the shadowHost element if it doesn't have unique_id+ if (!shadowHostId) {+ shadowHostId = await uniqueId();+ shadowHostEle.setAttribute("unique_id", shadowHostId);+ }+ elementObj.shadowHost = shadowHostId;+ }++ // get options for select element or for listbox element+ let selectOptions = null;+ let selectedValue = "";+ if (elementTagNameLower === "select") {+ [selectOptions, selectedValue] = getSelectOptions(element);+ }++ if (selectOptions) {+ elementObj.options = selectOptions;+ }+ if (selectedValue) {+ elementObj.attributes["selected"] = selectedValue;+ }++ return elementObj;+}++// build the element tree for the body+async function buildTreeFromBody(+ frame = "main.frame",+ frame_index = undefined,+ must_included_tags = [],+) {+ if (+ window.GlobalSkyvernFrameIndex === undefined &&+ frame_index !== undefined+ ) {+ window.GlobalSkyvernFrameIndex = frame_index;+ }+ const maxElementNumber = 15000;+ const elementsAndResultArray = await buildElementTree(+ document.documentElement,+ frame,+ false,+ undefined,+ maxElementNumber,+ must_included_tags,+ );+ DomUtils.elementListCache = elementsAndResultArray[0];+ return elementsAndResultArray;+}++async function buildElementTree(+ starter = document.documentElement,+ frame,+ full_tree = false,+ hoverStylesMap = undefined,+ maxElementNumber = 0,+ must_included_tags = [],+) {+ // Generate hover styles map at the start+ if (hoverStylesMap === undefined) {+ hoverStylesMap = await getHoverStylesMap();+ }++ if (must_included_tags.length > 0) {+ _jsConsoleLog(+ "full tree will be enabled as the must_included_tags is not empty",+ );+ full_tree = true;+ }+ if (window.GlobalEnableAllTextualElements === undefined) {+ window.GlobalEnableAllTextualElements = false;+ }++ var elements = [];+ var resultArray = [];++ async function processElement(+ element,+ parentId,+ parent_xpath,+ current_node_index,+ ) {+ if (element === null) {+ _jsConsoleLog("get a null element");+ return;+ }++ if (maxElementNumber > 0 && elements.length >= maxElementNumber) {+ _jsConsoleWarn(+ "Max element number reached, aborting the element tree building",+ );+ return;+ }++ const tagName = element.tagName?.toLowerCase();+ if (!tagName) {+ _jsConsoleLog("get a null tagName");+ return;+ }++ if (tagName === "head") {+ return;+ }++ // skip processing option element as they are already added to the select.options+ if (tagName === "option") {+ return;+ }++ let current_xpath = null;+ if (parent_xpath !== null) {+ // ignore the namespace, otherwise the xpath sometimes won't find anything, specially for SVG elements+ current_xpath =+ parent_xpath ++ "/" ++ '*[name()="' ++ tagName ++ '"]' ++ "[" ++ current_node_index ++ "]";+ }++ let shadowDOMchildren = [];+ // sometimes the shadowRoot is not visible, but the elements in the shadowRoot are visible+ if (element.shadowRoot) {+ shadowDOMchildren = getChildElements(element.shadowRoot);+ }+ const isVisible = isElementVisible(element);+ if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {+ let interactable = isInteractable(element, hoverStylesMap);+ let elementObj = null;+ let isParentSVG = null;+ if (must_included_tags.includes(tagName)) {+ interactable = true;+ }+ if (interactable) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (+ tagName === "frameset" ||+ tagName === "iframe" ||+ tagName === "frame"+ ) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (element.shadowRoot) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (isTableRelatedElement(element)) {+ // build all table related elements into skyvern element+ // we need these elements to preserve the DOM structure+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (hasBeforeOrAfterPseudoContent(element)) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (tagName === "svg") {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (+ (isParentSVG = element.closest("svg")) &&+ isParentSVG.getAttribute("unique_id")+ ) {+ // if element is the children of the <svg> with an unique_id+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (+ tagName === "embed" &&+ element.getAttribute("type")?.toLowerCase() === "application/pdf"+ ) {+ elementObj = await buildElementObject(+ frame,+ element,+ interactable,+ true,+ );+ } else if (+ getElementText(element).length > 0 &&+ getElementText(element).length <= 5000+ ) {+ if (window.GlobalEnableAllTextualElements) {+ // force all textual elements to be interactable+ interactable = true;+ }+ elementObj = await buildElementObject(frame, element, interactable);+ } else if (full_tree) {+ // when building full tree, we only get text from element itself+ // elements without text are purgeable+ elementObj = await buildElementObject(+ frame,+ element,+ interactable,+ true,+ );+ if (elementObj.text.length > 0) {+ elementObj.purgeable = false;+ }+ }++ if (elementObj) {+ elementObj.xpath = current_xpath;+ elements.push(elementObj);+ // If the element is interactable but has no interactable parent,+ // then it starts a new tree, so add it to the result array+ // and set its id as the interactable parent id for the next elements+ // under it+ if (parentId === null) {+ resultArray.push(elementObj);+ }+ // If the element is interactable and has an interactable parent,+ // then add it to the children of the parent+ else {+ // TODO: use dict/object so that we access these in O(1) instead+ elements+ .find((element) => element.id === parentId)+ .children.push(elementObj);+ }+ parentId = elementObj.id;+ }+ }++ const children = getChildElements(element);+ const xpathMap = new Map();++ for (let i = 0; i < children.length; i++) {+ const childElement = children[i];+ const tagName = childElement?.tagName?.toLowerCase();+ if (!tagName) {+ _jsConsoleLog("get a null tagName");+ continue;+ }+ let current_node_index = xpathMap.get(tagName);+ if (current_node_index == undefined) {+ current_node_index = 1;+ } else {+ current_node_index = current_node_index + 1;+ }+ xpathMap.set(tagName, current_node_index);+ await processElement(+ childElement,+ parentId,+ current_xpath,+ current_node_index,+ );+ }++ // FIXME: xpath won't work when the element is in shadow DOM+ for (let i = 0; i < shadowDOMchildren.length; i++) {+ const childElement = shadowDOMchildren[i];+ await processElement(childElement, parentId, null, 0);+ }+ return;+ }++ const trimDuplicatedText = (element) => {+ if (element.children.length === 0 && !element.options) {+ return;+ }++ // if the element has options, text will be duplicated with the option text+ if (element.options) {+ element.options.forEach((option) => {+ element.text = element.text.replace(option.text, "");+ });+ }++ // BFS to delete duplicated text+ element.children.forEach((child) => {+ // delete duplicated text in the tree+ element.text = element.text.replace(child.text, "");+ trimDuplicatedText(child);+ });++ // trim multiple ";"+ element.text = element.text.replace(/;+/g, ";");+ // trimleft and trimright ";"+ element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");+ };++ // some elements without children nodes should be removed out, such as <label>+ const removeOrphanNode = (results) => {+ const trimmedResults = [];+ for (let i = 0; i < results.length; i++) {+ const element = results[i];+ element.children = removeOrphanNode(element.children);+ if (element.tagName === "label") {+ const labelElement = document.querySelector(+ element.tagName + '[unique_id="' + element.id + '"]',+ );+ if (+ labelElement &&+ labelElement.childElementCount === 0 &&+ !labelElement.getAttribute("for") &&+ !element.text+ ) {+ continue;+ }+ }+ trimmedResults.push(element);+ }+ return trimmedResults;+ };++ let current_xpath = null;+ if (starter === document.documentElement) {+ current_xpath = "";+ }++ // setup before parsing the dom+ await processElement(starter, null, current_xpath, 1);++ for (var element of elements) {+ if (+ ((element.tagName === "input" && element.attributes["type"] === "text") ||+ element.tagName === "textarea") &&+ (element.attributes["required"] || element.attributes["aria-required"]) &&+ element.attributes.value === ""+ ) {+ // TODO (kerem): we may want to pass these elements to the LLM as empty but required fields in the future+ _jsConsoleLog(+ "input element with required attribute and no value",+ element,+ );+ }+ }++ resultArray = removeOrphanNode(resultArray);+ resultArray.forEach((root) => {+ trimDuplicatedText(root);+ });++ return [elements, resultArray];+}++function drawBoundingBoxes(elements) {+ // draw a red border around the elements+ DomUtils.clearVisibleClientRectCache();+ elements.forEach((element) => {+ const ele = getDOMElementBySkyvenElement(element);+ element.rect = ele ? DomUtils.getVisibleClientRect(ele, true) : null;+ });+ var groups = groupElementsVisually(elements);+ var hintMarkers = createHintMarkersForGroups(groups);+ addHintMarkersToPage(hintMarkers);+ DomUtils.clearVisibleClientRectCache();+}++async function buildElementsAndDrawBoundingBoxes(+ frame = "main.frame",+ frame_index = undefined,+) {+ if (DomUtils.elementListCache.length > 0) {+ drawBoundingBoxes(DomUtils.elementListCache);+ return;+ }+ _jsConsoleWarn("no element list cache, drawBoundingBoxes from scratch");+ var elementsAndResultArray = await buildTreeFromBody(frame, frame_index);+ drawBoundingBoxes(elementsAndResultArray[0]);+}++function captchaSolvedCallback() {+ _jsConsoleLog("captcha solved");+ if (!window["captchaSolvedCounter"]) {+ window["captchaSolvedCounter"] = 0;+ }+ // For some reason this isn't being called.. TODO figure out why+ window["captchaSolvedCounter"] = window["captchaSolvedCounter"] + 1;+}++function getCaptchaSolves() {+ if (!window["captchaSolvedCounter"]) {+ window["captchaSolvedCounter"] = 0;+ }+ return window["captchaSolvedCounter"];+}++function groupElementsVisually(elements) {+ // Quadtree O(n log n)+ const validElements = elements.filter((element) => element.rect);++ if (validElements.length === 0) return [];++ // Calculate bounds+ const bounds = calculateBounds(validElements);++ // Create quadtree+ const quadTree = new QuadTreeNode(bounds);+ validElements.forEach((element) => quadTree.insert(element));++ const groups = [];+ const processed = new Set();++ for (const element of validElements) {+ if (processed.has(element)) continue;++ const group = { elements: [element], rect: null };+ processed.add(element);++ // Find all elements overlapping with current element+ const overlapping = findOverlappingElements(+ element,+ validElements,+ quadTree,+ processed,+ );++ for (const overlappingElement of overlapping) {+ group.elements.push(overlappingElement);+ processed.add(overlappingElement);+ }++ group.rect = createRectangleForGroup(group);+ groups.push(group);+ }++ return groups;+}++// Helper functions+function calculateBounds(elements) {+ const rects = elements.map((el) => el.rect);+ const left = Math.min(...rects.map((r) => r.left));+ const top = Math.min(...rects.map((r) => r.top));+ const right = Math.max(...rects.map((r) => r.right));+ const bottom = Math.max(...rects.map((r) => r.bottom));++ return {+ x: left,+ y: top,+ width: right - left,+ height: bottom - top,+ };+}++function findOverlappingElements(element, allElements, quadTree, processed) {+ const result = [];+ const queue = [element];++ while (queue.length > 0) {+ const current = queue.shift();++ // Use quadtree to query nearby elements+ const nearby = quadTree.query(current.rect);++ for (const nearbyElement of nearby) {+ if (+ !processed.has(nearbyElement) &&+ nearbyElement !== current &&+ Rect.intersects(current.rect, nearbyElement.rect)+ ) {+ result.push(nearbyElement);+ processed.add(nearbyElement);+ queue.push(nearbyElement);+ }+ }+ }+ return result;+}++function createRectangleForGroup(group) {+ const rects = group.elements.map((element) => element.rect);+ const top = Math.min(...rects.map((rect) => rect.top));+ const left = Math.min(...rects.map((rect) => rect.left));+ const bottom = Math.max(...rects.map((rect) => rect.bottom));+ const right = Math.max(...rects.map((rect) => rect.right));+ return Rect.create(left, top, right, bottom);+}++function generateHintStrings(count) {+ const hintCharacters = "sadfjklewcmpgh";+ let hintStrings = [""];+ let offset = 0;++ while (hintStrings.length - offset < count || hintStrings.length === 1) {+ const hintString = hintStrings[offset++];+ for (const ch of hintCharacters) {+ hintStrings.push(ch + hintString);+ }+ }+ hintStrings = hintStrings.slice(offset, offset + count);++ // Shuffle the hints so that they're scattered; hints starting with the same character and short+ // hints are spread evenly throughout the array.+ return hintStrings.sort(); // .map((str) => str.reverse())+}++function createHintMarkersForGroups(groups) {+ if (groups.length === 0) {+ _jsConsoleLog("No groups found, not adding hint markers to page.");+ return [];+ }++ const hintMarkers = groups+ .filter((group) => group.elements.some((element) => element.interactable))+ .map((group) => createHintMarkerForGroup(group));+ // fill in marker text+ // const hintStrings = generateHintStrings(hintMarkers.length);+ for (let i = 0; i < hintMarkers.length; i++) {+ const hintMarker = hintMarkers[i];++ let interactableElementFound = false;++ for (let i = 0; i < hintMarker.group.elements.length; i++) {+ if (hintMarker.group.elements[i].interactable) {+ hintMarker.hintString = hintMarker.group.elements[i].id;+ interactableElementFound = true;+ break;+ }+ }++ if (!interactableElementFound) {+ hintMarker.hintString = "";+ }++ try {+ hintMarker.element.innerHTML = hintMarker.hintString;+ } catch (e) {+ // Ensure trustedTypes is available+ if (typeof trustedTypes !== "undefined") {+ try {+ const escapeHTMLPolicy = trustedTypes.createPolicy("hint-policy", {+ createHTML: (string) => string,+ });+ hintMarker.element.innerHTML = escapeHTMLPolicy.createHTML(+ hintMarker.hintString.toUpperCase(),+ );+ } catch (policyError) {+ _jsConsoleWarn("Could not create trusted types policy:", policyError);+ // Skip updating the hint marker if policy creation fails+ }+ } else {+ _jsConsoleError("trustedTypes is not supported in this environment.");+ }+ }+ }++ return hintMarkers;+}++function createHintMarkerForGroup(group) {+ // Calculate the position of the element relative to the document+ var scrollTop = window.pageYOffset || document.documentElement.scrollTop;+ var scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;++ const marker = {};+ // yellow annotation box with string+ const el = document.createElement("div");+ el.style.position = "absolute";+ el.style.left = group.rect.left + scrollLeft + "px";+ el.style.top = group.rect.top + scrollTop + "px";+ // Each group is assigned a different incremental z-index, we use the same z-index for the+ // bounding box and the hint marker+ el.style.zIndex = this.currentZIndex;++ // The bounding box around the group of hints.+ const boundingBox = document.createElement("div");++ // Set styles for the bounding box+ boundingBox.style.position = "absolute";+ boundingBox.style.display = "display";+ boundingBox.style.left = group.rect.left + scrollLeft + "px";+ boundingBox.style.top = group.rect.top + scrollTop + "px";+ boundingBox.style.width = group.rect.width + "px";+ boundingBox.style.height = group.rect.height + "px";+ boundingBox.style.bottom = boundingBox.style.top + boundingBox.style.height;+ boundingBox.style.right = boundingBox.style.left + boundingBox.style.width;+ boundingBox.style.border = "2px solid blue"; // Change the border color as needed+ boundingBox.style.pointerEvents = "none"; // Ensures the box doesn't interfere with other interactions+ boundingBox.style.zIndex = this.currentZIndex++;++ return Object.assign(marker, {+ element: el,+ boundingBox: boundingBox,+ group: group,+ });+}++function addHintMarkersToPage(hintMarkers) {+ const parent = document.createElement("div");+ parent.id = "boundingBoxContainer";+ for (const hintMarker of hintMarkers) {+ parent.appendChild(hintMarker.element);+ parent.appendChild(hintMarker.boundingBox);+ }+ document.documentElement.appendChild(parent);+}++function removeBoundingBoxes() {+ var hintMarkerContainer = document.querySelector("#boundingBoxContainer");+ if (hintMarkerContainer) {+ hintMarkerContainer.remove();+ }+}++function safeWindowScroll(x, y) {+ if (typeof window.scroll === "function") {+ window.scroll({ left: x, top: y, behavior: "instant" });+ } else if (typeof window.scrollTo === "function") {+ window.scrollTo({ left: x, top: y, behavior: "instant" });+ } else {+ _jsConsoleError("window.scroll and window.scrollTo are both not supported");+ }+}++async function safeScrollToTop(+ draw_boxes,+ frame = "main.frame",+ frame_index = undefined,+) {+ removeBoundingBoxes();+ safeWindowScroll(0, 0);+ if (draw_boxes) {+ await buildElementsAndDrawBoundingBoxes(frame, frame_index);+ }+ return window.scrollY;+}++function getScrollWidthAndHeight() {+ return [+ document.documentElement.scrollWidth,+ document.documentElement.scrollHeight,+ ];+}++function getScrollXY() {+ return [window.scrollX, window.scrollY];+}++function scrollToXY(x, y) {+ safeWindowScroll(x, y);+}++async function scrollToNextPage(+ draw_boxes,+ frame = "main.frame",+ frame_index = undefined,+ need_overlap = true,+) {+ // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again+ // return true if there is a next page, false otherwise+ removeBoundingBoxes();+ window.scrollBy({+ left: 0,+ top: need_overlap ? window.innerHeight - 200 : window.innerHeight,+ behavior: "instant",+ });+ if (draw_boxes) {+ await buildElementsAndDrawBoundingBoxes(frame, frame_index);+ }+ return window.scrollY;+}++function isWindowScrollable() {+ const documentBody = document.body;+ const documentElement = document.documentElement;+ if (!documentBody || !documentElement) {+ return false;+ }++ // Check if the body's overflow style is set to hidden+ const bodyOverflow = getElementComputedStyle(documentBody)?.overflow;+ const htmlOverflow = getElementComputedStyle(documentElement)?.overflow;++ // Check if the document height is greater than the window height+ const isScrollable =+ document.documentElement.scrollHeight > window.innerHeight;++ // If the overflow is set to 'hidden' or there is no content to scroll, return false+ if (bodyOverflow === "hidden" || htmlOverflow === "hidden" || !isScrollable) {+ return false;+ }++ return true;+}++/**+ * Find the nearest scrollable container relative to the given element and scroll it.+ * Three strategies in priority order:+ * 1) Walk up from element to find a scrollable ancestor (element is inside container)+ * 2) Walk up the DOM checking siblings at each level (element is beside container)+ * 3) Fall back to page-level scrolling (for pages where the body itself scrolls,+ * e.g. T&C pages with no scrollable sub-container)+ * Returns: true (sub-container scrolled), "page" (page-level scroll needed), or false (nothing scrollable).+ */+function scrollNearestScrollableContainer(element, direction) {+ function isContainerScrollable(node) {+ if (!node || node === document.documentElement || node === document.body)+ return false;+ const style = window.getComputedStyle(node);+ const oy = style.overflowY;+ return (+ (oy === "auto" || oy === "scroll") &&+ node.scrollHeight > node.clientHeight+ );+ }++ // Strategy 1: walk up from element to find a scrollable ancestor+ let target = element;+ while (target && target !== document.documentElement) {+ if (isContainerScrollable(target)) break;+ target = target.parentElement;+ }++ // Strategy 2: walk up the DOM checking siblings at each level+ if (!target || target === document.documentElement) {+ target = null;+ let level = element.parentElement;+ while (level && level !== document.documentElement && !target) {+ for (const child of level.children) {+ if (isContainerScrollable(child)) {+ target = child;+ break;+ }+ }+ level = level.parentElement;+ }+ }++ // Scroll the sub-container if found+ if (target) {+ if (direction === "down") {+ target.scrollTop = target.scrollHeight;+ } else {+ target.scrollTop = 0;+ }+ return true;+ }++ // Strategy 3: fall back to page-level scrolling when no sub-container exists.+ // Many pages (e.g. T&C agreements) render content inline with the page body+ // as the only scrollable area. Return "page" to signal the Python handler to+ // use mouse.wheel events for native browser scrolling, which reliably triggers+ // page JavaScript (IntersectionObserver, scroll listeners) that programmatic+ // scrollTo/scrollTop methods may not.+ if (isWindowScrollable()) {+ return "page";+ }++ return false;+}++function scrollToElementBottom(element, page_by_page = false) {+ const top = page_by_page+ ? element.clientHeight + element.scrollTop+ : element.scrollHeight;+ element.scroll({+ top: top,+ left: 0,+ behavior: "smooth",+ });+}++function scrollToElementTop(element) {+ element.scroll({+ top: 0,+ left: 0,+ behavior: "instant",+ });+}++/**+ * Get all styles associated with :hover selectors+ *+ * Chrome doesn't allow you to compute these in run-time because hover is a protected attribute (from JS code)+ *+ * Instead of checking the hover state, we can look at the stylesheet and find all the :hover selectors+ * and try to infer styles associated with them+ *+ * It's not 100% accurate, but it's a good start+ *+ * References:+ * https://stackoverflow.com/questions/23040926/how-can-i-get-elementhover-style+ * https://stackoverflow.com/questions/7013559/is-there-a-way-to-get-element-hover-style-while-the-element-not-in-hover-state+ * https://stackoverflow.com/questions/17226676/how-to-simulate-a-mouseover-in-pure-javascript-that-activates-the-css-hover+ */+async function getHoverStylesMap() {+ const hoverMap = new Map();+ const sheets = [...document.styleSheets];++ const parseCssSheet = (sheet) => {+ const rules = sheet.cssRules || sheet.rules;+ for (const rule of rules) {+ if (rule.type === 1 && rule.selectorText) {+ // Split multiple selectors (e.g., "a:hover, button:hover")+ const selectors = rule.selectorText.split(",").map((s) => s.trim());++ for (const selector of selectors) {+ // Check if this is a hover rule+ if (selector.includes(":hover")) {+ // Get all parts of the selector+ const parts = selector.split(/\s*[>+~]\s*/);++ // Get the main hoverable element (the one with :hover)+ const hoverPart = parts.find((part) => part.includes(":hover"));+ if (!hoverPart) continue;++ // Get base selector without :hover+ const baseSelector = hoverPart.replace(/:hover/g, "").trim();++ // Skip invalid selectors+ if (!isValidCSSSelector(baseSelector)) {+ continue;+ }++ // Get or create styles object for this selector+ let styles = hoverMap.get(baseSelector) || {};++ // Add all style properties+ for (const prop of rule.style) {+ styles[prop] = rule.style[prop];+ }++ // If this is a nested selector (like :hover > .something)+ // store it in a special format+ if (parts.length > 1) {+ const fullSelector = selector;+ styles["__nested__"] = styles["__nested__"] || [];+ styles["__nested__"].push({+ selector: fullSelector,+ styles: Object.fromEntries(+ [...rule.style].map((prop) => [prop, rule.style[prop]]),+ ),+ });+ }++ // only need the style which includes the cursor attribute.+ if (!("cursor" in styles)) {+ continue;+ }+ hoverMap.set(baseSelector, styles);+ }+ }+ }+ }+ };++ try {+ await Promise.all(+ sheets.map(async (sheet) => {+ try {+ parseCssSheet(sheet);+ } catch (e) {+ _jsConsoleWarn("Could not access stylesheet:", e);++ if ((e.name !== "SecurityError" && e.code !== 18) || !sheet.href) {+ return;+ }++ let newLink = null;+ try {+ const oldLink = sheet.ownerNode;+ const url = new URL(sheet.href);+ _jsConsoleLog("recreating the link element: ", sheet.href);+ newLink = document.createElement("link");+ newLink.rel = "stylesheet";+ url.searchParams.set("v", Date.now());+ newLink.href = url.toString();+ newLink.crossOrigin = "anonymous";+ // until the new link loaded, removing the old one+ document.head.append(newLink);++ // wait for a while until the sheet is fully loaded+ await asyncSleepFor(1500);+ const newSheets = [...document.styleSheets];+ const refreshedSheet = newSheets.find(+ (s) => s.href === newLink.href,+ );+ if (!refreshedSheet) {+ newLink.remove();+ return;+ }+ _jsConsoleLog("parsing recreated the link element: ", newLink.href);+ parseCssSheet(refreshedSheet);+ oldLink.remove();+ } catch (e) {+ _jsConsoleWarn("Error recreating the link element:", e);+ if (newLink) {+ newLink.remove();+ }+ }+ }+ }),+ );+ } catch (e) {+ _jsConsoleError("Error processing stylesheets:", e);+ }++ return hoverMap;+}++// Helper method for debugging+function findNodeById(arr, targetId, path = []) {+ for (let i = 0; i < arr.length; i++) {+ const currentPath = [...path, arr[i].id];+ if (arr[i].id === targetId) {+ _jsConsoleLog("Lineage:", currentPath.join(" -> "));+ return arr[i];+ }+ if (arr[i].children && arr[i].children.length > 0) {+ const result = findNodeById(arr[i].children, targetId, currentPath);+ if (result) {+ return result;+ }+ }+ }+ return null;+}++function getElementDomDepth(elementNode) {+ let depth = 0;+ const rootElement = elementNode.getRootNode().firstElementChild;+ while (elementNode !== rootElement && elementNode.parentElement) {+ depth++;+ elementNode = elementNode.parentElement;+ }+ return depth;+}++if (window.globalOneTimeIncrementElements === undefined) {+ window.globalOneTimeIncrementElements = [];+}++if (window.globalDomDepthMap === undefined) {+ window.globalDomDepthMap = new Map();+}++function isClassNameIncludesHidden(className) {+ // some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`+ return (+ className.toLowerCase().includes("hide") ||+ className.toLowerCase().includes("invisible") ||+ className.toLowerCase().includes("closed")+ );+}++function isClassNameIncludesActivatedStatus(className) {+ // some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click+ return (+ className.toLowerCase().includes("open") ||+ className.toLowerCase().includes("active")+ );+}++function waitForNextFrame() {+ return new Promise((resolve) => {+ requestAnimationFrame(() => resolve());+ });+}++function asyncSleepFor(ms) {+ return new Promise((resolve) => setTimeout(resolve, ms));+}++async function addIncrementalNodeToMap(parentNode, childrenNode) {+ const maxParsedElement = 3000;+ const maxElementToWait = 100;+ if ((await window.globalParsedElementCounter.get()) > maxParsedElement) {+ _jsConsoleWarn(+ "Too many elements parsed, stopping the observer to parse the elements",+ );+ await window.globalParsedElementCounter.add();+ return;+ }++ // make the dom parser async+ await waitForNextFrame();+ if (window.globalListnerFlag) {+ // calculate the depth of targetNode element for sorting+ const depth = getElementDomDepth(parentNode);+ let newNodesTreeList = [];+ if (window.globalDomDepthMap.has(depth)) {+ newNodesTreeList = window.globalDomDepthMap.get(depth);+ }++ try {+ for (const child of childrenNode) {+ // sleep for a while until animation ends+ if (+ (await window.globalParsedElementCounter.get()) < maxElementToWait+ ) {+ await asyncSleepFor(300);+ }+ // Pass -1 as frame_index to indicate the frame number is not sensitive in this case+ const [_, newNodeTree] = await buildElementTree(+ child,+ "",+ true,+ window.globalHoverStylesMap,+ );+ if (newNodeTree.length > 0) {+ newNodesTreeList.push(...newNodeTree);+ }+ }+ } catch (error) {+ _jsConsoleError("Error building incremental element node:", error);+ }+ window.globalDomDepthMap.set(depth, newNodesTreeList);+ }+ await window.globalParsedElementCounter.add();+}++if (window.globalObserverForDOMIncrement === undefined) {+ window.globalObserverForDOMIncrement = new MutationObserver(async function (+ mutationsList,+ observer,+ ) {+ // TODO: how to detect duplicated recreate element?+ for (const mutation of mutationsList) {+ const node = mutation.target;+ if (node.nodeType === Node.TEXT_NODE) continue;+ const tagName = node.tagName?.toLowerCase();++ // ignore unique_id change to avoid infinite loop about DOM changes+ if (mutation.attributeName === "unique_id") continue;++ // if the changing element is dropdown related elements, we should consider+ // they're the new element as long as the element is still visible on the page+ if (+ isDropdownRelatedElement(node) &&+ getElementComputedStyle(node)?.display !== "none"+ ) {+ window.globalOneTimeIncrementElements.push({+ targetNode: node,+ newNodes: [node],+ });+ await addIncrementalNodeToMap(node, [node]);+ continue;+ }++ // if they're not the dropdown related elements+ // we detect the element based on the following rules+ switch (mutation.type) {+ case "attributes": {+ switch (mutation.attributeName) {+ case "hidden": {+ if (!node.hidden) {+ window.globalOneTimeIncrementElements.push({+ targetNode: node,+ newNodes: [node],+ });+ await addIncrementalNodeToMap(node, [node]);+ }+ break;+ }+ case "style": {+ // TODO: need to confirm that elemnent is hidden previously+ if (tagName === "body") continue;+ if (getElementComputedStyle(node)?.display !== "none") {+ window.globalOneTimeIncrementElements.push({+ targetNode: node,+ newNodes: [node],+ });+ await addIncrementalNodeToMap(node, [node]);+ }+ break;+ }+ case "class": {+ if (tagName === "body") continue;+ if (!mutation.oldValue) continue;+ const currentClassName = node.className+ ? node.className.toString()+ : "";+ if (+ !isClassNameIncludesHidden(mutation.oldValue) &&+ !isClassNameIncludesActivatedStatus(currentClassName) &&+ !node.hasAttribute("data-menu-uid") && // google framework use this to trace dropdown menu+ !mutation.oldValue.includes("select__items") &&+ !(+ node.hasAttribute("data-testid") &&+ node.getAttribute("data-testid").includes("select-dropdown")+ )+ )+ continue;+ if (getElementComputedStyle(node)?.display !== "none") {+ window.globalOneTimeIncrementElements.push({+ targetNode: node,+ newNodes: [node],+ });+ await addIncrementalNodeToMap(node, [node]);+ }+ break;+ }+ }+ break;+ }+ case "childList": {+ let changedNode = {+ targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree+ };+ let newNodes = [];+ if (mutation.addedNodes && mutation.addedNodes.length > 0) {+ for (const node of mutation.addedNodes) {+ // skip the text nodes, they won't be interactable+ if (node.nodeType === Node.TEXT_NODE) continue;+ newNodes.push(node);+ }+ }+ if (+ newNodes.length == 0 &&+ (tagName === "ul" ||+ (tagName === "div" &&+ node.hasAttribute("role") &&+ node.getAttribute("role").toLowerCase() === "listbox"))+ ) {+ newNodes.push(node);+ }++ if (newNodes.length > 0) {+ changedNode.newNodes = newNodes;+ window.globalOneTimeIncrementElements.push(changedNode);+ await addIncrementalNodeToMap(+ changedNode.targetNode,+ changedNode.newNodes,+ );+ }+ break;+ }+ }+ }+ });+}++async function startGlobalIncrementalObserver(element = null) {+ window.globalListnerFlag = true;+ window.globalDomDepthMap = new Map();+ window.globalOneTimeIncrementElements = [];+ window.globalHoverStylesMap = await getHoverStylesMap();+ window.globalParsedElementCounter = new SafeCounter();+ window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data+ window.globalObserverForDOMIncrement.observe(document.body, {+ attributes: true,+ attributeOldValue: true,+ childList: true,+ subtree: true,+ characterData: true,+ });++ // if the element is in shadow DOM, we need to observe the shadow DOM as well+ if (element && element.getRootNode() instanceof ShadowRoot) {+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {+ attributes: true,+ attributeOldValue: true,+ childList: true,+ subtree: true,+ characterData: true,+ });+ }+}++async function stopGlobalIncrementalObserver() {+ window.globalListnerFlag = false;+ window.globalObserverForDOMIncrement.disconnect();+ window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data+ while (+ window.globalParsedElementCounter &&+ window.globalOneTimeIncrementElements &&+ (await window.globalParsedElementCounter.get()) <+ window.globalOneTimeIncrementElements.length+ ) {+ await asyncSleepFor(100);+ }+ window.globalOneTimeIncrementElements = [];+ window.globalDomDepthMap = new Map();+}++async function getIncrementElements(wait_until_finished = true) {+ if (wait_until_finished) {+ while (+ (await window.globalParsedElementCounter.get()) <+ window.globalOneTimeIncrementElements.length+ ) {+ await asyncSleepFor(100);+ }+ }++ // cleanup the children tree, remove the duplicated element+ // search starting from the shallowest node:+ // 1. if deeper, the node could only be the children of the shallower one or no related one.+ // 2. if depth is same, the node could only be duplicated one or no related one.+ const idToElement = new Map();+ const cleanedTreeList = [];+ const sortedDepth = Array.from(window.globalDomDepthMap.keys()).sort(+ (a, b) => a - b,+ );+ for (let idx = 0; idx < sortedDepth.length; idx++) {+ const depth = sortedDepth[idx];+ const treeList = window.globalDomDepthMap.get(depth);++ const removeDupAndConcatChildren = async (element) => {+ let children = element.children;+ for (let i = 0; i < children.length; i++) {+ const child = children[i];+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work+ if (child.shadowHost) {+ continue;+ }+ const domElement = document.querySelector(`[unique_id="${child.id}"]`);+ // if the element is still on the page, we rebuild the element to update the information+ if (domElement) {+ let newChild = await buildElementObject(+ "",+ domElement,+ child.interactable,+ child.purgeable,+ );+ newChild.children = child.children;+ children[i] = newChild;+ } else {+ children[i].interactable = false;+ }+ }++ if (idToElement.has(element.id)) {+ element = idToElement.get(element.id);+ for (let i = 0; i < children.length; i++) {+ const child = children[i];+ if (!idToElement.get(child.id)) {+ element.children.push(child);+ }+ }+ }+ idToElement.set(element.id, element);+ for (let i = 0; i < children.length; i++) {+ const child = children[i];+ await removeDupAndConcatChildren(child);+ }+ };++ for (let treeHeadElement of treeList) {+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work+ if (!treeHeadElement.shadowHost) {+ const domElement = document.querySelector(+ `[unique_id="${treeHeadElement.id}"]`,+ );+ // if the element is still on the page, we rebuild the element to update the information+ if (domElement) {+ let newHead = await buildElementObject(+ "",+ domElement,+ treeHeadElement.interactable,+ treeHeadElement.purgeable,+ );+ newHead.children = treeHeadElement.children;+ treeHeadElement = newHead;+ } else {+ treeHeadElement.interactable = false;+ }+ }++ // check if the element is existed+ if (!idToElement.has(treeHeadElement.id)) {+ cleanedTreeList.push(treeHeadElement);+ }+ await removeDupAndConcatChildren(treeHeadElement);+ }+ }++ return [Array.from(idToElement.values()), cleanedTreeList];+}++function isAnimationFinished() {+ const animations = document.getAnimations({ subtree: true });+ const unfinishedAnimations = animations.filter(+ (a) => a.playState !== "finished",+ );+ if (!unfinishedAnimations || unfinishedAnimations.length == 0) {+ return true;+ }+ return false;+}++/**+ * Remove unique_id attribute from all elements on the page.+ * This includes elements in the main document and shadow DOM.+ */+function removeAllUniqueIds() {+ // Function to recursively remove unique_id from an element and its children+ const removeUniqueIdFromElement = (element) => {+ if (element.hasAttribute("unique_id")) {+ element.removeAttribute("unique_id");+ }++ // Process children in the main DOM+ for (const child of Array.from(element.children)) {+ removeUniqueIdFromElement(child);+ }++ // Process elements in shadow DOM if present+ if (element.shadowRoot) {+ for (const shadowChild of Array.from(element.shadowRoot.children)) {+ removeUniqueIdFromElement(shadowChild);+ }+ }+ };++ // Start from document.documentElement to process the entire page+ removeUniqueIdFromElement(document.documentElement);+}++/**++// How to run the code:++// Get all interactable elements and draw boxes+buildElementsAndDrawBoundingBoxes();++// Remove the boxes+removeBoundingBoxes();++// Get the element tree+const [elements, tree] = buildTreeFromBody();+_jsConsoleLog(elements); // All elements+_jsConsoleLog(tree); // Tree structure++// Test if a specific element is interactable+const element = document.querySelector('button');+const hoverMap = getHoverStylesMap();+_jsConsoleLog(isInteractable(element, hoverMap));+ */diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.pyindex 4036d32e0..460c1211b 100644--- a/skyvern/webeye/scraper/scraped_page.py+++ b/skyvern/webeye/scraper/scraped_page.py@@ -2,6 +2,7 @@ import copyimport jsonimport typingfrom abc import ABC, abstractmethod+from collections import dequefrom enum import StrEnumfrom typing import Any, Awaitable, Callable, Self@@ -9,6 +10,7 @@ import structlogfrom playwright.async_api import Frame, Pagefrom pydantic import BaseModel, PrivateAttr+from skyvern.config import settingsfrom skyvern.exceptions import UnknownElementTreeFormatfrom skyvern.forge.sdk.api.crypto import calculate_sha256from skyvern.forge.sdk.core import skyvern_context@@ -33,10 +35,8 @@ def build_attribute(key: str, value: Any) -> str:return f'{key}="{str(value)}"' if value else key-def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:- """- if element is flagged as dropped, the html format is empty- """+def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:+ """Legacy: always deep-copies attributes (pre-V2)."""tag = element["tagName"]attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))@@ -50,18 +50,13 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:context = skyvern_context.ensure_context()- # FIXME: Theoretically, all href links with over 69(64+1+4) length could be hashed- # but currently, just hash length>150 links to confirm the solution goes wellif "href" in attributes and len(attributes.get("href", "")) > 150:href = attributes.get("href", "")- # jinja style can't accept the variable name starts with number- # adding "_" to make sure the variable name is valid.hashed_href = "_" + calculate_sha256(href)context.hashed_href_map[hashed_href] = hrefattributes["href"] = "{{" + hashed_href + "}}"if need_skyvern_attrs:- # adding the node attribute to attributesfor attr in ELEMENT_NODE_ATTRIBUTES:value = element.get(attr)if value is None:@@ -70,6 +65,92 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())+ if element.get("isSelectable", False):+ tag = "select"++ text = element.get("text", "")+ children_html = "".join(+ _json_to_html_legacy(child, need_skyvern_attrs=need_skyvern_attrs) for child in element.get("children", [])+ )+ option_html = "".join(+ f'<option index="{option.get("optionIndex")}">{option.get("text")}</option>'+ if option.get("text")+ else f'<option index="{option.get("optionIndex")}" value="{option.get("value")}">{option.get("text")}</option>'+ for option in element.get("options", [])+ )++ if element.get("purgeable", False):+ return children_html + option_html++ before_pseudo_text = element.get("beforePseudoText") or ""+ after_pseudo_text = element.get("afterPseudoText") or ""++ if (+ tag in ["img", "input", "br", "hr", "meta", "link"]+ and not option_html+ and not children_html+ and not before_pseudo_text+ and not after_pseudo_text+ ):+ return f"<{tag} {attributes_html}/>" if attributes_html else f"<{tag}/>"++ return (+ (f"<{tag} {attributes_html}>" if attributes_html else f"<{tag}>")+ + before_pseudo_text+ + text+ + children_html+ + option_html+ + after_pseudo_text+ + f"</{tag}>"+ )+++def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:+ """+ if element is flagged as dropped, the html format is empty+ """+ if not settings.ENABLE_DOM_PARSER_V2:+ return _json_to_html_legacy(element, need_skyvern_attrs)++ tag = element["tagName"]+ original_attrs = element.get("attributes", {})++ interactable = element.get("interactable", False)+ if element.get("isDropped", False):+ if not interactable:+ return ""+ else:+ LOG.debug("Element is interactable. Trimmed all attributes instead of dropping it", element=element)+ original_attrs = {}++ context = skyvern_context.ensure_context()++ # Only shallow-copy attributes when we actually need to mutate them.+ # This avoids dict() allocation on every element — most elements don't need it.+ attributes = original_attrs+ href_val = original_attrs.get("href", "")++ if href_val and len(href_val) > 150:+ attributes = dict(original_attrs)+ # jinja style can't accept the variable name starts with number+ # adding "_" to make sure the variable name is valid.+ hashed_href = "_" + calculate_sha256(href_val)+ context.hashed_href_map[hashed_href] = href_val+ attributes["href"] = "{{" + hashed_href + "}}"++ if need_skyvern_attrs:+ # adding the node attribute to attributes+ has_skyvern_attrs = any(element.get(attr) is not None for attr in ELEMENT_NODE_ATTRIBUTES)+ if has_skyvern_attrs:+ if attributes is original_attrs:+ attributes = dict(original_attrs)+ for attr in ELEMENT_NODE_ATTRIBUTES:+ value = element.get(attr)+ if value is not None:+ attributes[attr] = value++ attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())+if element.get("isSelectable", False):tag = "select"@@ -275,25 +356,45 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):raise UnknownElementTreeFormat(fmt=fmt)- def _process_element_for_economy_tree(self, element: dict) -> dict | None:- """- Helper method to process an element for the economy tree using BFS.- Removes SVG elements and their children.- """- # Skip SVG elements entirely+ @staticmethod+ def _process_element_for_economy_tree_legacy(element: dict) -> dict | None:+ """Legacy: recursive SVG removal (pre-V2)."""if element.get("tagName", "").lower() == "svg":return None- # Process children using BFSif "children" in element:new_children = []for child in element["children"]:- processed_child = self._process_element_for_economy_tree(child)+ processed_child = ScrapedPage._process_element_for_economy_tree_legacy(child)if processed_child:new_children.append(processed_child)element["children"] = new_childrenreturn element+ @staticmethod+ def _process_element_for_economy_tree(element: dict) -> dict | None:+ """+ Process an element for the economy tree. V2 uses iterative BFS.+ Removes SVG elements and their children.+ """+ if not settings.ENABLE_DOM_PARSER_V2:+ return ScrapedPage._process_element_for_economy_tree_legacy(element)++ if element.get("tagName", "").lower() == "svg":+ return None++ # BFS to filter SVG children at every level+ queue: deque[dict] = deque([element])+ while queue:+ node = queue.popleft()+ children = node.get("children")+ if not children:+ continue+ filtered = [c for c in children if c.get("tagName", "").lower() != "svg"]+ node["children"] = filtered+ queue.extend(filtered)+ return element+async def refresh(self, draw_boxes: bool = True, scroll: bool = True, max_retries: int = 0) -> Self:refreshed_page = await self._browser_state.scrape_website(url=self.url,diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 854a90be5..0ef2c08c6 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -1,7 +1,8 @@import asyncioimport copyimport json-from collections import defaultdict+import time+from collections import defaultdict, dequeimport structlogfrom playwright._impl._errors import TimeoutError@@ -109,6 +110,7 @@ def clean_element_before_hashing(element: dict) -> dict:def hash_element(element: dict) -> str:hash_ready_element = clean_element_before_hashing(element)# Sort the keys to ensure consistent ordering+ # NOTE: Do NOT change separators — hashes are persisted to the database for cached action matching.element_string = json.dumps(hash_ready_element, sort_keys=True)return calculate_sha256(element_string)@@ -131,7 +133,10 @@ def build_element_dict(id_to_frame_dict[element_id] = element["frame"]element_hash = hash_element(element)id_to_element_hash[element_id] = element_hash- hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]+ if SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:+ hash_to_element_ids.setdefault(element_hash, []).append(element_id)+ else:+ hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]return id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids@@ -296,6 +301,7 @@ async def scrape_web_unsafe(# We check if the scroll_y_px_old is the same as scroll_y_px to determine if we have reached the end of the page.# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after# clicking start my quote)+ scrape_start_time = time.time()url = page.urlif url == "about:blank" and not support_empty_page:# Allow scraping if the page has child frames with meaningful content@@ -309,23 +315,46 @@ async def scrape_web_unsafe(frame_count=len(meaningful_frames),)+ t0 = time.time()skyvern_frame = await SkyvernFrame.create_instance(page)+ js_inject_time = time.time() - t0await skyvern_frame.safe_wait_for_animation_end()if wait_seconds > 0:LOG.info(f"Waiting for {wait_seconds} seconds before scraping the website.", wait_seconds=wait_seconds)await asyncio.sleep(wait_seconds)- elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)+ t0 = time.time()+ elements, element_tree, main_frame_time, child_frames_time, num_child_frames = await get_interactable_element_tree(+ page, scrape_exclude, must_included_tags+ )+ element_tree_time = time.time() - t0if not elements and not support_empty_page:LOG.warning("No elements found on the page, wait and retry")await empty_page_retry_wait()- elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)-- element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))+ t0 = time.time()+ (+ elements,+ element_tree,+ main_frame_time,+ child_frames_time,+ num_child_frames,+ ) = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)+ element_tree_time = time.time() - t0++ # Deep copy once for cleanup (mutates in-place), then copy the cleaned result for trim.+ # The second copy of the cleaned tree is cheaper since cleanup may remove elements.+ t0 = time.time()+ element_tree_copy = copy.deepcopy(element_tree)+ element_tree = await cleanup_element_tree(page, url, element_tree_copy)+ cleanup_time = time.time() - t0++ t0 = time.time()element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))+ trim_time = time.time() - t0screenshots = []+ screenshot_time = 0.0if take_screenshots:element_tree_trimmed_html_str = "".join(json_to_html(element, need_skyvern_attrs=False) for element in element_tree_trimmed@@ -343,6 +372,7 @@ async def scrape_web_unsafe(except Exception:LOG.warning("Failed to get current x, y position of the page", exc_info=True)+ t0 = time.time()screenshots = await SkyvernFrame.take_split_screenshots(page=page,url=url,@@ -350,21 +380,26 @@ async def scrape_web_unsafe(max_number=max_screenshot_number,scroll=scroll,)+ screenshot_time = time.time() - t0# scroll back to the original x, y position of the pageif x is not None and y is not None:await skyvern_frame.safe_scroll_to_x_y(x, y)LOG.debug("Scrolled back to the original x, y position of the page after scraping", x=x, y=y)+ t0 = time.time()id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids = build_element_dict(elements)+ build_dict_time = time.time() - t0# if there are no elements, fail the scraping unless support_empty_page is Trueif not elements and not support_empty_page:raise NoElementFound()+ t0 = time.time()text_content = await get_frame_text(page.main_frame)+ text_extraction_time = time.time() - t0html = ""window_dimension = None@@ -380,6 +415,25 @@ async def scrape_web_unsafe(exc_info=True,)+ total_scrape_time = time.time() - scrape_start_time+ LOG.debug(+ "Scraping performance metrics",+ url=url,+ total_scrape_time=total_scrape_time,+ js_inject_time=js_inject_time,+ element_tree_time=element_tree_time,+ main_frame_time=main_frame_time,+ child_frames_time=child_frames_time,+ num_child_frames=num_child_frames,+ cleanup_time=cleanup_time,+ trim_time=trim_time,+ screenshot_time=screenshot_time,+ build_dict_time=build_dict_time,+ text_extraction_time=text_extraction_time,+ num_elements=len(elements),+ num_screenshots=len(screenshots),+ )+return ScrapedPage(elements=elements,id_to_css_dict=id_to_css_dict,@@ -480,17 +534,20 @@ async def get_interactable_element_tree(page: Page,scrape_exclude: ScrapeExcludeFunc | None = None,must_included_tags: list[str] | None = None,-) -> tuple[list[dict], list[dict]]:+) -> tuple[list[dict], list[dict], float, float, int]:"""Get the element tree of the page, including all the elements that are interactable.:param page: Page instance to get the element tree from.- :return: Tuple containing the element tree and a map of element IDs to elements.+ :return: Tuple of (elements, element_tree, main_frame_time, child_frames_time, num_child_frames)."""# main page index is 0skyvern_page = await SkyvernFrame.create_instance(page)++ t0 = time.time()elements, element_tree = await skyvern_page.build_tree_from_body(frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags)+ main_frame_time = time.time() - t0context = skyvern_context.ensure_context()frames = await get_all_children_frames(page)@@ -502,6 +559,7 @@ async def get_interactable_element_tree(frame_index = len(context.frame_index_map) + 1context.frame_index_map[frame] = frame_index+ t0 = time.time()for frame in frames:frame_index = context.frame_index_map[frame]elements, element_tree = await add_frame_interactable_elements(@@ -511,8 +569,9 @@ async def get_interactable_element_tree(element_tree,must_included_tags,)+ child_frames_time = time.time() - t0- return elements, element_tree+ return elements, element_tree, main_frame_time, child_frames_time, len(frames)class IncrementalScrapePage(ElementTreeBuilder):@@ -557,7 +616,9 @@ class IncrementalScrapePage(ElementTreeBuilder):self.elements = incremental_elements- incremental_tree = await cleanup_element_tree(frame, frame.url, copy.deepcopy(incremental_tree))+ incremental_tree_copy = copy.deepcopy(incremental_tree)+ incremental_tree = await cleanup_element_tree(frame, frame.url, incremental_tree_copy)+ # Second copy of cleaned tree is cheaper since cleanup may have removed elementstrimmed_element_tree = trim_element_tree(copy.deepcopy(incremental_tree))self.element_tree = incremental_tree@@ -699,7 +760,8 @@ def _should_keep_unique_id(element: dict) -> bool:return element.get("interactable", False)-def trim_element(element: dict) -> dict:+def _trim_element_legacy(element: dict) -> dict:+ """Legacy: list-based BFS with two-pass attribute filtering (pre-V2)."""queue = [element]while queue:queue_ele = queue.pop(0)@@ -725,7 +787,6 @@ def trim_element(element: dict) -> dict:queue_ele["attributes"] = new_attributeselse:del queue_ele["attributes"]- # remove the tag, don't need it in the HTML treeif "keepAllAttr" in queue_ele:del queue_ele["keepAllAttr"]@@ -754,6 +815,53 @@ def trim_element(element: dict) -> dict:return element+def trim_element(element: dict) -> dict:+ if not SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:+ return _trim_element_legacy(element)++ queue: deque = deque([element])+ while queue:+ queue_ele = queue.popleft()+ queue_ele.pop("frame", None)+ queue_ele.pop("frame_index", None)++ if "id" in queue_ele and not _should_keep_unique_id(queue_ele):+ del queue_ele["id"]++ # Single-pass attribute filtering: remove base64 data AND whitelist in one pass+ keep_all = queue_ele.get("keepAllAttr", False)+ attributes = queue_ele.get("attributes")+ if attributes:+ new_attributes = _filter_attributes(attributes, keep_all)+ if new_attributes:+ queue_ele["attributes"] = new_attributes+ else:+ queue_ele.pop("attributes", None)++ queue_ele.pop("keepAllAttr", None)++ children = queue_ele.get("children")+ if children:+ queue.extend(children)+ else:+ queue_ele.pop("children", None)++ text = queue_ele.get("text")+ if text is not None and not str(text).strip():+ del queue_ele["text"]++ # Remove empty pseudo text fields using pop-and-reinsert pattern to avoid double lookup+ before = queue_ele.pop("beforePseudoText", None)+ if before:+ queue_ele["beforePseudoText"] = before++ after = queue_ele.pop("afterPseudoText", None)+ if after:+ queue_ele["afterPseudoText"] = after++ return element++def trim_element_tree(elements: list[dict]) -> list[dict]:for element in elements:trim_element(element)@@ -783,6 +891,24 @@ def _trimmed_attributes(attributes: dict) -> dict:return new_attributes+def _filter_attributes(attributes: dict, keep_all: bool) -> dict:+ """Single-pass attribute filtering: removes base64 data, applies whitelist, and truncates long names."""+ new_attributes: dict = {}+ for key, value in attributes.items():+ # Skip base64 data URIs+ if key in BASE64_INCLUDE_ATTRIBUTES and isinstance(value, str) and "data:" in value:+ continue+ # Apply whitelist (unless keepAllAttr is True)+ if keep_all or key in RESERVED_ATTRIBUTES:+ # Truncate long name attributes in the same pass+ if key == "name" and isinstance(value, str) and len(value) > 500:+ value = value[:500]+ new_attributes[key] = value+ elif key == "role" and value in ("listbox", "option"):+ new_attributes[key] = value+ return new_attributes++def _remove_unique_id(element: dict) -> None:if "attributes" not in element:returndiff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 6b0a4b6c6..a0b172d92 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -20,12 +20,9 @@ from skyvern.forge.sdk.trace import tracedLOG = structlog.get_logger()-def load_js_script() -> str:- # TODO: Handle file location better. This is a hacky way to find the file location.- path = f"{SKYVERN_DIR}/webeye/scraper/domUtils.js"+def load_js_script(filename: str = "domUtils.js") -> str:+ path = f"{SKYVERN_DIR}/webeye/scraper/{filename}"try:- # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.- # This will allow our code to be type safe.with open(path, encoding="utf-8") as f:return f.read()except FileNotFoundError as e:@@ -33,7 +30,8 @@ def load_js_script() -> str:raise e-JS_FUNCTION_DEFS = load_js_script()+JS_FUNCTION_DEFS = load_js_script("domUtils.js")+JS_FUNCTION_DEFS_LEGACY = load_js_script("domUtils_legacy.js")class ScreenshotMode(StrEnum):@@ -379,8 +377,17 @@ class SkyvernFrame:@classmethodasync def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:instance = cls(frame=frame)- await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)- if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:+ settings = SettingsManager.get_settings()+ js_to_inject = JS_FUNCTION_DEFS if settings.ENABLE_DOM_PARSER_V2 else JS_FUNCTION_DEFS_LEGACY+ t0 = time.time()+ await cls.evaluate(frame=instance.frame, expression=js_to_inject)+ js_inject_elapsed = time.time() - t0+ LOG.debug(+ "domUtils.js injection time",+ elapsed_time=js_inject_elapsed,+ dom_parser_v2=settings.ENABLE_DOM_PARSER_V2,+ )+ if settings.ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:await instance.evaluate(frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true")@@ -543,12 +550,23 @@ class SkyvernFrame:) -> tuple[list[dict], list[dict]]:must_included_tags = must_included_tags or []js_script = "async ([frame_name, frame_index, must_included_tags]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags)"- return await self.evaluate(+ t0 = time.time()+ result = await self.evaluate(frame=self.frame,expression=js_script,timeout_ms=timeout_ms,arg=[frame_name, frame_index, must_included_tags],)+ elapsed = time.time() - t0+ num_elements = len(result[0]) if result and len(result) > 0 else 0+ LOG.debug(+ "buildTreeFromBody JS execution time",+ frame_name=frame_name,+ frame_index=frame_index,+ elapsed_time=elapsed,+ num_elements=num_elements,+ )+ return result@traced()async def get_incremental_element_tree(diff --git a/tests/unit/test_cli_bootstrap.py b/tests/unit/test_cli_bootstrap.pynew file mode 100644index 000000000..bb29b7367--- /dev/null+++ b/tests/unit/test_cli_bootstrap.py@@ -0,0 +1,46 @@+from __future__ import annotations++import logging+from types import SimpleNamespace++import skyvern._cli_bootstrap as cli_bootstrap+++def test_bootstrap_defaults_to_warning_without_explicit_log_level(monkeypatch) -> None:+ setup_calls: list[str] = []+ fake_settings = SimpleNamespace(LOG_LEVEL="INFO", model_fields_set=set())++ monkeypatch.setattr("skyvern.config.settings", fake_settings)+ monkeypatch.setattr("skyvern.forge.sdk.forge_log.setup_logger", lambda: setup_calls.append("called"))++ logger_names = ("", "skyvern", "httpx", "litellm", "playwright", "httpcore")+ previous_levels = {name: logging.getLogger(name).level for name in logger_names}+ try:+ cli_bootstrap.configure_cli_bootstrap_logging()+ assert setup_calls == ["called"]+ assert fake_settings.LOG_LEVEL == "WARNING"+ for name in logger_names:+ assert logging.getLogger(name).level == logging.WARNING+ finally:+ for name, level in previous_levels.items():+ logging.getLogger(name).setLevel(level)+++def test_bootstrap_honors_explicit_log_level(monkeypatch) -> None:+ setup_calls: list[str] = []+ fake_settings = SimpleNamespace(LOG_LEVEL="DEBUG", model_fields_set={"LOG_LEVEL"})++ monkeypatch.setattr("skyvern.config.settings", fake_settings)+ monkeypatch.setattr("skyvern.forge.sdk.forge_log.setup_logger", lambda: setup_calls.append("called"))++ logger_names = ("", "skyvern", "httpx", "litellm", "playwright", "httpcore")+ previous_levels = {name: logging.getLogger(name).level for name in logger_names}+ try:+ cli_bootstrap.configure_cli_bootstrap_logging()+ assert setup_calls == ["called"]+ assert fake_settings.LOG_LEVEL == "DEBUG"+ for name in logger_names:+ assert logging.getLogger(name).level == logging.DEBUG+ finally:+ for name, level in previous_levels.items():+ logging.getLogger(name).setLevel(level)diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.pyindex 2dc5a3e68..a3472252b 100644--- a/tests/unit/test_cli_commands.py+++ b/tests/unit/test_cli_commands.py@@ -452,6 +452,32 @@ class TestWorkflowCommands:assert parsed["ok"] is Falseassert parsed["error"]["code"] == "RUN_NOT_FOUND"+ def test_workflow_status_cli_preserves_full_detail_behavior(+ self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture+ ) -> None:+ from skyvern.cli import workflow as workflow_cmd++ tool = AsyncMock(+ return_value={+ "ok": True,+ "action": "skyvern_workflow_status",+ "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},+ "data": {"run_id": "wr_123", "recording_url": "https://example.com/recording"},+ "artifacts": [],+ "timing_ms": {},+ "warnings": [],+ "error": None,+ }+ )+ monkeypatch.setattr(workflow_cmd, "tool_workflow_status", tool)++ workflow_cmd.workflow_status(run_id="wr_123", json_output=True)++ assert tool.await_args.kwargs == {"run_id": "wr_123", "verbosity": "full"}+ parsed = json.loads(capsys.readouterr().out)+ assert parsed["ok"] is True+ assert parsed["data"]["recording_url"] == "https://example.com/recording"+def test_workflow_update_missing_definition_file_raises_bad_parameter(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:diff --git a/tests/unit/test_cli_commands_logging.py b/tests/unit/test_cli_commands_logging.pyindex b714838d4..2d5aef1f6 100644--- a/tests/unit/test_cli_commands_logging.py+++ b/tests/unit/test_cli_commands_logging.py@@ -1,28 +1,17 @@from __future__ import annotations-import logging-import skyvern.cli.commands as cli_commandsdef test_configure_cli_logging_is_idempotent(monkeypatch) -> None:setup_calls: list[str] = []- monkeypatch.setattr(cli_commands, "_setup_logger", lambda: setup_calls.append("called"))+ monkeypatch.setattr(cli_commands, "_configure_cli_bootstrap_logging", lambda: setup_calls.append("called"))monkeypatch.setattr(cli_commands, "_cli_logging_configured", False)+ cli_commands.configure_cli_logging()+ assert setup_calls == ["called"]- logger_names = ("skyvern", "httpx", "litellm", "playwright", "httpcore")- previous_levels = {name: logging.getLogger(name).level for name in logger_names}- try:- cli_commands.configure_cli_logging()- assert setup_calls == ["called"]- for name in logger_names:- assert logging.getLogger(name).level == logging.WARNING-- cli_commands.configure_cli_logging()- assert setup_calls == ["called"]- finally:- for name, level in previous_levels.items():- logging.getLogger(name).setLevel(level)+ cli_commands.configure_cli_logging()+ assert setup_calls == ["called"]def test_cli_callback_configures_logging(monkeypatch) -> None:diff --git a/tests/unit/test_dom_parser_js_benchmark.py b/tests/unit/test_dom_parser_js_benchmark.pynew file mode 100644index 000000000..ca603f5e1--- /dev/null+++ b/tests/unit/test_dom_parser_js_benchmark.py@@ -0,0 +1,255 @@+"""+Playwright-based benchmark for JavaScript DOM parsing (domUtils.js).++Tests run a real browser to measure actual buildTreeFromBody() performance+on a synthetic HTML page with many elements.++These tests require playwright to be installed with browser binaries.+They are automatically skipped if the browser binary is missing.+Run `playwright install chromium` to enable these tests.+"""++import pytest++from skyvern.webeye.scraper.scraper import JS_FUNCTION_DEFS++# Only run if playwright is available+playwright = pytest.importorskip("playwright")+++def _launch_browser():+ """Try to launch a Chromium browser. Returns (browser, playwright_instance) or raises."""+ from playwright.sync_api import sync_playwright++ pw = sync_playwright().start()+ try:+ browser = pw.chromium.launch(headless=True)+ return browser, pw+ except Exception:+ pw.stop()+ raise++[email protected](scope="module")+def browser_context():+ """Launch a browser for JS benchmarks. Skips if browser binary is missing."""+ try:+ browser, pw = _launch_browser()+ except Exception as e:+ pytest.skip(f"Playwright browser not available: {e}")++ context = browser.new_context()+ yield context+ context.close()+ browser.close()+ pw.stop()+++def _generate_test_html(num_elements: int) -> str:+ """Generate an HTML page with the specified number of elements."""+ elements = []+ for i in range(num_elements):+ tag = ["div", "span", "button", "input", "a"][i % 5]+ if tag == "input":+ elements.append(+ f'<{tag} type="text" name="field_{i}" placeholder="Field {i}" '+ f'class="form-control" aria-label="Field {i}">'+ )+ elif tag == "a":+ elements.append(+ f'<{tag} href="https://example.com/{i}" class="link-{i}" aria-label="Link {i}">Link text {i}</{tag}>'+ )+ elif tag == "button":+ elements.append(f'<{tag} type="button" class="btn btn-{i}" aria-label="Button {i}">Click {i}</{tag}>')+ else:+ elements.append(f'<{tag} class="element-{i}" style="cursor: pointer;">Text content {i}</{tag}>')++ body_content = "\n".join(elements)+ return f"""+ <!DOCTYPE html>+ <html>+ <head><title>Benchmark Page</title></head>+ <body>+ <div id="main-container">+ {body_content}+ </div>+ </body>+ </html>+ """+++class TestDomUtilsJsBenchmark:+ """Benchmark buildTreeFromBody() in a real browser."""++ @pytest.mark.parametrize("num_elements", [500, 2000, 5000])+ def test_build_tree_performance(self, browser_context, num_elements: int):+ page = browser_context.new_page()+ html = _generate_test_html(num_elements)+ page.set_content(html)++ # Load domUtils.js+ page.evaluate(JS_FUNCTION_DEFS)++ # Benchmark buildTreeFromBody+ result = page.evaluate("""+ async () => {+ const start = performance.now();+ const [elements, tree] = await buildTreeFromBody('main.frame', 0);+ const elapsed = performance.now() - start;+ return {+ elapsed_ms: elapsed,+ element_count: elements.length,+ tree_root_count: tree.length,+ };+ }+ """)++ elapsed_ms = result["elapsed_ms"]+ element_count = result["element_count"]+ tree_root_count = result["tree_root_count"]++ print(f"\nbuildTreeFromBody({num_elements} DOM nodes):")+ print(f" Time: {elapsed_ms:.1f}ms")+ print(f" Elements found: {element_count}")+ print(f" Tree roots: {tree_root_count}")++ # Should complete within reasonable time+ assert elapsed_ms < 10000, f"buildTreeFromBody took {elapsed_ms:.1f}ms for {num_elements} elements"+ assert element_count > 0, "Should find at least some elements"++ page.close()++ def test_hover_styles_map_performance(self, browser_context):+ """Benchmark getHoverStylesMap() with many stylesheets."""+ page = browser_context.new_page()++ # Create a page with many CSS rules+ css_rules = "\n".join([f".element-{i}:hover {{ cursor: pointer; background: #{i:06x}; }}" for i in range(500)])+ html = f"""+ <!DOCTYPE html>+ <html>+ <head>+ <style>{css_rules}</style>+ <style>+ .non-hover-1 {{ color: red; }}+ .non-hover-2 {{ color: blue; }}+ </style>+ </head>+ <body><div>Test</div></body>+ </html>+ """+ page.set_content(html)+ page.evaluate(JS_FUNCTION_DEFS)++ result = page.evaluate("""+ async () => {+ const start = performance.now();+ const hoverMap = await getHoverStylesMap();+ const elapsed = performance.now() - start;+ return {+ elapsed_ms: elapsed,+ hover_selectors: hoverMap.size,+ };+ }+ """)++ elapsed_ms = result["elapsed_ms"]+ hover_selectors = result["hover_selectors"]++ print("\ngetHoverStylesMap (500 hover rules + 2 non-hover rules):")+ print(f" Time: {elapsed_ms:.1f}ms")+ print(f" Hover selectors found: {hover_selectors}")++ assert elapsed_ms < 5000, f"getHoverStylesMap took {elapsed_ms:.1f}ms"+ assert hover_selectors > 0, "Should find hover selectors"++ page.close()++ def test_draw_bounding_boxes_performance(self, browser_context):+ """Benchmark drawBoundingBoxes() with visual grouping."""+ page = browser_context.new_page()+ html = _generate_test_html(1000)+ page.set_content(html)+ page.evaluate(JS_FUNCTION_DEFS)++ result = page.evaluate("""+ async () => {+ // First build the tree (populates cache)+ await buildTreeFromBody('main.frame', 0);++ const start = performance.now();+ await buildElementsAndDrawBoundingBoxes('main.frame', 0);+ const elapsed = performance.now() - start;++ // Count bounding boxes added+ const container = document.querySelector('#boundingBoxContainer');+ const boxCount = container ? container.children.length : 0;++ return {+ elapsed_ms: elapsed,+ box_count: boxCount,+ };+ }+ """)++ elapsed_ms = result["elapsed_ms"]+ box_count = result["box_count"]++ print("\nbuildElementsAndDrawBoundingBoxes (1000 elements):")+ print(f" Time: {elapsed_ms:.1f}ms")+ print(f" Bounding boxes: {box_count}")++ assert elapsed_ms < 10000, f"drawBoundingBoxes took {elapsed_ms:.1f}ms"++ page.close()++ def test_tree_correctness(self, browser_context):+ """Verify the tree structure is correct after optimizations."""+ page = browser_context.new_page()+ html = """+ <!DOCTYPE html>+ <html>+ <body>+ <div id="parent">+ <button id="btn1">Click me</button>+ <input type="text" name="field1" placeholder="Enter text">+ <a href="https://example.com">Link</a>+ <select name="dropdown">+ <option value="a">Option A</option>+ <option value="b">Option B</option>+ </select>+ </div>+ </body>+ </html>+ """+ page.set_content(html)+ page.evaluate(JS_FUNCTION_DEFS)++ result = page.evaluate("""+ async () => {+ const [elements, tree] = await buildTreeFromBody('main.frame', 0);+ return {+ element_count: elements.length,+ tree_root_count: tree.length,+ // Check that interactable elements were found+ interactable_count: elements.filter(e => e.interactable).length,+ // Check a button was found+ has_button: elements.some(e => e.tagName === 'button'),+ // Check an input was found+ has_input: elements.some(e => e.tagName === 'input'),+ // Check a link was found+ has_link: elements.some(e => e.tagName === 'a'),+ // Check select was found with options+ has_select: elements.some(e => e.tagName === 'select' && e.options && e.options.length === 2),+ };+ }+ """)++ assert result["element_count"] > 0+ assert result["interactable_count"] >= 4 # button, input, a, select+ assert result["has_button"]+ assert result["has_input"]+ assert result["has_link"]+ assert result["has_select"]++ page.close()diff --git a/tests/unit/test_dom_parser_performance.py b/tests/unit/test_dom_parser_performance.pynew file mode 100644index 000000000..24a5c0534--- /dev/null+++ b/tests/unit/test_dom_parser_performance.py@@ -0,0 +1,405 @@+"""+Benchmark and regression tests for DOM parser pipeline performance.++Tests the Python-side processing: trim_element_tree, _filter_attributes,+json_to_html, and cleanup traversal patterns.+"""++import copy+import time+from collections import deque++import pytest++from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_html+from skyvern.webeye.scraper.scraper import (+ _filter_attributes,+ _trimmed_attributes,+ _trimmed_base64_data,+ build_element_dict,+ trim_element,+ trim_element_tree,+)++[email protected](autouse=True)+def _setup_skyvern_context():+ """Ensure a SkyvernContext exists for tests that call json_to_html."""+ ctx = SkyvernContext()+ skyvern_context.set(ctx)+ yield+ skyvern_context.reset()+++def _make_element(+ element_id: str,+ tag: str = "div",+ interactable: bool = True,+ text: str = "sample text",+ num_attrs: int = 10,+ children: list | None = None,+) -> dict:+ """Generate a realistic element dict for testing."""+ attrs = {+ "class": f"cls-{element_id}",+ "id": f"html-id-{element_id}",+ "data-testid": f"test-{element_id}",+ "style": "display: flex; align-items: center;",+ "aria-label": f"Element {element_id}",+ "role": "button",+ "type": "button",+ "name": f"name-{element_id}",+ "value": f"val-{element_id}",+ "placeholder": "Enter value...",+ }+ # Add extra non-reserved attributes to test filtering+ for i in range(max(0, num_attrs - len(attrs))):+ attrs[f"data-extra-{i}"] = f"extra-value-{i}"++ return {+ "id": element_id,+ "tagName": tag,+ "interactable": interactable,+ "text": text,+ "attributes": attrs,+ "children": children or [],+ "frame": "main.frame",+ "frame_index": 0,+ "keepAllAttr": False,+ "beforePseudoText": "",+ "afterPseudoText": "",+ "purgeable": False,+ "rect": {"top": 0, "left": 0, "bottom": 100, "right": 200, "width": 200, "height": 100},+ }+++def _make_element_tree(num_elements: int, depth: int = 3) -> list[dict]:+ """Generate a tree of elements for benchmarking."""+ element_counter = 0+ children_per_node = max(1, num_elements // (depth + 1))++ def build_level(current_depth: int, remaining: int) -> list[dict]:+ nonlocal element_counter+ level_elements = []+ while remaining > 0 and element_counter < num_elements:+ element_counter += 1+ children = []+ if current_depth < depth and remaining > 1:+ child_count = min(children_per_node, remaining - 1)+ children = build_level(current_depth + 1, child_count)+ remaining -= len(children)+ el = _make_element(+ element_id=f"el_{element_counter:04d}",+ tag=["div", "span", "button", "input", "a"][element_counter % 5],+ interactable=element_counter % 3 == 0,+ text=f"Text content for element {element_counter}" if element_counter % 2 == 0 else "",+ children=children,+ )+ level_elements.append(el)+ remaining -= 1+ return level_elements++ return build_level(0, num_elements)+++class TestFilterAttributesMerged:+ """Test that the new merged _filter_attributes produces the same output as the two-pass approach."""++ def test_basic_whitelist(self):+ attrs = {"class": "foo", "aria-label": "bar", "name": "baz", "data-x": "y"}+ result = _filter_attributes(attrs, keep_all=False)+ assert "aria-label" in result+ assert "name" in result+ assert "class" not in result+ assert "data-x" not in result++ def test_base64_removal(self):+ attrs = {"href": "data:image/png;base64,abc123", "name": "test", "src": "data:text/html;base64,xyz"}+ result = _filter_attributes(attrs, keep_all=False)+ assert "href" not in result+ assert "src" not in result+ assert result["name"] == "test"++ def test_keep_all_attr(self):+ attrs = {"class": "foo", "aria-label": "bar", "data-x": "y"}+ result = _filter_attributes(attrs, keep_all=True)+ # keepAllAttr=True should keep everything except base64+ assert "class" in result+ assert "data-x" in result++ def test_role_listbox_option(self):+ attrs = {"role": "listbox", "class": "foo"}+ result = _filter_attributes(attrs, keep_all=False)+ assert result["role"] == "listbox"+ assert "class" not in result++ def test_name_truncation_in_filter(self):+ """Name truncation should happen inside _filter_attributes (single pass)."""+ attrs = {"name": "x" * 1000, "aria-label": "test"}+ result = _filter_attributes(attrs, keep_all=False)+ assert len(result["name"]) == 500+ assert result["aria-label"] == "test"++ def test_equivalence_with_old_approach(self):+ """The new _filter_attributes should produce the same result as the old two-pass approach."""+ attrs = {+ "class": "foo",+ "aria-label": "bar",+ "name": "baz",+ "href": "data:image/png;base64,abc",+ "src": "https://example.com/image.png",+ "role": "option",+ "data-x": "y",+ "type": "button",+ }+ # Old approach: two passes+ old_pass1 = _trimmed_base64_data(attrs)+ old_result = _trimmed_attributes(old_pass1)++ # New approach: single pass+ new_result = _filter_attributes(attrs, keep_all=False)++ assert old_result == new_result+++class TestTrimElement:+ """Test trim_element correctness after optimization."""++ def test_removes_frame_fields(self):+ el = _make_element("test1")+ trim_element(el)+ assert "frame" not in el+ assert "frame_index" not in el++ def test_removes_keep_all_attr(self):+ el = _make_element("test2")+ trim_element(el)+ assert "keepAllAttr" not in el++ def test_removes_empty_text(self):+ el = _make_element("test3", text="")+ trim_element(el)+ assert "text" not in el++ def test_removes_empty_pseudo_text(self):+ el = _make_element("test4")+ el["beforePseudoText"] = ""+ el["afterPseudoText"] = ""+ trim_element(el)+ assert "beforePseudoText" not in el+ assert "afterPseudoText" not in el++ def test_keeps_interactable_id(self):+ el = _make_element("test5", interactable=True)+ trim_element(el)+ assert "id" in el++ def test_removes_non_interactable_id(self):+ el = _make_element("test6", interactable=False)+ el["attributes"].pop("disabled", None)+ el["attributes"].pop("aria-disabled", None)+ el["attributes"].pop("readonly", None)+ el["attributes"].pop("aria-readonly", None)+ el.pop("hoverOnly", None)+ trim_element(el)+ assert "id" not in el++ def test_filters_attributes(self):+ el = _make_element("test7")+ trim_element(el)+ attrs = el.get("attributes", {})+ # Non-reserved attributes should be removed+ assert "class" not in attrs+ assert "data-testid" not in attrs+ assert "style" not in attrs+ # Reserved attributes should remain+ assert "aria-label" in attrs+ assert "type" in attrs++ def test_truncates_long_name(self):+ el = _make_element("test8")+ el["attributes"]["name"] = "x" * 1000+ trim_element(el)+ assert len(el["attributes"]["name"]) == 500++ def test_processes_children(self):+ child = _make_element("child1")+ parent = _make_element("parent1", children=[child])+ trim_element(parent)+ assert "frame" not in child+ assert "keepAllAttr" not in child+++class TestTrimElementTreePerformance:+ """Benchmark tests for trim_element_tree at various scales."""++ @pytest.mark.parametrize("num_elements", [100, 1000, 5000])+ def test_trim_performance(self, num_elements: int):+ tree = _make_element_tree(num_elements)+ tree_copy = copy.deepcopy(tree)++ start = time.perf_counter()+ trim_element_tree(tree_copy)+ elapsed = time.perf_counter() - start++ # Log timing for visibility+ print(f"\ntrim_element_tree({num_elements} elements): {elapsed:.4f}s")+ # Should complete in reasonable time (< 1s for 5000 elements)+ assert elapsed < 2.0, f"trim_element_tree took too long: {elapsed:.4f}s for {num_elements} elements"+++class TestJsonToHtmlPerformance:+ """Benchmark json_to_html at various scales."""++ @pytest.mark.parametrize("num_elements", [100, 1000, 5000])+ def test_json_to_html_performance(self, num_elements: int):+ tree = _make_element_tree(num_elements)+ # Trim first (like the real pipeline)+ tree = trim_element_tree(copy.deepcopy(tree))++ start = time.perf_counter()+ result = "".join(json_to_html(element) for element in tree)+ elapsed = time.perf_counter() - start++ print(f"\njson_to_html({num_elements} elements): {elapsed:.4f}s, output: {len(result)} chars")+ assert elapsed < 2.0, f"json_to_html took too long: {elapsed:.4f}s for {num_elements} elements"+ assert len(result) > 0++ def test_json_to_html_correctness(self):+ """Verify basic HTML output structure."""+ el = {+ "tagName": "button",+ "id": "btn1",+ "interactable": True,+ "text": "Click me",+ "attributes": {"type": "submit", "aria-label": "Submit"},+ "children": [],+ }+ html = json_to_html(el)+ assert "<button" in html+ assert "Click me" in html+ assert 'type="submit"' in html+ assert "</button>" in html+++class TestDequeVsList:+ """Verify deque.popleft() is faster than list.pop(0) for BFS."""++ def test_deque_faster_than_list(self):+ n = 10000+ items = list(range(n))++ # List pop(0)+ lst = list(items)+ start = time.perf_counter()+ while lst:+ lst.pop(0)+ list_time = time.perf_counter() - start++ # Deque popleft+ dq = deque(items)+ start = time.perf_counter()+ while dq:+ dq.popleft()+ deque_time = time.perf_counter() - start++ print(f"\nlist.pop(0): {list_time:.6f}s, deque.popleft(): {deque_time:.6f}s")+ # deque should be significantly faster+ assert deque_time < list_time+++class TestBuildElementDict:+ """Test build_element_dict correctness and hash collision handling."""++ def test_basic_dict_building(self):+ elements = [+ {"id": "e1", "tagName": "button", "frame": "main", "attributes": {"type": "submit"}},+ {"id": "e2", "tagName": "input", "frame": "main", "attributes": {"name": "email"}},+ ]+ css_dict, elem_dict, frame_dict, hash_dict, hash_to_ids = build_element_dict(elements)++ assert "e1" in css_dict+ assert "e2" in css_dict+ assert elem_dict["e1"]["tagName"] == "button"+ assert frame_dict["e1"] == "main"+ assert "e1" in hash_dict+ assert "e2" in hash_dict++ def test_hash_collision_uses_append(self):+ """Verify that hash_to_element_ids uses list append (not concat) for collisions."""+ # Two identical elements (same tag, same attrs) will have the same hash+ el1 = {"id": "e1", "tagName": "div", "frame": "main", "attributes": {}}+ el2 = {"id": "e2", "tagName": "div", "frame": "main", "attributes": {}}+ _, _, _, hash_dict, hash_to_ids = build_element_dict([el1, el2])++ h1 = hash_dict["e1"]+ h2 = hash_dict["e2"]+ assert h1 == h2, "Identical elements should produce the same hash"+ assert hash_to_ids[h1] == ["e1", "e2"]+++class TestEconomyTreeProcessing:+ """Test the economy tree SVG filtering logic."""++ def test_filters_svg_root(self):+ svg_el = {"tagName": "svg", "children": [{"tagName": "path"}]}+ result = ScrapedPage._process_element_for_economy_tree(svg_el)+ assert result is None++ def test_filters_svg_children(self):+ tree = {+ "tagName": "div",+ "children": [+ {"tagName": "button", "children": []},+ {"tagName": "svg", "children": [{"tagName": "path"}]},+ {"tagName": "span", "children": []},+ ],+ }+ result = ScrapedPage._process_element_for_economy_tree(tree)+ assert result is not None+ children = result["children"]+ assert len(children) == 2+ assert children[0]["tagName"] == "button"+ assert children[1]["tagName"] == "span"++ def test_filters_nested_svg(self):+ tree = {+ "tagName": "div",+ "children": [+ {+ "tagName": "span",+ "children": [+ {"tagName": "svg", "children": []},+ {"tagName": "a", "children": []},+ ],+ },+ ],+ }+ result = ScrapedPage._process_element_for_economy_tree(tree)+ assert result is not None+ inner = result["children"][0]+ assert len(inner["children"]) == 1+ assert inner["children"][0]["tagName"] == "a"++ def test_filters_svg_case_insensitive(self):+ """SVG filtering is case-insensitive via .lower()."""+ tree = {+ "tagName": "div",+ "children": [+ {"tagName": "SVG", "children": []},+ {"tagName": "Svg", "children": []},+ {"tagName": "button", "children": []},+ ],+ }+ result = ScrapedPage._process_element_for_economy_tree(tree)+ assert result is not None+ assert len(result["children"]) == 1+ assert result["children"][0]["tagName"] == "button"++ def test_no_children(self):+ el = {"tagName": "input"}+ result = ScrapedPage._process_element_for_economy_tree(el)+ assert result is not None+ assert result["tagName"] == "input"diff --git a/tests/unit/test_mcp_concise_responses.py b/tests/unit/test_mcp_concise_responses.pyindex 4522cf07a..48535fb0f 100644--- a/tests/unit/test_mcp_concise_responses.py+++ b/tests/unit/test_mcp_concise_responses.py@@ -2,10 +2,12 @@from __future__ import annotations+import jsonfrom collections.abc import Iteratorimport pytest+import skyvern.cli.mcp_tools.workflow as workflow_toolsfrom skyvern.cli.core.result import Artifact, BrowserContext, make_result, set_concise_responses@@ -183,3 +185,70 @@ def test_verbose_returns_all_fields() -> None:assert result["data"]["resolved_selector"] is not Noneassert result["artifacts"] == []assert result["warnings"] == []+++def test_workflow_status_summary_stays_bounded_for_heavy_payload() -> None:+ long_url = "https://artifacts.skyvern.example/" + ("x" * 1450)+ heavy_run = {+ "workflow_run_id": "wr_heavy",+ "status": "terminated",+ "failure_reason": "Execution terminated after repeated navigation failures",+ "workflow_title": "Heavy workflow",+ "recording_url": long_url,+ "screenshot_urls": [f"{long_url}-{idx}" for idx in range(6)],+ "downloaded_files": [{"url": f"{long_url}-download", "filename": "case-export.csv"}],+ "outputs": {+ "collect_customer_data": {+ "task_screenshot_artifact_ids": [f"art_task_{idx}" for idx in range(12)],+ "workflow_screenshot_artifact_ids": [f"art_workflow_{idx}" for idx in range(12)],+ "task_screenshots": [f"{long_url}-task-{idx}" for idx in range(4)],+ "workflow_screenshots": [f"{long_url}-workflow-{idx}" for idx in range(4)],+ "extracted_information": [{"account_id": "acct_123", "status": "terminated"}],+ },+ "submit_case": [+ {+ "task_screenshot_artifact_ids": [f"art_nested_{idx}" for idx in range(6)],+ "workflow_screenshot_artifact_ids": [f"art_followup_{idx}" for idx in range(6)],+ "task_screenshots": [f"{long_url}-nested-task-{idx}" for idx in range(3)],+ "workflow_screenshots": [f"{long_url}-nested-workflow-{idx}" for idx in range(4)],+ }+ ],+ "extracted_information": [{"duplicated_rollup": True}],+ },+ "run_with": "code",+ }++ full_payload = workflow_tools._serialize_run_full(heavy_run)+ summary_payload = workflow_tools._serialize_run_summary(heavy_run)++ assert len(json.dumps(full_payload)) > 20_000+ assert len(json.dumps(summary_payload)) < 2_000++ result = make_result("skyvern_workflow_status", data=summary_payload)+ assert len(json.dumps(result)) < 2_200+ assert "recording_url" not in result["data"]+ assert "output" not in result["data"]+ assert result["data"]["artifact_summary"]["artifact_id_count"] == 36+++def test_workflow_status_summary_shrinks_simple_payload() -> None:+ long_url = "https://artifacts.skyvern.example/" + ("y" * 1450)+ simple_run = {+ "workflow_run_id": "wr_simple",+ "status": "completed",+ "workflow_title": "Simple workflow",+ "recording_url": long_url,+ "screenshot_urls": [f"{long_url}-shot"],+ "outputs": {+ "result": "success",+ "order_id": "ord_123",+ },+ "run_with": "code",+ }++ full_payload = workflow_tools._serialize_run_full(simple_run)+ summary_payload = workflow_tools._serialize_run_summary(simple_run)++ assert len(json.dumps(full_payload)) > 1_500+ assert len(json.dumps(summary_payload)) < 800+ assert summary_payload["output_summary"]["scalar_preview"] == {"result": "success", "order_id": "ord_123"}diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.pyindex 2ec94b0c3..50db933b7 100644--- a/tests/unit/test_mcp_workflow_tools.py+++ b/tests/unit/test_mcp_workflow_tools.py@@ -6,8 +6,10 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, patchimport pytest+from fastmcp import Clientimport skyvern.cli.mcp_tools.workflow as workflow_tools+from skyvern.cli.mcp_tools import mcpdef _fake_workflow_response() -> SimpleNamespace:@@ -26,6 +28,64 @@ def _fake_workflow_response() -> SimpleNamespace:)+def _fake_http_response(payload: dict[str, object], status_code: int = 200) -> SimpleNamespace:+ return SimpleNamespace(+ status_code=status_code,+ json=lambda: payload,+ text=json.dumps(payload),+ )+++def _heavy_workflow_run_payload(*, include_expanded_outputs: bool = True) -> dict[str, object]:+ long_url = "https://artifacts.skyvern.example/" + ("x" * 1450)+ screenshot_urls = [f"{long_url}-{idx}" for idx in range(6)]+ task_artifact_ids = [f"art_task_{idx}" for idx in range(12)]+ workflow_artifact_ids = [f"art_workflow_{idx}" for idx in range(12)]+ outputs: dict[str, object] = {+ "collect_customer_data": {+ "task_screenshot_artifact_ids": task_artifact_ids,+ "workflow_screenshot_artifact_ids": workflow_artifact_ids,+ "extracted_information": [{"account_id": "acct_123", "status": "terminated"}],+ "status": "terminated",+ },+ "submit_case": [+ {+ "workflow_screenshot_artifact_ids": [f"art_followup_{idx}" for idx in range(6)],+ "task_screenshot_artifact_ids": [f"art_nested_{idx}" for idx in range(6)],+ "result": "retry_required",+ }+ ],+ }+ if include_expanded_outputs:+ outputs["collect_customer_data"] |= {+ "task_screenshots": screenshot_urls[:4],+ "workflow_screenshots": screenshot_urls[2:6],+ }+ outputs["submit_case"][0] |= {+ "task_screenshots": screenshot_urls[:3],+ "workflow_screenshots": screenshot_urls[1:5],+ }+ outputs["extracted_information"] = [{"duplicated_rollup": True}]++ return {+ "workflow_id": "wpid_heavy",+ "workflow_run_id": "wr_heavy",+ "status": "terminated",+ "failure_reason": "Execution terminated after repeated navigation failures",+ "workflow_title": "Heavy workflow",+ "recording_url": long_url,+ "screenshot_urls": screenshot_urls,+ "downloaded_files": [+ {+ "url": f"{long_url}-download",+ "filename": "case-export.csv",+ }+ ],+ "outputs": outputs,+ "run_with": "code",+ }++@pytest.mark.asyncioasync def test_workflow_create_normalizes_invalid_text_prompt_llm_key(monkeypatch: pytest.MonkeyPatch) -> None:fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))@@ -439,3 +499,134 @@ async def test_mcp_text_prompt_without_llm_key_stays_null(monkeypatch: pytest.Moassert result["ok"] is Trueassert sent_block.llm_key is Noneassert sent_block.model is None+++async def test_workflow_status_uses_workflow_run_route_for_wr_ids(monkeypatch: pytest.MonkeyPatch) -> None:+ payload = _heavy_workflow_run_payload(include_expanded_outputs=False)+ request = AsyncMock(return_value=_fake_http_response(payload))+ fake_client = SimpleNamespace(+ get_run=AsyncMock(),+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),+ )+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ result = await workflow_tools.skyvern_workflow_status(run_id="wr_heavy")++ assert result["ok"] is True+ request.assert_awaited_once_with(+ "api/v1/workflows/runs/wr_heavy",+ method="GET",+ params={"include_output_details": False},+ )+ fake_client.get_run.assert_not_awaited()+ data = result["data"]+ assert data["run_id"] == "wr_heavy"+ assert data["run_type"] == "workflow_run"+ assert "recording_url" not in data+ assert "output" not in data+ assert data["artifact_summary"]["recording_available"] is True+ assert data["artifact_summary"]["artifact_id_count"] == 36+ assert data["output_summary"]["nested_screenshot_count"] == 0+ assert data["output_summary"]["has_extracted_information"] is True+++async def test_workflow_status_full_preserves_expanded_workflow_details(monkeypatch: pytest.MonkeyPatch) -> None:+ payload = _heavy_workflow_run_payload(include_expanded_outputs=True)+ request = AsyncMock(return_value=_fake_http_response(payload))+ fake_client = SimpleNamespace(+ get_run=AsyncMock(),+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),+ )+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ result = await workflow_tools.skyvern_workflow_status(run_id="wr_heavy", verbosity="full")++ assert result["ok"] is True+ request.assert_awaited_once_with(+ "api/v1/workflows/runs/wr_heavy",+ method="GET",+ params={"include_output_details": True},+ )+ data = result["data"]+ assert data["run_id"] == "wr_heavy"+ assert data["recording_url"] == payload["recording_url"]+ assert data["output"] == payload["outputs"]+ assert data["workflow_title"] == "Heavy workflow"+++async def test_workflow_status_task_runs_still_use_get_run(monkeypatch: pytest.MonkeyPatch) -> None:+ task_run = SimpleNamespace(+ run_id="tsk_v2_123",+ status="completed",+ run_type="task_v2",+ output={"answer": "42"},+ failure_reason=None,+ step_count=4,+ recording_url=None,+ app_url=None,+ browser_session_id=None,+ run_with=None,+ created_at=None,+ modified_at=None,+ started_at=None,+ finished_at=None,+ queued_at=None,+ )+ fake_client = SimpleNamespace(get_run=AsyncMock(return_value=task_run))+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ result = await workflow_tools.skyvern_workflow_status(run_id="tsk_v2_123")++ fake_client.get_run.assert_awaited_once_with("tsk_v2_123")+ data = result["data"]+ assert data["run_id"] == "tsk_v2_123"+ assert data["step_count"] == 4+ assert data["output_summary"]["scalar_preview"] == {"answer": "42"}+++async def test_workflow_status_summary_via_mcp_client(monkeypatch: pytest.MonkeyPatch) -> None:+ payload = _heavy_workflow_run_payload(include_expanded_outputs=False)+ request = AsyncMock(return_value=_fake_http_response(payload))+ fake_client = SimpleNamespace(+ get_run=AsyncMock(),+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),+ )+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ async with Client(mcp) as client:+ result = await client.call_tool("skyvern_workflow_status", {"run_id": "wr_heavy"})++ assert result.is_error is False+ assert isinstance(result.data, dict)+ assert result.data["ok"] is True+ data = result.data["data"]+ assert data["run_id"] == "wr_heavy"+ assert data["artifact_summary"]["artifact_id_count"] == 36+ assert "recording_url" not in data+ assert "output" not in data+++async def test_workflow_status_full_via_mcp_client(monkeypatch: pytest.MonkeyPatch) -> None:+ payload = _heavy_workflow_run_payload(include_expanded_outputs=True)+ request = AsyncMock(return_value=_fake_http_response(payload))+ fake_client = SimpleNamespace(+ get_run=AsyncMock(),+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),+ )+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)++ async with Client(mcp) as client:+ result = await client.call_tool("skyvern_workflow_status", {"run_id": "wr_heavy", "verbosity": "full"})++ assert result.is_error is False+ assert isinstance(result.data, dict)+ assert result.data["ok"] is True+ data = result.data["data"]+ assert data["run_id"] == "wr_heavy"+ assert data["recording_url"] == payload["recording_url"]+ assert data["output"] == payload["outputs"]