Skip to content

Automate browser based workflows with AI

[SKY-8322] Compact MCP workflow status responses and fix wr lookup (#5084)

Marc Kelechava965fac68c72f
diff --git a/pyproject.toml b/pyproject.toml
index cf4525667..7f9ca99f5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -223,7 +223,7 @@ skip = ["webeye/actions/__init__.py", "forge/sdk/__init__.py"]
plugins = "sqlalchemy.ext.mypy.plugin"
[project.scripts]
-skyvern = "skyvern.cli.commands:cli_app"
+skyvern = "skyvern.__main__:main"
[tool.pytest.ini_options]
norecursedirs = ["eval", "tests/sdk"]
diff --git a/skyvern/__main__.py b/skyvern/__main__.py
index d9410e40e..127657ce4 100644
--- a/skyvern/__main__.py
+++ b/skyvern/__main__.py
@@ -1,4 +1,11 @@
-from skyvern.cli.commands import cli_app
+def main() -> None:
+ from skyvern._cli_bootstrap import configure_cli_bootstrap_logging # noqa: PLC0415
+
+ configure_cli_bootstrap_logging()
+ from skyvern.cli.commands import cli_app # noqa: PLC0415
-if __name__ == "__main__":
cli_app() # type: ignore
+
+
+if __name__ == "__main__":
+ main()
diff --git a/skyvern/_cli_bootstrap.py b/skyvern/_cli_bootstrap.py
new file mode 100644
index 000000000..2a238f826
--- /dev/null
+++ b/skyvern/_cli_bootstrap.py
@@ -0,0 +1,32 @@
+import logging
+from collections.abc import Set
+
+_DEFAULT_CLI_LOG_LEVEL = "WARNING"
+
+_QUIET_CLI_LOGGERS = ("skyvern", "httpx", "litellm", "playwright", "httpcore")
+
+
+def _resolve_cli_log_level_name(settings: object) -> str:
+ """Honor explicit settings while keeping CLI defaults quiet."""
+
+ fields_set: Set[str] = getattr(settings, "model_fields_set", set())
+ configured_level = str(getattr(settings, "LOG_LEVEL", _DEFAULT_CLI_LOG_LEVEL)).upper()
+ if "LOG_LEVEL" in fields_set:
+ return configured_level
+ return _DEFAULT_CLI_LOG_LEVEL
+
+
+def configure_cli_bootstrap_logging() -> None:
+ """Clamp CLI process logging before importing the command tree."""
+ from skyvern.config import settings # noqa: PLC0415
+ from skyvern.forge.sdk.forge_log import setup_logger # noqa: PLC0415
+
+ log_level_name = _resolve_cli_log_level_name(settings)
+ settings.LOG_LEVEL = log_level_name
+ setup_logger()
+ log_level = logging.getLevelName(log_level_name)
+ if not isinstance(log_level, int):
+ log_level = logging.WARNING
+ logging.getLogger().setLevel(log_level)
+ for logger_name in _QUIET_CLI_LOGGERS:
+ logging.getLogger(logger_name).setLevel(log_level)
diff --git a/skyvern/cli/commands/__init__.py b/skyvern/cli/commands/__init__.py
index 2771c1e3f..221bf587a 100644
--- a/skyvern/cli/commands/__init__.py
+++ b/skyvern/cli/commands/__init__.py
@@ -1,9 +1,7 @@
-import logging
-
import typer
from dotenv import load_dotenv
-from skyvern.forge.sdk.forge_log import setup_logger as _setup_logger
+from skyvern._cli_bootstrap import configure_cli_bootstrap_logging as _configure_cli_bootstrap_logging
from skyvern.utils.env_paths import resolve_backend_env_path
from ..auth_command import login as login_command
@@ -34,10 +32,8 @@ def configure_cli_logging() -> None:
return
_cli_logging_configured = True
- # Suppress noisy SDK/third-party logs for CLI execution only.
- for logger_name in ("skyvern", "httpx", "litellm", "playwright", "httpcore"):
- logging.getLogger(logger_name).setLevel(logging.WARNING)
- _setup_logger()
+ # Keep callback-time execution aligned with the entrypoint bootstrap.
+ _configure_cli_bootstrap_logging()
cli_app = typer.Typer(
diff --git a/skyvern/cli/commands/__main__.py b/skyvern/cli/commands/__main__.py
index 3ab570f15..ca79ac561 100644
--- a/skyvern/cli/commands/__main__.py
+++ b/skyvern/cli/commands/__main__.py
@@ -1,9 +1,16 @@
from dotenv import load_dotenv
+from skyvern._cli_bootstrap import configure_cli_bootstrap_logging
from skyvern.utils.env_paths import resolve_backend_env_path
-from . import cli_app
-if __name__ == "__main__": # pragma: no cover - manual CLI invocation
+def main() -> None:
+ configure_cli_bootstrap_logging()
+ from . import cli_app # noqa: PLC0415
+
load_dotenv(resolve_backend_env_path())
cli_app()
+
+
+if __name__ == "__main__": # pragma: no cover - manual CLI invocation
+ main()
diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.py
index 3ccc710ab..203c06e53 100644
--- a/skyvern/cli/mcp_tools/workflow.py
+++ b/skyvern/cli/mcp_tools/workflow.py
@@ -9,8 +9,9 @@ from __future__ import annotations
import asyncio
import json
+from datetime import datetime
from enum import Enum
-from typing import Annotated, Any
+from typing import Annotated, Any, Literal
import structlog
import yaml
@@ -26,6 +27,13 @@ from ._session import get_skyvern
from ._validation import validate_folder_id
LOG = structlog.get_logger()
+_SUMMARY_TOP_LEVEL_KEY_LIMIT = 8
+_SUMMARY_SCALAR_PREVIEW_LIMIT = 3
+_SUMMARY_ARTIFACT_PREVIEW_LIMIT = 4
+_SUMMARY_STRING_PREVIEW_LIMIT = 120
+_SUMMARY_RECURSION_LIMIT = 10
+_SCREENSHOT_LIST_KEYS = frozenset({"task_screenshots", "workflow_screenshots", "screenshot_urls"})
+_SCREENSHOT_ARTIFACT_ID_KEYS = frozenset({"task_screenshot_artifact_ids", "workflow_screenshot_artifact_ids"})
# ---------------------------------------------------------------------------
# Helpers
@@ -98,6 +106,268 @@ def _serialize_run(run: Any) -> dict[str, Any]:
return data
+def _get_value(obj: Any, key: str, default: Any = None) -> Any:
+ if isinstance(obj, dict):
+ return obj.get(key, default)
+ return getattr(obj, key, default)
+
+
+def _get_run_id(run: Any) -> str | None:
+ return _get_value(run, "run_id") or _get_value(run, "workflow_run_id")
+
+
+def _jsonable(value: Any) -> Any:
+ if hasattr(value, "model_dump"):
+ return value.model_dump(mode="json")
+ if isinstance(value, datetime):
+ return value.isoformat()
+ if isinstance(value, list):
+ return [_jsonable(item) for item in value]
+ if isinstance(value, tuple):
+ return [_jsonable(item) for item in value]
+ if isinstance(value, dict):
+ return {k: _jsonable(v) for k, v in value.items()}
+ return value
+
+
+def _truncate_preview(value: Any) -> Any:
+ value = _jsonable(value)
+ if isinstance(value, str) and len(value) > _SUMMARY_STRING_PREVIEW_LIMIT:
+ return f"{value[: _SUMMARY_STRING_PREVIEW_LIMIT - 3]}..."
+ return value
+
+
+def _is_scalarish(value: Any) -> bool:
+ return value is None or isinstance(value, str | int | float | bool)
+
+
+def _init_output_stats() -> dict[str, Any]:
+ return {
+ "has_extracted_information": False,
+ "nested_screenshot_count": 0,
+ "artifact_id_count": 0,
+ "artifact_ids_preview": [],
+ }
+
+
+def _note_artifact_ids(stats: dict[str, Any], values: list[Any]) -> None:
+ stats["artifact_id_count"] += len(values)
+ preview = stats["artifact_ids_preview"]
+ for value in values:
+ if len(preview) >= _SUMMARY_ARTIFACT_PREVIEW_LIMIT:
+ break
+ value_str = str(value)
+ if value_str not in preview:
+ preview.append(value_str)
+
+
+def _scan_output_value(value: Any, stats: dict[str, Any], depth: int = 0) -> None:
+ if depth > _SUMMARY_RECURSION_LIMIT:
+ return
+
+ value = _jsonable(value)
+
+ if isinstance(value, dict):
+ for key, nested_value in value.items():
+ if key == "extracted_information" and nested_value is not None:
+ stats["has_extracted_information"] = True
+ if key in _SCREENSHOT_LIST_KEYS and isinstance(nested_value, list):
+ stats["nested_screenshot_count"] += len(nested_value)
+ if key in _SCREENSHOT_ARTIFACT_ID_KEYS and isinstance(nested_value, list):
+ _note_artifact_ids(stats, nested_value)
+ _scan_output_value(nested_value, stats, depth + 1)
+ return
+
+ if isinstance(value, list):
+ for item in value:
+ _scan_output_value(item, stats, depth + 1)
+
+
+def _summarize_output_value(output_value: Any) -> tuple[dict[str, Any], dict[str, Any]]:
+ if output_value is None:
+ return {"present": False}, _init_output_stats()
+
+ output_value = _jsonable(output_value)
+ stats = _init_output_stats()
+ _scan_output_value(output_value, stats)
+
+ summary: dict[str, Any] = {"present": True}
+
+ if isinstance(output_value, dict):
+ top_level_keys = list(output_value.keys())
+ summary["top_level_keys"] = top_level_keys[:_SUMMARY_TOP_LEVEL_KEY_LIMIT]
+ if len(top_level_keys) > _SUMMARY_TOP_LEVEL_KEY_LIMIT:
+ summary["top_level_key_count"] = len(top_level_keys)
+ summary["block_output_count"] = len([key for key in top_level_keys if key != "extracted_information"])
+
+ scalar_preview: dict[str, Any] = {}
+ for key, value in output_value.items():
+ if key == "extracted_information":
+ continue
+ if _is_scalarish(value):
+ scalar_preview[key] = _truncate_preview(value)
+ elif (
+ isinstance(value, list)
+ and value
+ and len(value) <= _SUMMARY_SCALAR_PREVIEW_LIMIT
+ and all(_is_scalarish(item) for item in value)
+ ):
+ scalar_preview[key] = [_truncate_preview(item) for item in value]
+ if len(scalar_preview) >= _SUMMARY_SCALAR_PREVIEW_LIMIT:
+ break
+ if scalar_preview:
+ summary["scalar_preview"] = scalar_preview
+ elif isinstance(output_value, list):
+ summary["item_count"] = len(output_value)
+ if (
+ output_value
+ and len(output_value) <= _SUMMARY_SCALAR_PREVIEW_LIMIT
+ and all(_is_scalarish(item) for item in output_value)
+ ):
+ summary["scalar_preview"] = [_truncate_preview(item) for item in output_value]
+ else:
+ summary["scalar_preview"] = _truncate_preview(output_value)
+
+ summary["has_extracted_information"] = stats["has_extracted_information"]
+ summary["nested_screenshot_count"] = stats["nested_screenshot_count"]
+ summary["artifact_id_count"] = stats["artifact_id_count"]
+
+ return summary, stats
+
+
+def _summarize_artifacts(run: Any, output_stats: dict[str, Any]) -> dict[str, Any]:
+ downloaded_files = _jsonable(_get_value(run, "downloaded_files")) or []
+ screenshot_urls = _jsonable(_get_value(run, "screenshot_urls")) or []
+
+ summary: dict[str, Any] = {
+ "recording_available": bool(_get_value(run, "recording_url")),
+ "workflow_screenshot_count": len(screenshot_urls),
+ "downloaded_file_count": len(downloaded_files),
+ "artifact_id_count": output_stats["artifact_id_count"],
+ }
+
+ filenames = [
+ filename
+ for filename in (_get_value(file_info, "filename") for file_info in downloaded_files)
+ if isinstance(filename, str) and filename
+ ]
+ if filenames:
+ summary["downloaded_file_names"] = filenames[:_SUMMARY_SCALAR_PREVIEW_LIMIT]
+
+ if output_stats["artifact_ids_preview"]:
+ summary["artifact_ids_preview"] = output_stats["artifact_ids_preview"]
+
+ return summary
+
+
+def _serialize_run_summary(run: Any) -> dict[str, Any]:
+ run_id = _get_run_id(run)
+ run_type = _get_value(run, "run_type")
+ if run_type is None and _get_value(run, "workflow_run_id"):
+ run_type = "workflow_run"
+
+ output_value = _get_value(run, "output")
+ if output_value is None and _get_value(run, "outputs") is not None:
+ output_value = _get_value(run, "outputs")
+
+ output_summary, output_stats = _summarize_output_value(output_value)
+
+ summary: dict[str, Any] = {
+ "run_id": run_id,
+ "status": str(_get_value(run, "status")) if _get_value(run, "status") is not None else None,
+ "run_type": str(run_type) if run_type is not None else None,
+ "artifact_summary": _summarize_artifacts(run, output_stats),
+ "output_summary": output_summary,
+ }
+
+ failure_reason = _get_value(run, "failure_reason")
+ if failure_reason:
+ summary["failure_reason"] = failure_reason
+
+ run_with = _get_value(run, "run_with")
+ if run_with:
+ summary["run_with"] = run_with
+
+ workflow_title = _get_value(run, "workflow_title")
+ if workflow_title:
+ summary["workflow_title"] = workflow_title
+
+ step_count = _get_value(run, "step_count")
+ total_steps = _get_value(run, "total_steps")
+ if step_count is not None:
+ summary["step_count"] = step_count
+ elif total_steps is not None:
+ summary["total_steps"] = total_steps
+
+ return {key: value for key, value in summary.items() if value is not None}
+
+
+def _serialize_run_full(run: Any) -> dict[str, Any]:
+ if not isinstance(run, dict):
+ return _serialize_run(run)
+
+ data: dict[str, Any] = {
+ "run_id": _get_run_id(run),
+ "status": str(_get_value(run, "status")) if _get_value(run, "status") is not None else None,
+ "run_type": "workflow_run" if _get_value(run, "workflow_run_id") else _get_value(run, "run_type"),
+ }
+
+ for field in (
+ "workflow_id",
+ "workflow_title",
+ "failure_reason",
+ "recording_url",
+ "screenshot_urls",
+ "downloaded_files",
+ "downloaded_file_urls",
+ "parameters",
+ "errors",
+ "browser_session_id",
+ "browser_profile_id",
+ "run_with",
+ "total_steps",
+ ):
+ value = _get_value(run, field)
+ if value is not None:
+ data[field] = _jsonable(value)
+
+ outputs = _get_value(run, "outputs")
+ if outputs is not None:
+ data["output"] = _jsonable(outputs)
+
+ for ts_field in ("created_at", "modified_at", "started_at", "finished_at", "queued_at"):
+ value = _get_value(run, ts_field)
+ if value is not None:
+ data[ts_field] = _jsonable(value)
+
+ return {key: value for key, value in data.items() if value is not None}
+
+
+async def _get_workflow_run_status(
+ workflow_run_id: str,
+ *,
+ include_output_details: bool,
+) -> dict[str, Any]:
+ skyvern = get_skyvern()
+ # The generated SDK only exposes get_run() for /v1/runs/{run_id}; wr_... IDs
+ # require the workflow-run detail route until a public SDK helper exists.
+ response = await skyvern._client_wrapper.httpx_client.request(
+ f"api/v1/workflows/runs/{workflow_run_id}",
+ method="GET",
+ params={"include_output_details": include_output_details},
+ )
+ if response.status_code == 404:
+ raise NotFoundError(body={"detail": f"Workflow run {workflow_run_id!r} not found"})
+ if response.status_code >= 400:
+ detail = ""
+ try:
+ detail = response.json().get("detail", response.text)
+ except Exception:
+ detail = response.text
+ raise RuntimeError(f"HTTP {response.status_code}: {detail}")
+ return response.json()
+
+
def _validate_workflow_id(workflow_id: str, action: str) -> dict[str, Any] | None:
"""Validate workflow_id format. Returns a make_result error dict or None if valid."""
if "/" in workflow_id or "\\" in workflow_id:
@@ -911,17 +1181,36 @@ async def skyvern_workflow_run(
async def skyvern_workflow_status(
run_id: Annotated[str, "Run ID to check (wr_... for workflow runs, tsk_v2_... for task runs)"],
+ verbosity: Annotated[
+ Literal["summary", "full"],
+ Field(description="`summary` returns a compact status payload. `full` includes outputs, timestamps, and URLs."),
+ ] = "summary",
) -> dict[str, Any]:
"""Check the status and progress of a workflow or task run. Use when you need to monitor
a running workflow, check if it completed, or retrieve its output."""
if err := _validate_run_id(run_id, "skyvern_workflow_status"):
return err
-
- skyvern = get_skyvern()
+ if verbosity not in {"summary", "full"}:
+ return make_result(
+ "skyvern_workflow_status",
+ ok=False,
+ error=make_error(
+ ErrorCode.INVALID_INPUT,
+ f"Invalid verbosity: {verbosity!r}",
+ "Use verbosity='summary' for compact status or verbosity='full' for full detail.",
+ ),
+ )
with Timer() as timer:
try:
- run = await skyvern.get_run(run_id)
+ if run_id.startswith("wr_"):
+ run = await _get_workflow_run_status(
+ run_id,
+ include_output_details=verbosity == "full",
+ )
+ else:
+ skyvern = get_skyvern()
+ run = await skyvern.get_run(run_id)
timer.mark("sdk")
except NotFoundError:
return make_result(
@@ -942,8 +1231,14 @@ async def skyvern_workflow_status(
error=make_error(ErrorCode.API_ERROR, str(e), "Check the run ID and your API key"),
)
- data = _serialize_run(run)
- data["sdk_equivalent"] = f"await skyvern.get_run({run_id!r})"
+ data = _serialize_run_full(run) if verbosity == "full" else _serialize_run_summary(run)
+ if run_id.startswith("wr_"):
+ data["sdk_equivalent"] = f"await skyvern_workflow_status(run_id={run_id!r}, verbosity={verbosity!r})"
+ else:
+ verbosity_arg = "" if verbosity == "summary" else f", verbosity={verbosity!r}"
+ data["sdk_equivalent"] = (
+ f"await skyvern.get_run({run_id!r}) # or skyvern_workflow_status(run_id={run_id!r}{verbosity_arg})"
+ )
return make_result("skyvern_workflow_status", data=data, timing_ms=timer.timing_ms)
diff --git a/skyvern/cli/workflow.py b/skyvern/cli/workflow.py
index edd8295db..faf0c4c1b 100644
--- a/skyvern/cli/workflow.py
+++ b/skyvern/cli/workflow.py
@@ -253,7 +253,7 @@ def workflow_status(
"""Get workflow run status."""
async def _run() -> dict[str, Any]:
- return await tool_workflow_status(run_id=run_id)
+ return await tool_workflow_status(run_id=run_id, verbosity="full")
_run_tool(_run, json_output=json_output, hint_on_exception="Check the run ID and API key.")
diff --git a/skyvern/config.py b/skyvern/config.py
index ebc461c64..040d64514 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -28,6 +28,7 @@ class Settings(BaseSettings):
# settings for experimentation
ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE: bool = False
+ ENABLE_DOM_PARSER_V2: bool = False
ADDITIONAL_MODULES: list[str] = []
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 22fce7ea1..fca0798a8 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -1,6 +1,7 @@
import asyncio
import copy
import hashlib
+from collections import deque
from datetime import timedelta
from typing import Dict, List
@@ -81,10 +82,8 @@ def _get_shape_cache_key(hash: str) -> str:
return f"skyvern:shape:{hash}"
-def _remove_skyvern_attributes(element: Dict) -> Dict:
- """
- To get the original HTML element without skyvern attributes
- """
+def _remove_skyvern_attributes_legacy(element: Dict) -> Dict:
+ """Legacy: deep-copy based attribute removal (pre-V2)."""
element_copied = copy.deepcopy(element)
for attr in ELEMENT_NODE_ATTRIBUTES:
if element_copied.get(attr):
@@ -102,12 +101,36 @@ def _remove_skyvern_attributes(element: Dict) -> Dict:
trimmed_children = []
for child in children:
- trimmed_children.append(_remove_skyvern_attributes(child))
+ trimmed_children.append(_remove_skyvern_attributes_legacy(child))
element_copied["children"] = trimmed_children
return element_copied
+def _remove_skyvern_attributes(element: Dict) -> Dict:
+ """
+ To get the original HTML element without skyvern attributes.
+ V2 uses shallow copy + filtered dicts instead of double deepcopy for performance.
+ """
+ if not settings.ENABLE_DOM_PARSER_V2:
+ return _remove_skyvern_attributes_legacy(element)
+
+ element_copied = dict(element)
+ for attr in ELEMENT_NODE_ATTRIBUTES:
+ element_copied.pop(attr, None)
+
+ if "attributes" in element_copied:
+ element_copied["attributes"] = {
+ k: v for k, v in element_copied["attributes"].items() if k not in USELESS_SHAPE_ATTRIBUTE
+ }
+
+ children: List[Dict] | None = element_copied.get("children")
+ if children is not None:
+ element_copied["children"] = [_remove_skyvern_attributes(child) for child in children]
+
+ return element_copied
+
+
def _add_to_dropped_css_svg_element_map(hashed_key: str | None) -> None:
context = skyvern_context.ensure_context()
if hashed_key:
@@ -519,7 +542,7 @@ class AgentFunction:
skyvern_frame = await SkyvernFrame.create_instance(frame=frame)
current_frame_index = context.frame_index_map.get(frame, 0)
- queue = []
+ queue: deque = deque()
element_cnt = 0
eligible_svgs = [] # List to store eligible SVGs and their frames
@@ -527,7 +550,7 @@ class AgentFunction:
queue.append(element)
while queue:
- queue_ele = queue.pop(0)
+ queue_ele = queue.popleft()
element_cnt += 1
if element_cnt == MAX_ELEMENT_CNT:
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index af412daf4..8b1ff6e0b 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -157,7 +157,8 @@ class DomUtils {
// this.
let computedStyle;
if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {
- for (const child of Array.from(element.children)) {
+ for (let ci = 0; ci < element.children.length; ci++) {
+ const child = element.children[ci];
computedStyle = getElementComputedStyle(child, null);
if (!computedStyle) {
continue;
@@ -360,11 +361,27 @@ class QuadTreeNode {
}
}
-// from playwright
+// from playwright — with per-tree-build cache for non-pseudo styles
+const _computedStyleCache = new WeakMap();
+
function getElementComputedStyle(element, pseudo) {
- return element.ownerDocument && element.ownerDocument.defaultView
- ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)
- : undefined;
+ if (!element.ownerDocument || !element.ownerDocument.defaultView) {
+ return undefined;
+ }
+ // Cache only non-pseudo styles (the hot path called 3-5× per element)
+ if (pseudo == null) {
+ const cached = _computedStyleCache.get(element);
+ if (cached !== undefined) {
+ return cached;
+ }
+ const style = element.ownerDocument.defaultView.getComputedStyle(
+ element,
+ null,
+ );
+ _computedStyleCache.set(element, style);
+ return style;
+ }
+ return element.ownerDocument.defaultView.getComputedStyle(element, pseudo);
}
// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98
@@ -411,8 +428,14 @@ function hasASPClientControl() {
return typeof ASPxClientControl !== "undefined";
}
+// Cache for isHoverOnlyElement — called up to 3× per element
+// (isElementVisible, isInteractable, buildElementObject)
+const _hoverOnlyCache = new WeakMap();
+
// Check if element is only visible on hover (e.g., hover-only buttons)
function isHoverOnlyElement(element) {
+ const cached = _hoverOnlyCache.get(element);
+ if (cached !== undefined) return cached;
// Check for common hover-only patterns in class names
const className = element.className?.toString() ?? "";
const parentClassName = element.parentElement?.className?.toString() ?? "";
@@ -424,6 +447,7 @@ function isHoverOnlyElement(element) {
parentClassName.includes("hover-") ||
parentClassName.includes("-hover")
) {
+ _hoverOnlyCache.set(element, true);
return true;
}
@@ -440,12 +464,14 @@ function isHoverOnlyElement(element) {
parentClass.includes("item")
) {
// This element might be revealed on parent hover
+ _hoverOnlyCache.set(element, true);
return true;
}
parent = parent.parentElement;
depth += 1;
}
+ _hoverOnlyCache.set(element, false);
return false;
}
@@ -454,9 +480,10 @@ function isHoverOnlyElement(element) {
function isElementVisible(element) {
// TODO: This is a hack to not check visibility for option elements
// because they are not visible by default. We check their parent instead for visibility.
+ const elTagName = element.tagName.toLowerCase();
if (
- element.tagName.toLowerCase() === "option" ||
- (element.tagName.toLowerCase() === "input" &&
+ elTagName === "option" ||
+ (elTagName === "input" &&
(element.type === "radio" || element.type === "checkbox"))
)
return element.parentElement && isElementVisible(element.parentElement);
@@ -622,11 +649,10 @@ function expectHitTarget(hitPoint, targetElement) {
}
function getChildElements(element) {
- if (element.childElementCount !== 0) {
- return Array.from(element.children);
- } else {
- return [];
- }
+ // Return the live HTMLCollection directly — callers iterate by index,
+ // and processElement only modifies attributes (not child structure),
+ // so the live collection remains stable during iteration.
+ return element.children;
}
function isParent(parent, child) {
@@ -661,9 +687,10 @@ function isHidden(element) {
return true;
}
if (element.hidden) {
+ const tn = element.tagName.toLowerCase();
if (
style?.cursor === "pointer" &&
- element.tagName.toLowerCase() === "input" &&
+ tn === "input" &&
(element.type === "submit" || element.type === "button")
) {
// there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.
@@ -723,6 +750,25 @@ function hasAngularClickBinding(element) {
);
}
+// Pre-computed Set for widget roles — avoids recreating array on every hasWidgetRole call
+const WIDGET_ROLES = new Set([
+ "button",
+ "link",
+ "checkbox",
+ "menuitem",
+ "menuitemcheckbox",
+ "menuitemradio",
+ "radio",
+ "tab",
+ "combobox",
+ "searchbox",
+ "slider",
+ "spinbutton",
+ "switch",
+ "gridcell",
+ "option",
+]);
+
function hasWidgetRole(element) {
const role = element.getAttribute("role");
if (!role) {
@@ -730,44 +776,15 @@ function hasWidgetRole(element) {
}
// https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles
// Not all roles make sense for the time being so we only check for the ones that do
- if (role.toLowerCase().trim() === "textbox") {
+ const normalizedRole = role.toLowerCase().trim();
+ if (normalizedRole === "textbox") {
return !isReadonlyElement(element);
}
-
- const widgetRoles = [
- "button",
- "link",
- "checkbox",
- "menuitem",
- "menuitemcheckbox",
- "menuitemradio",
- "radio",
- "tab",
- "combobox",
- "searchbox",
- "slider",
- "spinbutton",
- "switch",
- "gridcell",
- "option",
- ];
- return widgetRoles.includes(role.toLowerCase().trim());
+ return WIDGET_ROLES.has(normalizedRole);
}
function isTableRelatedElement(element) {
- const tagName = element.tagName.toLowerCase();
- return [
- "table",
- "caption",
- "thead",
- "tbody",
- "tfoot",
- "tr",
- "th",
- "td",
- "colgroup",
- "col",
- ].includes(tagName);
+ return TABLE_RELATED_TAGS.has(element.tagName.toLowerCase());
}
function isDOMNodeRepresentDiv(element) {
@@ -844,17 +861,74 @@ function isValidCSSSelector(selector) {
}
}
-function isInteractable(element, hoverStylesMap) {
- if (!isElementVisible(element)) {
- return false;
- }
+// Pre-computed tag sets for O(1) lookup in isInteractable hot path
+const ALWAYS_INTERACTABLE_TAGS = new Set([
+ "button",
+ "select",
+ "option",
+ "textarea",
+]);
+const POINTER_CHECK_TAGS = new Set([
+ "div",
+ "img",
+ "span",
+ "a",
+ "i",
+ "li",
+ "p",
+ "td",
+ "svg",
+ "strong",
+ "h1",
+ "h2",
+ "h3",
+ "h4",
+]);
+const SKIP_TAGS = new Set(["html", "iframe", "frameset", "frame"]);
+// Attributes that should be normalized to boolean true/false in buildElementObject
+const BOOLEAN_ATTRS = new Set([
+ "required",
+ "aria-required",
+ "checked",
+ "aria-checked",
+ "selected",
+ "aria-selected",
+ "readonly",
+ "aria-readonly",
+ "disabled",
+ "aria-disabled",
+]);
+const TABLE_RELATED_TAGS = new Set([
+ "table",
+ "caption",
+ "thead",
+ "tbody",
+ "tfoot",
+ "tr",
+ "th",
+ "td",
+ "colgroup",
+ "col",
+]);
+
+function isInteractable(
+ element,
+ hoverStylesMap,
+ skipVisibilityChecks = false,
+ pageFlags = {},
+) {
+ if (!skipVisibilityChecks) {
+ if (!isElementVisible(element)) {
+ return false;
+ }
- if (isHidden(element)) {
- return false;
- }
+ if (isHidden(element)) {
+ return false;
+ }
- if (isScriptOrStyle(element)) {
- return false;
+ if (isScriptOrStyle(element)) {
+ return false;
+ }
}
if (hasWidgetRole(element)) {
@@ -879,19 +953,7 @@ function isInteractable(element, hoverStylesMap) {
}
const tagName = element.tagName.toLowerCase();
- if (tagName === "html") {
- return false;
- }
-
- if (tagName === "iframe") {
- return false;
- }
-
- if (tagName === "frameset") {
- return false;
- }
-
- if (tagName === "frame") {
+ if (SKIP_TAGS.has(tagName)) {
return false;
}
@@ -904,12 +966,7 @@ function isInteractable(element, hoverStylesMap) {
return false;
}
- if (
- tagName === "button" ||
- tagName === "select" ||
- tagName === "option" ||
- tagName === "textarea"
- ) {
+ if (ALWAYS_INTERACTABLE_TAGS.has(tagName)) {
return true;
}
@@ -943,18 +1000,11 @@ function isInteractable(element, hoverStylesMap) {
// support listbox and options underneath it
// div element should be checked here before the css pointer
- if (
- (tagName === "ul" || tagName === "div") &&
- element.hasAttribute("role") &&
- element.getAttribute("role").toLowerCase() === "listbox"
- ) {
+ const roleAttr = element.getAttribute("role")?.toLowerCase();
+ if ((tagName === "ul" || tagName === "div") && roleAttr === "listbox") {
return true;
}
- if (
- (tagName === "li" || tagName === "div") &&
- element.hasAttribute("role") &&
- element.getAttribute("role").toLowerCase() === "option"
- ) {
+ if ((tagName === "li" || tagName === "div") && roleAttr === "option") {
return true;
}
@@ -974,6 +1024,7 @@ function isInteractable(element, hoverStylesMap) {
if (
tagName === "div" &&
className.includes("pac-item") &&
+ pageFlags.hasPacContainers !== false &&
element.closest('div[class*="pac-container"]')
) {
return true;
@@ -981,33 +1032,23 @@ function isInteractable(element, hoverStylesMap) {
if (
tagName === "div" &&
- element.hasAttribute("aria-disabled") &&
- element.getAttribute("aria-disabled").toLowerCase() === "false"
+ element.getAttribute("aria-disabled")?.toLowerCase() === "false"
) {
return true;
}
- if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {
+ if (
+ tagName === "span" &&
+ pageFlags.hasDropdownContainers !== false &&
+ element.closest('div[id*="dropdown-container"]')
+ ) {
return true;
}
// FIXME: maybe we need to enable the pointer check for all elements?
if (
- tagName === "div" ||
- tagName === "img" ||
- tagName === "span" ||
- tagName === "a" ||
- tagName === "i" ||
- tagName === "li" ||
- tagName === "p" ||
- tagName === "td" ||
- tagName === "svg" ||
- tagName === "strong" ||
- tagName === "h1" ||
- tagName === "h2" ||
- tagName === "h3" ||
- tagName === "h4" ||
- // sometime it's a customized element like <my-login-button>, we should check pointer style
+ POINTER_CHECK_TAGS.has(tagName) ||
+ // customized elements like <my-login-button>, check pointer style
tagName.includes("button") ||
tagName.includes("select") ||
tagName.includes("option") ||
@@ -1046,11 +1087,7 @@ function isInteractable(element, hoverStylesMap) {
}
// consider <div tabindex="0"> as interactable
- if (
- tagName.toLowerCase() === "div" &&
- element.hasAttribute("tabindex") &&
- element.getAttribute("tabindex").toLowerCase() === "0"
- ) {
+ if (tagName === "div" && element.getAttribute("tabindex") === "0") {
return true;
}
@@ -1099,15 +1136,10 @@ const isComboboxDropdown = (element) => {
if (element.tagName.toLowerCase() !== "input") {
return false;
}
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
- const readonly =
- element.getAttribute("readonly") &&
- element.getAttribute("readonly").toLowerCase() !== "false";
+ const role = element.getAttribute("role")?.toLowerCase() ?? "";
+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
+ const readonlyVal = element.getAttribute("readonly");
+ const readonly = readonlyVal && readonlyVal.toLowerCase() !== "false";
const controls = element.hasAttribute("aria-controls");
return role && haspopup && controls && readonly;
};
@@ -1117,24 +1149,16 @@ const isDivComboboxDropdown = (element) => {
if (tagName !== "div") {
return false;
}
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
+ const role = element.getAttribute("role")?.toLowerCase() ?? "";
+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
const controls = element.hasAttribute("aria-controls");
return role === "combobox" && controls && haspopup;
};
const isDropdownButton = (element) => {
const tagName = element.tagName.toLowerCase();
- const type = element.getAttribute("type")
- ? element.getAttribute("type").toLowerCase()
- : "";
- const haspopup = element.getAttribute("aria-haspopup")
- ? element.getAttribute("aria-haspopup").toLowerCase()
- : "";
+ const type = element.getAttribute("type")?.toLowerCase() ?? "";
+ const haspopup = element.getAttribute("aria-haspopup")?.toLowerCase() ?? "";
const hasExpanded = element.hasAttribute("aria-expanded");
return (
tagName === "button" &&
@@ -1146,9 +1170,7 @@ const isDropdownButton = (element) => {
const isSelect2Dropdown = (element) => {
const tagName = element.tagName.toLowerCase();
const className = element.className.toString();
- const role = element.getAttribute("role")
- ? element.getAttribute("role").toLowerCase()
- : "";
+ const role = element.getAttribute("role")?.toLowerCase() ?? "";
if (tagName === "a") {
return className.includes("select2-choice");
@@ -1249,9 +1271,7 @@ const isAngularDropdown = (element) => {
const tagName = element.tagName.toLowerCase();
if (tagName === "input" || tagName === "span") {
- const ariaLabel = element.hasAttribute("aria-label")
- ? element.getAttribute("aria-label").toLowerCase()
- : "";
+ const ariaLabel = element.getAttribute("aria-label")?.toLowerCase() ?? "";
return ariaLabel.includes("select") || ariaLabel.includes("choose");
}
@@ -1272,21 +1292,34 @@ const isAngularMaterialDatePicker = (element) => {
);
};
+// Cache for pseudo-element content — called in hasBeforeOrAfterPseudoContent
+// and again in buildElementObject for the same element
+const _pseudoContentCache = new WeakMap();
+
function getPseudoContent(element, pseudo) {
- const pseudoStyle = getElementComputedStyle(element, pseudo);
- if (!pseudoStyle) {
- return null;
+ let elementCache = _pseudoContentCache.get(element);
+ if (elementCache && pseudo in elementCache) {
+ return elementCache[pseudo];
}
- const content = pseudoStyle
- .getPropertyValue("content")
- .replace(/"/g, "")
- .trim();
- if (content === "none" || !content) {
- return null;
+ const pseudoStyle = getElementComputedStyle(element, pseudo);
+ let result = null;
+ if (pseudoStyle) {
+ const content = pseudoStyle
+ .getPropertyValue("content")
+ .replace(/"/g, "")
+ .trim();
+ if (content !== "none" && content) {
+ result = content;
+ }
}
- return content;
+ if (!elementCache) {
+ elementCache = {};
+ _pseudoContentCache.set(element, elementCache);
+ }
+ elementCache[pseudo] = result;
+ return result;
}
function hasBeforeOrAfterPseudoContent(element) {
@@ -1312,12 +1345,8 @@ function removeMultipleSpaces(str) {
return str;
}
- // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement
- if (
- str.indexOf(" ") === -1 &&
- str.indexOf("\t") === -1 &&
- str.indexOf("\n") === -1
- ) {
+ // Optimization: check if contains whitespace worth collapsing before running regex
+ if (!/\s{2}|[\t\n]/.test(str)) {
return str;
}
@@ -1346,10 +1375,9 @@ function cleanupText(text) {
}
const checkStringIncludeRequire = (str) => {
+ const lower = str.toLowerCase();
return (
- str.toLowerCase().includes("*") ||
- str.toLowerCase().includes("✱") ||
- str.toLowerCase().includes("require")
+ lower.includes("*") || lower.includes("✱") || lower.includes("require")
);
};
@@ -1434,10 +1462,10 @@ function getElementText(element) {
}
function getSelectOptions(element) {
- const options = Array.from(element.options);
const selectOptions = [];
- for (const option of options) {
+ for (let i = 0; i < element.options.length; i++) {
+ const option = element.options[i];
selectOptions.push({
optionIndex: option.index,
text: removeMultipleSpaces(option.textContent),
@@ -1518,6 +1546,7 @@ async function buildElementObject(
element,
interactable,
purgeable = false,
+ cachedIsInSvg = null,
) {
var element_id = element.getAttribute("unique_id") ?? (await uniqueId());
var elementTagNameLower = element.tagName.toLowerCase();
@@ -1527,23 +1556,9 @@ async function buildElementObject(
if (element.attributes[Symbol.iterator]) {
for (const attr of element.attributes) {
var attrValue = attr.value;
- if (
- attr.name === "required" ||
- attr.name === "aria-required" ||
- attr.name === "checked" ||
- attr.name === "aria-checked" ||
- attr.name === "selected" ||
- attr.name === "aria-selected" ||
- attr.name === "readonly" ||
- attr.name === "aria-readonly" ||
- attr.name === "disabled" ||
- attr.name === "aria-disabled"
- ) {
- if (attrValue && attrValue.toLowerCase() === "false") {
- attrValue = false;
- } else {
- attrValue = true;
- }
+ if (BOOLEAN_ATTRS.has(attr.name)) {
+ attrValue =
+ attrValue && attrValue.toLowerCase() === "false" ? false : true;
}
attrs[attr.name] = attrValue;
}
@@ -1637,7 +1652,10 @@ async function buildElementObject(
purgeable: purgeable,
// don't trim any attr of this element if keepAllAttr=True
keepAllAttr:
- elementTagNameLower === "svg" || element.closest("svg") !== null,
+ elementTagNameLower === "svg" ||
+ (cachedIsInSvg !== null
+ ? cachedIsInSvg
+ : element.closest("svg") !== null),
isSelectable:
elementTagNameLower === "select" ||
isDatePickerSelector(element) ||
@@ -1729,6 +1747,15 @@ async function buildElementTree(
var elements = [];
var resultArray = [];
+ // O(1) parent lookup map — replaces the O(n) elements.find() that caused O(n²) tree building
+ const elementIdMap = new Map();
+ // Page-level check: does the page have any dropdown-container divs?
+ // If not, skip expensive .closest() for every span element.
+ const hasDropdownContainers =
+ document.querySelector('div[id*="dropdown-container"]') !== null;
+ // Page-level check: does the page have any Google Maps pac-container?
+ const hasPacContainers =
+ document.querySelector('div[class*="pac-container"]') !== null;
async function processElement(
element,
@@ -1784,7 +1811,12 @@ async function buildElementTree(
}
const isVisible = isElementVisible(element);
if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {
- let interactable = isInteractable(element, hoverStylesMap);
+ // Pass skipVisibilityChecks=true since we already verified visible+not hidden+not script/style.
+ // Pass pageFlags so .closest() calls are skipped when the page lacks those containers.
+ let interactable = isInteractable(element, hoverStylesMap, true, {
+ hasDropdownContainers,
+ hasPacContainers,
+ });
let elementObj = null;
let isParentSVG = null;
if (must_included_tags.includes(tagName)) {
@@ -1807,65 +1839,76 @@ async function buildElementTree(
} else if (hasBeforeOrAfterPseudoContent(element)) {
elementObj = await buildElementObject(frame, element, interactable);
} else if (tagName === "svg") {
- elementObj = await buildElementObject(frame, element, interactable);
+ elementObj = await buildElementObject(
+ frame,
+ element,
+ interactable,
+ false,
+ true,
+ );
} else if (
(isParentSVG = element.closest("svg")) &&
isParentSVG.getAttribute("unique_id")
) {
// if element is the children of the <svg> with an unique_id
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (
- tagName === "embed" &&
- element.getAttribute("type")?.toLowerCase() === "application/pdf"
- ) {
elementObj = await buildElementObject(
frame,
element,
interactable,
+ false,
true,
);
+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
+ elementObj = await buildElementObject(frame, element, interactable);
} else if (
- getElementText(element).length > 0 &&
- getElementText(element).length <= 5000
+ tagName === "embed" &&
+ element.getAttribute("type")?.toLowerCase() === "application/pdf"
) {
- if (window.GlobalEnableAllTextualElements) {
- // force all textual elements to be interactable
- interactable = true;
- }
- elementObj = await buildElementObject(frame, element, interactable);
- } else if (full_tree) {
- // when building full tree, we only get text from element itself
- // elements without text are purgeable
elementObj = await buildElementObject(
frame,
element,
interactable,
true,
);
- if (elementObj.text.length > 0) {
- elementObj.purgeable = false;
+ } else {
+ // Cache getElementText() to avoid calling it twice
+ const cachedText = getElementText(element);
+ if (cachedText.length > 0 && cachedText.length <= 5000) {
+ if (window.GlobalEnableAllTextualElements) {
+ // force all textual elements to be interactable
+ interactable = true;
+ }
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (full_tree) {
+ // when building full tree, we only get text from element itself
+ // elements without text are purgeable
+ elementObj = await buildElementObject(
+ frame,
+ element,
+ interactable,
+ true,
+ );
+ if (elementObj.text.length > 0) {
+ elementObj.purgeable = false;
+ }
}
}
if (elementObj) {
elementObj.xpath = current_xpath;
elements.push(elementObj);
- // If the element is interactable but has no interactable parent,
+ elementIdMap.set(elementObj.id, elementObj);
+ // If the element has no parent in the tree,
// then it starts a new tree, so add it to the result array
- // and set its id as the interactable parent id for the next elements
- // under it
if (parentId === null) {
resultArray.push(elementObj);
}
- // If the element is interactable and has an interactable parent,
- // then add it to the children of the parent
+ // Otherwise add it to the children of the parent via O(1) map lookup
else {
- // TODO: use dict/object so that we access these in O(1) instead
- elements
- .find((element) => element.id === parentId)
- .children.push(elementObj);
+ const parentObj = elementIdMap.get(parentId);
+ if (parentObj) {
+ parentObj.children.push(elementObj);
+ }
}
parentId = elementObj.id;
}
@@ -1909,24 +1952,51 @@ async function buildElementTree(
return;
}
+ let text = element.text;
+ if (!text) {
+ // Still recurse into children
+ for (let i = 0; i < element.children.length; i++) {
+ trimDuplicatedText(element.children[i]);
+ }
+ return;
+ }
+
+ // Collect all substrings to remove, then remove them in one pass
+ const textsToRemove = [];
+
// if the element has options, text will be duplicated with the option text
if (element.options) {
- element.options.forEach((option) => {
- element.text = element.text.replace(option.text, "");
- });
+ for (let i = 0; i < element.options.length; i++) {
+ if (element.options[i].text) {
+ textsToRemove.push(element.options[i].text);
+ }
+ }
}
- // BFS to delete duplicated text
- element.children.forEach((child) => {
- // delete duplicated text in the tree
- element.text = element.text.replace(child.text, "");
- trimDuplicatedText(child);
- });
+ for (let i = 0; i < element.children.length; i++) {
+ if (element.children[i].text) {
+ textsToRemove.push(element.children[i].text);
+ }
+ }
- // trim multiple ";"
- element.text = element.text.replace(/;+/g, ";");
- // trimleft and trimright ";"
- element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");
+ // Remove all collected substrings from parent text
+ for (let i = 0; i < textsToRemove.length; i++) {
+ const idx = text.indexOf(textsToRemove[i]);
+ if (idx !== -1) {
+ text =
+ text.substring(0, idx) +
+ text.substring(idx + textsToRemove[i].length);
+ }
+ }
+
+ // Clean up semicolons in one pass using a single regex
+ text = text.replace(/;+/g, ";").replace(/^;+|;+$/g, "");
+ element.text = text;
+
+ // Recurse into children
+ for (let i = 0; i < element.children.length; i++) {
+ trimDuplicatedText(element.children[i]);
+ }
};
// some elements without children nodes should be removed out, such as <label>
@@ -2068,14 +2138,19 @@ function groupElementsVisually(elements) {
return groups;
}
-// Helper functions
+// Helper functions — single-pass min/max avoids 4 intermediate arrays + spread
function calculateBounds(elements) {
- const rects = elements.map((el) => el.rect);
- const left = Math.min(...rects.map((r) => r.left));
- const top = Math.min(...rects.map((r) => r.top));
- const right = Math.max(...rects.map((r) => r.right));
- const bottom = Math.max(...rects.map((r) => r.bottom));
-
+ let left = Infinity,
+ top = Infinity,
+ right = -Infinity,
+ bottom = -Infinity;
+ for (let i = 0; i < elements.length; i++) {
+ const r = elements[i].rect;
+ if (r.left < left) left = r.left;
+ if (r.top < top) top = r.top;
+ if (r.right > right) right = r.right;
+ if (r.bottom > bottom) bottom = r.bottom;
+ }
return {
x: left,
y: top,
@@ -2110,11 +2185,18 @@ function findOverlappingElements(element, allElements, quadTree, processed) {
}
function createRectangleForGroup(group) {
- const rects = group.elements.map((element) => element.rect);
- const top = Math.min(...rects.map((rect) => rect.top));
- const left = Math.min(...rects.map((rect) => rect.left));
- const bottom = Math.max(...rects.map((rect) => rect.bottom));
- const right = Math.max(...rects.map((rect) => rect.right));
+ const elems = group.elements;
+ let top = Infinity,
+ left = Infinity,
+ bottom = -Infinity,
+ right = -Infinity;
+ for (let i = 0; i < elems.length; i++) {
+ const r = elems[i].rect;
+ if (r.top < top) top = r.top;
+ if (r.left < left) left = r.left;
+ if (r.bottom > bottom) bottom = r.bottom;
+ if (r.right > right) right = r.right;
+ }
return Rect.create(left, top, right, bottom);
}
@@ -2427,12 +2509,16 @@ function scrollToElementTop(element) {
*/
async function getHoverStylesMap() {
const hoverMap = new Map();
- const sheets = [...document.styleSheets];
+ const sheets = Array.from(document.styleSheets);
const parseCssSheet = (sheet) => {
const rules = sheet.cssRules || sheet.rules;
for (const rule of rules) {
if (rule.type === 1 && rule.selectorText) {
+ // Fast-path: skip rules that don't contain :hover at all
+ if (!rule.selectorText.includes(":hover")) {
+ continue;
+ }
// Split multiple selectors (e.g., "a:hover, button:hover")
const selectors = rule.selectorText.split(",").map((s) => s.trim());
@@ -2454,8 +2540,17 @@ async function getHoverStylesMap() {
continue;
}
+ // Early check: only keep rules that set cursor (or already have it from prior rules)
+ const existingStyles = hoverMap.get(baseSelector);
+ const hasCursorInRule = rule.style.cursor;
+ const hasCursorInExisting =
+ existingStyles && "cursor" in existingStyles;
+ if (!hasCursorInRule && !hasCursorInExisting) {
+ continue;
+ }
+
// Get or create styles object for this selector
- let styles = hoverMap.get(baseSelector) || {};
+ let styles = existingStyles || {};
// Add all style properties
for (const prop of rule.style) {
@@ -2467,18 +2562,16 @@ async function getHoverStylesMap() {
if (parts.length > 1) {
const fullSelector = selector;
styles["__nested__"] = styles["__nested__"] || [];
+ const nestedStyles = {};
+ for (const prop of rule.style) {
+ nestedStyles[prop] = rule.style[prop];
+ }
styles["__nested__"].push({
selector: fullSelector,
- styles: Object.fromEntries(
- [...rule.style].map((prop) => [prop, rule.style[prop]]),
- ),
+ styles: nestedStyles,
});
}
- // only need the style which includes the cursor attribute.
- if (!("cursor" in styles)) {
- continue;
- }
hoverMap.set(baseSelector, styles);
}
}
@@ -2508,15 +2601,35 @@ async function getHoverStylesMap() {
url.searchParams.set("v", Date.now());
newLink.href = url.toString();
newLink.crossOrigin = "anonymous";
- // until the new link loaded, removing the old one
document.head.append(newLink);
- // wait for a while until the sheet is fully loaded
- await asyncSleepFor(1500);
- const newSheets = [...document.styleSheets];
- const refreshedSheet = newSheets.find(
- (s) => s.href === newLink.href,
- );
+ // Wait for the stylesheet to load via event instead of fixed 1500ms sleep
+ await new Promise((resolve) => {
+ const timeout = setTimeout(() => {
+ _jsConsoleWarn(
+ "Stylesheet load timed out after 3s:",
+ newLink.href,
+ );
+ resolve();
+ }, 3000);
+ newLink.addEventListener("load", () => {
+ clearTimeout(timeout);
+ resolve();
+ });
+ newLink.addEventListener("error", () => {
+ clearTimeout(timeout);
+ resolve();
+ });
+ });
+
+ const allSheets = document.styleSheets;
+ let refreshedSheet = null;
+ for (let si = 0; si < allSheets.length; si++) {
+ if (allSheets[si].href === newLink.href) {
+ refreshedSheet = allSheets[si];
+ break;
+ }
+ }
if (!refreshedSheet) {
newLink.remove();
return;
@@ -2578,19 +2691,18 @@ if (window.globalDomDepthMap === undefined) {
function isClassNameIncludesHidden(className) {
// some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`
+ const lower = className.toLowerCase();
return (
- className.toLowerCase().includes("hide") ||
- className.toLowerCase().includes("invisible") ||
- className.toLowerCase().includes("closed")
+ lower.includes("hide") ||
+ lower.includes("invisible") ||
+ lower.includes("closed")
);
}
function isClassNameIncludesActivatedStatus(className) {
// some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click
- return (
- className.toLowerCase().includes("open") ||
- className.toLowerCase().includes("active")
- );
+ const lower = className.toLowerCase();
+ return lower.includes("open") || lower.includes("active");
}
function waitForNextFrame() {
@@ -2926,21 +3038,25 @@ function isAnimationFinished() {
* This includes elements in the main document and shadow DOM.
*/
function removeAllUniqueIds() {
- // Function to recursively remove unique_id from an element and its children
+ // Function to recursively remove unique_id from an element and its children.
+ // We iterate element.children directly (live HTMLCollection) since
+ // removeAttribute only modifies attributes, not child structure.
const removeUniqueIdFromElement = (element) => {
if (element.hasAttribute("unique_id")) {
element.removeAttribute("unique_id");
}
// Process children in the main DOM
- for (const child of Array.from(element.children)) {
- removeUniqueIdFromElement(child);
+ const children = element.children;
+ for (let i = 0; i < children.length; i++) {
+ removeUniqueIdFromElement(children[i]);
}
// Process elements in shadow DOM if present
if (element.shadowRoot) {
- for (const shadowChild of Array.from(element.shadowRoot.children)) {
- removeUniqueIdFromElement(shadowChild);
+ const shadowChildren = element.shadowRoot.children;
+ for (let i = 0; i < shadowChildren.length; i++) {
+ removeUniqueIdFromElement(shadowChildren[i]);
}
}
};
diff --git a/skyvern/webeye/scraper/domUtils_legacy.js b/skyvern/webeye/scraper/domUtils_legacy.js
new file mode 100644
index 000000000..7cd6804c1
--- /dev/null
+++ b/skyvern/webeye/scraper/domUtils_legacy.js
@@ -0,0 +1,2972 @@
+// LEGACY FROZEN BASELINE — do not modify. See domUtils.js for the optimized version.
+// we only use chromium browser for now
+let browserNameForWorkarounds = "chromium";
+
+let _jsConsoleLog = console?.log ?? function () {}; // prevent no console.log error
+let _jsConsoleError = console?.error ?? _jsConsoleLog;
+let _jsConsoleWarn = console?.warn ?? _jsConsoleLog;
+
+class SafeCounter {
+ constructor() {
+ this.value = 0;
+ this.lock = Promise.resolve();
+ }
+
+ async add() {
+ await this.lock;
+ this.lock = new Promise((resolve) => {
+ this.value += 1;
+ resolve();
+ });
+ return this.value;
+ }
+
+ async get() {
+ await this.lock;
+ return this.value;
+ }
+}
+
+// Commands for manipulating rects.
+// Want to debug this? Run chromium, go to sources, and create a new snippet with the code in domUtils.js
+class Rect {
+ // Create a rect given the top left and bottom right corners.
+ static create(x1, y1, x2, y2) {
+ return {
+ bottom: y2,
+ top: y1,
+ left: x1,
+ right: x2,
+ width: x2 - x1,
+ height: y2 - y1,
+ };
+ }
+
+ static copy(rect) {
+ return {
+ bottom: rect.bottom,
+ top: rect.top,
+ left: rect.left,
+ right: rect.right,
+ width: rect.width,
+ height: rect.height,
+ };
+ }
+
+ // Translate a rect by x horizontally and y vertically.
+ static translate(rect, x, y) {
+ if (x == null) x = 0;
+ if (y == null) y = 0;
+ return {
+ bottom: rect.bottom + y,
+ top: rect.top + y,
+ left: rect.left + x,
+ right: rect.right + x,
+ width: rect.width,
+ height: rect.height,
+ };
+ }
+
+ // Determine whether two rects overlap.
+ static intersects(rect1, rect2) {
+ return (
+ rect1.right > rect2.left &&
+ rect1.left < rect2.right &&
+ rect1.bottom > rect2.top &&
+ rect1.top < rect2.bottom
+ );
+ }
+
+ static equals(rect1, rect2) {
+ for (const property of [
+ "top",
+ "bottom",
+ "left",
+ "right",
+ "width",
+ "height",
+ ]) {
+ if (rect1[property] !== rect2[property]) return false;
+ }
+ return true;
+ }
+}
+
+class DomUtils {
+ static elementListCache = [];
+ static visibleClientRectCache = new WeakMap();
+ //
+ // Bounds the rect by the current viewport dimensions. If the rect is offscreen or has a height or
+ // width < 3 then null is returned instead of a rect.
+ //
+ static cropRectToVisible(rect) {
+ const boundedRect = Rect.create(
+ Math.max(rect.left, 0),
+ Math.max(rect.top, 0),
+ rect.right,
+ rect.bottom,
+ );
+ if (
+ boundedRect.top >= window.innerHeight - 4 ||
+ boundedRect.left >= window.innerWidth - 4
+ ) {
+ return null;
+ } else {
+ return boundedRect;
+ }
+ }
+
+ // add cache to optimize performance
+ static getVisibleClientRect(element, testChildren = false) {
+ // check cache
+ const cacheKey = `${testChildren}`;
+ if (DomUtils.visibleClientRectCache.has(element)) {
+ const elementCache = DomUtils.visibleClientRectCache.get(element);
+ if (elementCache.has(cacheKey)) {
+ _jsConsoleLog("hit cache to get the rect of element");
+ return elementCache.get(cacheKey);
+ }
+ }
+
+ // Note: this call will be expensive if we modify the DOM in between calls.
+ let clientRect;
+ const clientRects = (() => {
+ const result = [];
+ for (clientRect of element.getClientRects()) {
+ result.push(Rect.copy(clientRect));
+ }
+ return result;
+ })();
+
+ // Inline elements with font-size: 0px; will declare a height of zero, even if a child with
+ // non-zero font-size contains text.
+ let isInlineZeroHeight = function () {
+ const elementComputedStyle = getElementComputedStyle(element, null);
+ const isInlineZeroFontSize =
+ 0 ===
+ elementComputedStyle?.getPropertyValue("display").indexOf("inline") &&
+ elementComputedStyle?.getPropertyValue("font-size") === "0px";
+ // Override the function to return this value for the rest of this context.
+ isInlineZeroHeight = () => isInlineZeroFontSize;
+ return isInlineZeroFontSize;
+ };
+
+ let result = null;
+
+ for (clientRect of clientRects) {
+ // If the link has zero dimensions, it may be wrapping visible but floated elements. Check for
+ // this.
+ let computedStyle;
+ if ((clientRect.width === 0 || clientRect.height === 0) && testChildren) {
+ for (const child of Array.from(element.children)) {
+ computedStyle = getElementComputedStyle(child, null);
+ if (!computedStyle) {
+ continue;
+ }
+ // Ignore child elements which are not floated and not absolutely positioned for parent
+ // elements with zero width/height, as long as the case described at isInlineZeroHeight
+ // does not apply.
+ // NOTE(mrmr1993): This ignores floated/absolutely positioned descendants nested within
+ // inline children.
+ const position = computedStyle.getPropertyValue("position");
+ if (
+ computedStyle.getPropertyValue("float") === "none" &&
+ !["absolute", "fixed"].includes(position) &&
+ !(
+ clientRect.height === 0 &&
+ isInlineZeroHeight() &&
+ 0 === computedStyle.getPropertyValue("display").indexOf("inline")
+ )
+ ) {
+ continue;
+ }
+ const childClientRect = this.getVisibleClientRect(child, true);
+ if (
+ childClientRect === null ||
+ childClientRect.width < 3 ||
+ childClientRect.height < 3
+ )
+ continue;
+ result = childClientRect;
+ break;
+ }
+ if (result) break;
+ } else {
+ clientRect = this.cropRectToVisible(clientRect);
+
+ if (
+ clientRect === null ||
+ clientRect.width < 3 ||
+ clientRect.height < 3
+ )
+ continue;
+
+ // eliminate invisible elements (see test_harnesses/visibility_test.html)
+ computedStyle = getElementComputedStyle(element, null);
+ if (!computedStyle) {
+ continue;
+ }
+ if (computedStyle.getPropertyValue("visibility") !== "visible")
+ continue;
+
+ result = clientRect;
+ break;
+ }
+ }
+
+ // cache result
+ if (!DomUtils.visibleClientRectCache.has(element)) {
+ DomUtils.visibleClientRectCache.set(element, new Map());
+ }
+ DomUtils.visibleClientRectCache.get(element).set(cacheKey, result);
+
+ return result;
+ }
+
+ // clear cache
+ static clearVisibleClientRectCache() {
+ DomUtils.visibleClientRectCache = new WeakMap();
+ }
+
+ static getViewportTopLeft() {
+ const box = document.documentElement;
+ const style = getComputedStyle(box);
+ const rect = box.getBoundingClientRect();
+ if (
+ style &&
+ style.position === "static" &&
+ !/content|paint|strict/.test(style.contain || "")
+ ) {
+ // The margin is included in the client rect, so we need to subtract it back out.
+ const marginTop = parseInt(style.marginTop);
+ const marginLeft = parseInt(style.marginLeft);
+ return {
+ top: -rect.top + marginTop,
+ left: -rect.left + marginLeft,
+ };
+ } else {
+ const { clientTop, clientLeft } = box;
+ return {
+ top: -rect.top - clientTop,
+ left: -rect.left - clientLeft,
+ };
+ }
+ }
+}
+
+class QuadTreeNode {
+ constructor(bounds, maxElements = 10, maxDepth = 4) {
+ this.bounds = bounds; // {x, y, width, height}
+ this.maxElements = maxElements;
+ this.maxDepth = maxDepth;
+ this.elements = [];
+ this.children = null;
+ this.depth = 0;
+ }
+
+ insert(element) {
+ if (!this.contains(element.rect)) {
+ return false;
+ }
+
+ if (this.children === null && this.elements.length < this.maxElements) {
+ this.elements.push(element);
+ return true;
+ }
+
+ if (this.children === null) {
+ this.subdivide();
+ }
+
+ for (const child of this.children) {
+ if (child.insert(element)) {
+ return true;
+ }
+ }
+
+ this.elements.push(element);
+ return true;
+ }
+
+ subdivide() {
+ const x = this.bounds.x;
+ const y = this.bounds.y;
+ const w = this.bounds.width / 2;
+ const h = this.bounds.height / 2;
+
+ this.children = [
+ new QuadTreeNode(
+ { x, y, width: w, height: h },
+ this.maxElements,
+ this.maxDepth,
+ ),
+ new QuadTreeNode(
+ { x: x + w, y, width: w, height: h },
+ this.maxElements,
+ this.maxDepth,
+ ),
+ new QuadTreeNode(
+ { x, y: y + h, width: w, height: h },
+ this.maxElements,
+ this.maxDepth,
+ ),
+ new QuadTreeNode(
+ { x: x + w, y: y + h, width: w, height: h },
+ this.maxElements,
+ this.maxDepth,
+ ),
+ ];
+
+ for (const child of this.children) {
+ child.depth = this.depth + 1;
+ }
+ }
+
+ contains(rect) {
+ return (
+ rect.left >= this.bounds.x &&
+ rect.right <= this.bounds.x + this.bounds.width &&
+ rect.top >= this.bounds.y &&
+ rect.bottom <= this.bounds.y + this.bounds.height
+ );
+ }
+
+ query(rect) {
+ const result = [];
+ this.queryRecursive(rect, result);
+ return result;
+ }
+
+ queryRecursive(rect, result) {
+ if (!this.intersects(rect)) {
+ return;
+ }
+
+ result.push(...this.elements);
+
+ if (this.children) {
+ for (const child of this.children) {
+ child.queryRecursive(rect, result);
+ }
+ }
+ }
+
+ intersects(rect) {
+ return (
+ rect.left < this.bounds.x + this.bounds.width &&
+ rect.right > this.bounds.x &&
+ rect.top < this.bounds.y + this.bounds.height &&
+ rect.bottom > this.bounds.y
+ );
+ }
+}
+
+// from playwright
+function getElementComputedStyle(element, pseudo) {
+ return element.ownerDocument && element.ownerDocument.defaultView
+ ? element.ownerDocument.defaultView.getComputedStyle(element, pseudo)
+ : undefined;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L76-L98
+function isElementStyleVisibilityVisible(element, style) {
+ style = style ?? getElementComputedStyle(element);
+ if (!style) return true;
+ // Element.checkVisibility checks for content-visibility and also looks at
+ // styles up the flat tree including user-agent ShadowRoots, such as the
+ // details element for example.
+ // All the browser implement it, but WebKit has a bug which prevents us from using it:
+ // https://bugs.webkit.org/show_bug.cgi?id=264733
+ // @ts-ignore
+ if (
+ Element.prototype.checkVisibility &&
+ browserNameForWorkarounds !== "webkit"
+ ) {
+ if (!element.checkVisibility()) return false;
+ } else {
+ // Manual workaround for WebKit that does not have checkVisibility.
+ const detailsOrSummary = element.closest("details,summary");
+ if (
+ detailsOrSummary !== element &&
+ detailsOrSummary?.nodeName === "DETAILS" &&
+ !detailsOrSummary.open
+ )
+ return false;
+ }
+ if (style.visibility !== "visible") return false;
+
+ // TODO: support style.clipPath and style.clipRule?
+ // if element is clipped with rect(0px, 0px, 0px, 0px), it means it's invisible on the page
+ // FIXME: need a better algorithm to calculate the visible rect area, using (right-left)*(bottom-top) from rect(top, right, bottom, left)
+ if (
+ style.clip === "rect(0px, 0px, 0px, 0px)" ||
+ style.clip === "rect(1px, 1px, 1px, 1px)"
+ ) {
+ return false;
+ }
+
+ return true;
+}
+
+function hasASPClientControl() {
+ return typeof ASPxClientControl !== "undefined";
+}
+
+// Check if element is only visible on hover (e.g., hover-only buttons)
+function isHoverOnlyElement(element) {
+ // Check for common hover-only patterns in class names
+ const className = element.className?.toString() ?? "";
+ const parentClassName = element.parentElement?.className?.toString() ?? "";
+
+ // Common hover-only class patterns
+ if (
+ className.includes("hover-") ||
+ className.includes("-hover") ||
+ parentClassName.includes("hover-") ||
+ parentClassName.includes("-hover")
+ ) {
+ return true;
+ }
+
+ // Check if parent has hover-related attributes or classes that might reveal this element
+ let parent = element.parentElement;
+ let depth = 0;
+ // Cap recursion to avoid walking the entire tree and bloating prompts
+ const maxDepth = 5;
+ while (parent && parent !== document.body && depth < maxDepth) {
+ const parentClass = parent.className?.toString() ?? "";
+ if (
+ parentClass.includes("hover") ||
+ parentClass.includes("card") ||
+ parentClass.includes("item")
+ ) {
+ // This element might be revealed on parent hover
+ return true;
+ }
+ parent = parent.parentElement;
+ depth += 1;
+ }
+
+ return false;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119
+// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.
+function isElementVisible(element) {
+ // TODO: This is a hack to not check visibility for option elements
+ // because they are not visible by default. We check their parent instead for visibility.
+ if (
+ element.tagName.toLowerCase() === "option" ||
+ (element.tagName.toLowerCase() === "input" &&
+ (element.type === "radio" || element.type === "checkbox"))
+ )
+ return element.parentElement && isElementVisible(element.parentElement);
+
+ const className = element.className ? element.className.toString() : "";
+ if (
+ className.includes("select2-offscreen") ||
+ className.includes("select2-hidden") ||
+ className.includes("ui-select-offscreen")
+ ) {
+ return false;
+ }
+
+ const style = getElementComputedStyle(element);
+ if (!style) return true;
+ if (style.display === "contents") {
+ // display:contents is not rendered itself, but its child nodes are.
+ for (let child = element.firstChild; child; child = child.nextSibling) {
+ if (
+ child.nodeType === 1 /* Node.ELEMENT_NODE */ &&
+ isElementVisible(child)
+ )
+ return true;
+ if (child.nodeType === 3 /* Node.TEXT_NODE */ && isVisibleTextNode(child))
+ return true;
+ }
+ return false;
+ }
+ if (!isElementStyleVisibilityVisible(element, style)) return false;
+ const rect = element.getBoundingClientRect();
+ if (rect.width <= 0 || rect.height <= 0) {
+ // Check if this element might be visible on hover before marking as invisible
+ if (isHoverOnlyElement(element)) {
+ return true;
+ }
+ return false;
+ }
+
+ // if the center point of the element is not in the page, we tag it as an non-interactable element
+ // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X
+ const center_x = (rect.left + rect.width) / 2 + window.scrollX;
+ if (center_x < 0) {
+ return false;
+ }
+ // const center_y = (rect.top + rect.height) / 2 + window.scrollY;
+ // if (center_x < 0 || center_y < 0) {
+ // return false;
+ // }
+
+ return true;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L121-L127
+function isVisibleTextNode(node) {
+ // https://stackoverflow.com/questions/1461059/is-there-an-equivalent-to-getboundingclientrect-for-text-nodes
+ const range = node.ownerDocument.createRange();
+ range.selectNode(node);
+ const rect = range.getBoundingClientRect();
+ if (rect.width <= 0 || rect.height <= 0) {
+ return false;
+ }
+
+ // if the center point of the element is not in the page, we tag it as an non-interactable element
+ // FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X
+ const center_x = (rect.left + rect.width) / 2 + window.scrollX;
+ if (center_x < 0) {
+ return false;
+ }
+ // const center_y = (rect.top + rect.height) / 2 + window.scrollY;
+ // if (center_x < 0 || center_y < 0) {
+ // return false;
+ // }
+ return true;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L37-L44
+function parentElementOrShadowHost(element) {
+ if (element.parentElement) return element.parentElement;
+ if (!element.parentNode) return;
+ if (
+ element.parentNode.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ &&
+ element.parentNode.host
+ )
+ return element.parentNode.host;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/domUtils.ts#L46-L52
+function enclosingShadowRootOrDocument(element) {
+ let node = element;
+ while (node.parentNode) node = node.parentNode;
+ if (
+ node.nodeType === 11 /* Node.DOCUMENT_FRAGMENT_NODE */ ||
+ node.nodeType === 9 /* Node.DOCUMENT_NODE */
+ )
+ return node;
+}
+
+// from playwright: https://github.com/microsoft/playwright/blob/d685763c491e06be38d05675ef529f5c230388bb/packages/playwright-core/src/server/injected/injectedScript.ts#L799-L859
+function expectHitTarget(hitPoint, targetElement) {
+ const roots = [];
+
+ // Get all component roots leading to the target element.
+ // Go from the bottom to the top to make it work with closed shadow roots.
+ let parentElement = targetElement;
+ while (parentElement) {
+ const root = enclosingShadowRootOrDocument(parentElement);
+ if (!root) break;
+ roots.push(root);
+ if (root.nodeType === 9 /* Node.DOCUMENT_NODE */) break;
+ parentElement = root.host;
+ }
+
+ // Hit target in each component root should point to the next component root.
+ // Hit target in the last component root should point to the target or its descendant.
+ let hitElement;
+ for (let index = roots.length - 1; index >= 0; index--) {
+ const root = roots[index];
+ // All browsers have different behavior around elementFromPoint and elementsFromPoint.
+ // https://github.com/w3c/csswg-drafts/issues/556
+ // http://crbug.com/1188919
+ const elements = root.elementsFromPoint(hitPoint.x, hitPoint.y);
+ const singleElement = root.elementFromPoint(hitPoint.x, hitPoint.y);
+ if (
+ singleElement &&
+ elements[0] &&
+ parentElementOrShadowHost(singleElement) === elements[0]
+ ) {
+ const style = getElementComputedStyle(singleElement);
+ if (style?.display === "contents") {
+ // Workaround a case where elementsFromPoint misses the inner-most element with display:contents.
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=1342092
+ elements.unshift(singleElement);
+ }
+ }
+ if (
+ elements[0] &&
+ elements[0].shadowRoot === root &&
+ elements[1] === singleElement
+ ) {
+ // Workaround webkit but where first two elements are swapped:
+ // <host>
+ // #shadow root
+ // <target>
+ // elementsFromPoint produces [<host>, <target>], while it should be [<target>, <host>]
+ // In this case, just ignore <host>.
+ elements.shift();
+ }
+ const innerElement = elements[0];
+ if (!innerElement) break;
+ hitElement = innerElement;
+ if (index && innerElement !== roots[index - 1].host) break;
+ }
+
+ // Check whether hit target is the target or its descendant.
+ const hitParents = [];
+ while (hitElement && hitElement !== targetElement) {
+ hitParents.push(hitElement);
+ hitElement = parentElementOrShadowHost(hitElement);
+ }
+ if (hitElement === targetElement) return null;
+
+ return hitParents[0] || document.documentElement;
+}
+
+function getChildElements(element) {
+ if (element.childElementCount !== 0) {
+ return Array.from(element.children);
+ } else {
+ return [];
+ }
+}
+
+function isParent(parent, child) {
+ return parent.contains(child);
+}
+
+function isSibling(el1, el2) {
+ return el1.parentElement === el2.parentElement;
+}
+
+function getBlockElementUniqueID(element) {
+ const rect = element.getBoundingClientRect();
+
+ const hitElement = expectHitTarget(
+ {
+ x: rect.left + rect.width / 2,
+ y: rect.top + rect.height / 2,
+ },
+ element,
+ );
+
+ if (!hitElement) {
+ return ["", false];
+ }
+
+ return [hitElement.getAttribute("unique_id") ?? "", true];
+}
+
+function isHidden(element) {
+ const style = getElementComputedStyle(element);
+ if (style?.display === "none") {
+ return true;
+ }
+ if (element.hidden) {
+ if (
+ style?.cursor === "pointer" &&
+ element.tagName.toLowerCase() === "input" &&
+ (element.type === "submit" || element.type === "button")
+ ) {
+ // there are cases where the input is a "submit" button and the cursor is a pointer but the element has the hidden attr.
+ // such an element is not really hidden
+ return false;
+ }
+ return true;
+ }
+ return false;
+}
+
+function isHiddenOrDisabled(element) {
+ return isHidden(element) || element.disabled;
+}
+
+function isScriptOrStyle(element) {
+ const tagName = element.tagName.toLowerCase();
+ return tagName === "script" || tagName === "style";
+}
+
+function isReadonlyElement(element) {
+ if (element.readOnly) {
+ return true;
+ }
+
+ if (element.hasAttribute("readonly")) {
+ return true;
+ }
+
+ if (element.hasAttribute("aria-readonly")) {
+ // only aria-readonly="false" should be considered as "not readonly"
+ return (
+ element.getAttribute("aria-readonly").toLowerCase().trim() !== "false"
+ );
+ }
+
+ return false;
+}
+
+function isDropdownRelatedElement(element) {
+ const tagName = element.tagName?.toLowerCase();
+ if (tagName === "select") {
+ return true;
+ }
+
+ const role = element.getAttribute("role")?.toLowerCase();
+ if (role === "option" || role === "listbox") {
+ return true;
+ }
+
+ return false;
+}
+
+function hasAngularClickBinding(element) {
+ return (
+ element.hasAttribute("ng-click") || element.hasAttribute("data-ng-click")
+ );
+}
+
+function hasWidgetRole(element) {
+ const role = element.getAttribute("role");
+ if (!role) {
+ return false;
+ }
+ // https://developer.mozilla.org/en-US/docs/Web/Accessibility/ARIA/Roles#2._widget_roles
+ // Not all roles make sense for the time being so we only check for the ones that do
+ if (role.toLowerCase().trim() === "textbox") {
+ return !isReadonlyElement(element);
+ }
+
+ const widgetRoles = [
+ "button",
+ "link",
+ "checkbox",
+ "menuitem",
+ "menuitemcheckbox",
+ "menuitemradio",
+ "radio",
+ "tab",
+ "combobox",
+ "searchbox",
+ "slider",
+ "spinbutton",
+ "switch",
+ "gridcell",
+ "option",
+ ];
+ return widgetRoles.includes(role.toLowerCase().trim());
+}
+
+function isTableRelatedElement(element) {
+ const tagName = element.tagName.toLowerCase();
+ return [
+ "table",
+ "caption",
+ "thead",
+ "tbody",
+ "tfoot",
+ "tr",
+ "th",
+ "td",
+ "colgroup",
+ "col",
+ ].includes(tagName);
+}
+
+function isDOMNodeRepresentDiv(element) {
+ if (element?.tagName?.toLowerCase() !== "div") {
+ return false;
+ }
+ const style = getElementComputedStyle(element);
+ const children = getChildElements(element);
+ // flex usually means there are multiple elements in the div as a line or a column
+ // if the children elements are not just one, we should keep it in the HTML tree to represent a tree structure
+ if (style?.display === "flex" && children.length > 1) {
+ return true;
+ }
+ return false;
+}
+
+function isHoverPointerElement(element, hoverStylesMap) {
+ const tagName = element.tagName.toLowerCase();
+ const elementClassName = element.className.toString();
+ const elementCursor = getElementComputedStyle(element)?.cursor;
+ if (elementCursor === "pointer") {
+ return true;
+ }
+
+ // Check if element has hover styles that change cursor to pointer
+ // This is to handle the case where an element's cursor is "auto", but resolves to "pointer" on hover
+ if (elementCursor === "auto" || elementCursor === "default") {
+ // TODO: we need a better algorithm to match the selector with better performance
+ for (const [selector, styles] of hoverStylesMap) {
+ let shouldMatch = false;
+ for (const className of element.classList) {
+ if (selector.includes(className)) {
+ shouldMatch = true;
+ break;
+ }
+ }
+ if (shouldMatch || selector.includes(tagName)) {
+ if (element.matches(selector) && styles.cursor === "pointer") {
+ return true;
+ }
+ }
+ }
+ }
+
+ // FIXME: hardcode to fix the bug about hover style now
+ if (elementClassName.includes("hover:cursor-pointer")) {
+ return true;
+ }
+
+ return false;
+}
+
+function isInteractableInput(element, hoverStylesMap) {
+ const tagName = element.tagName.toLowerCase();
+ if (tagName !== "input") {
+ // let other checks decide
+ return false;
+ }
+ // Browsers default to "text" when the type is not set or is invalid
+ // Here's the list of valid types: https://developer.mozilla.org/en-US/docs/Web/HTML/Element/input#input_types
+ // Examples of unrecognized types that we've seen and caused issues because we didn't mark them interactable:
+ // "city", "state", "zip", "country"
+ // That's the reason I (Kerem) removed the valid input types check
+ var type = element.getAttribute("type")?.toLowerCase().trim() ?? "text";
+ return isHoverPointerElement(element, hoverStylesMap) || type !== "hidden";
+}
+
+function isValidCSSSelector(selector) {
+ try {
+ document.querySelector(selector);
+ return true;
+ } catch (e) {
+ return false;
+ }
+}
+
+function isInteractable(element, hoverStylesMap) {
+ if (!isElementVisible(element)) {
+ return false;
+ }
+
+ if (isHidden(element)) {
+ return false;
+ }
+
+ if (isScriptOrStyle(element)) {
+ return false;
+ }
+
+ if (hasWidgetRole(element)) {
+ return true;
+ }
+
+ // element with pointer-events: none should not be considered as interactable
+ // but for elements which are disabled, we should not use this logic to test the interactable
+ // https://developer.mozilla.org/en-US/docs/Web/CSS/pointer-events#none
+ const elementPointerEvent = getElementComputedStyle(element)?.pointerEvents;
+ if (elementPointerEvent === "none" && !element.disabled) {
+ // Some CTAs stay hidden until the parent is hovered
+ // When we can infer that the element is revealed on hover, keep it interactable so the agent
+ // has a chance to hover the parent before clicking.
+ if (!isHoverOnlyElement(element)) {
+ return false;
+ }
+ }
+
+ if (isInteractableInput(element, hoverStylesMap)) {
+ return true;
+ }
+
+ const tagName = element.tagName.toLowerCase();
+ if (tagName === "html") {
+ return false;
+ }
+
+ if (tagName === "iframe") {
+ return false;
+ }
+
+ if (tagName === "frameset") {
+ return false;
+ }
+
+ if (tagName === "frame") {
+ return false;
+ }
+
+ if (tagName === "a" && element.href) {
+ return true;
+ }
+
+ // Check if the option's parent (select) is hidden or disabled
+ if (tagName === "option" && isHiddenOrDisabled(element.parentElement)) {
+ return false;
+ }
+
+ if (
+ tagName === "button" ||
+ tagName === "select" ||
+ tagName === "option" ||
+ tagName === "textarea"
+ ) {
+ return true;
+ }
+
+ if (tagName === "label" && element.control && !element.control.disabled) {
+ return true;
+ }
+
+ if (
+ element.hasAttribute("onclick") ||
+ element.isContentEditable ||
+ element.hasAttribute("jsaction")
+ ) {
+ return true;
+ }
+
+ const className = element.className?.toString() ?? "";
+
+ if (tagName === "div" || tagName === "span") {
+ if (hasAngularClickBinding(element)) {
+ return true;
+ }
+ if (className.includes("blinking-cursor")) {
+ return true;
+ }
+ // https://www.oxygenxml.com/dita/1.3/specs/langRef/technicalContent/svg-container.html
+ // svg-container is usually used for clickable elements that wrap SVGs
+ if (className.includes("svg-container")) {
+ return true;
+ }
+ }
+
+ // support listbox and options underneath it
+ // div element should be checked here before the css pointer
+ if (
+ (tagName === "ul" || tagName === "div") &&
+ element.hasAttribute("role") &&
+ element.getAttribute("role").toLowerCase() === "listbox"
+ ) {
+ return true;
+ }
+ if (
+ (tagName === "li" || tagName === "div") &&
+ element.hasAttribute("role") &&
+ element.getAttribute("role").toLowerCase() === "option"
+ ) {
+ return true;
+ }
+
+ if (
+ tagName === "li" &&
+ (className.includes("ui-menu-item") ||
+ className.includes("dropdown-item") ||
+ className === "option" ||
+ className.includes("rddlItem"))
+ ) {
+ return true;
+ }
+
+ // google map address auto complete
+ // https://developers.google.com/maps/documentation/javascript/place-autocomplete#style-autocomplete
+ // demo: https://developers.google.com/maps/documentation/javascript/examples/places-autocomplete-addressform
+ if (
+ tagName === "div" &&
+ className.includes("pac-item") &&
+ element.closest('div[class*="pac-container"]')
+ ) {
+ return true;
+ }
+
+ if (
+ tagName === "div" &&
+ element.hasAttribute("aria-disabled") &&
+ element.getAttribute("aria-disabled").toLowerCase() === "false"
+ ) {
+ return true;
+ }
+
+ if (tagName === "span" && element.closest('div[id*="dropdown-container"]')) {
+ return true;
+ }
+
+ // FIXME: maybe we need to enable the pointer check for all elements?
+ if (
+ tagName === "div" ||
+ tagName === "img" ||
+ tagName === "span" ||
+ tagName === "a" ||
+ tagName === "i" ||
+ tagName === "li" ||
+ tagName === "p" ||
+ tagName === "td" ||
+ tagName === "svg" ||
+ tagName === "strong" ||
+ tagName === "h1" ||
+ tagName === "h2" ||
+ tagName === "h3" ||
+ tagName === "h4" ||
+ // sometime it's a customized element like <my-login-button>, we should check pointer style
+ tagName.includes("button") ||
+ tagName.includes("select") ||
+ tagName.includes("option") ||
+ tagName.includes("textarea")
+ ) {
+ if (isHoverPointerElement(element, hoverStylesMap)) {
+ return true;
+ }
+ }
+
+ if (hasASPClientControl() && tagName === "tr") {
+ return true;
+ }
+
+ if (tagName === "div" && element.hasAttribute("data-selectable")) {
+ return true;
+ }
+
+ try {
+ if (window.jQuery && window.jQuery._data) {
+ const events = window.jQuery._data(element, "events");
+ if (events && "click" in events) {
+ return true;
+ }
+ }
+ } catch (e) {
+ _jsConsoleError("Error getting jQuery click events:", e);
+ }
+
+ try {
+ if (hasAngularClickEvent(element)) {
+ return true;
+ }
+ } catch (e) {
+ _jsConsoleError("Error checking angular click event:", e);
+ }
+
+ // consider <div tabindex="0"> as interactable
+ if (
+ tagName.toLowerCase() === "div" &&
+ element.hasAttribute("tabindex") &&
+ element.getAttribute("tabindex").toLowerCase() === "0"
+ ) {
+ return true;
+ }
+
+ return false;
+}
+
+function isScrollable(element) {
+ const scrollHeight = element.scrollHeight || 0;
+ const clientHeight = element.clientHeight || 0;
+ const scrollWidth = element.scrollWidth || 0;
+ const clientWidth = element.clientWidth || 0;
+
+ const hasScrollableContent =
+ scrollHeight > clientHeight || scrollWidth > clientWidth;
+ const hasScrollableOverflow = isScrollableOverflow(element);
+ return hasScrollableContent && hasScrollableOverflow;
+}
+
+function isScrollableOverflow(element) {
+ const style = getElementComputedStyle(element);
+ if (!style) {
+ return false;
+ }
+ return (
+ style.overflow === "auto" ||
+ style.overflow === "scroll" ||
+ style.overflowX === "auto" ||
+ style.overflowX === "scroll" ||
+ style.overflowY === "auto" ||
+ style.overflowY === "scroll"
+ );
+}
+
+function isDatePickerSelector(element) {
+ const tagName = element.tagName.toLowerCase();
+ if (
+ tagName === "button" &&
+ element.getAttribute("data-testid")?.includes("date")
+ ) {
+ return true;
+ }
+ return false;
+}
+
+const isComboboxDropdown = (element) => {
+ if (element.tagName.toLowerCase() !== "input") {
+ return false;
+ }
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
+ const readonly =
+ element.getAttribute("readonly") &&
+ element.getAttribute("readonly").toLowerCase() !== "false";
+ const controls = element.hasAttribute("aria-controls");
+ return role && haspopup && controls && readonly;
+};
+
+const isDivComboboxDropdown = (element) => {
+ const tagName = element.tagName.toLowerCase();
+ if (tagName !== "div") {
+ return false;
+ }
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
+ const controls = element.hasAttribute("aria-controls");
+ return role === "combobox" && controls && haspopup;
+};
+
+const isDropdownButton = (element) => {
+ const tagName = element.tagName.toLowerCase();
+ const type = element.getAttribute("type")
+ ? element.getAttribute("type").toLowerCase()
+ : "";
+ const haspopup = element.getAttribute("aria-haspopup")
+ ? element.getAttribute("aria-haspopup").toLowerCase()
+ : "";
+ const hasExpanded = element.hasAttribute("aria-expanded");
+ return (
+ tagName === "button" &&
+ type === "button" &&
+ (hasExpanded || haspopup === "listbox")
+ );
+};
+
+const isSelect2Dropdown = (element) => {
+ const tagName = element.tagName.toLowerCase();
+ const className = element.className.toString();
+ const role = element.getAttribute("role")
+ ? element.getAttribute("role").toLowerCase()
+ : "";
+
+ if (tagName === "a") {
+ return className.includes("select2-choice");
+ }
+
+ if (tagName === "span") {
+ return className.includes("select2-selection") && role === "combobox";
+ }
+
+ return false;
+};
+
+const isSelect2MultiChoice = (element) => {
+ return (
+ element.tagName.toLowerCase() === "input" &&
+ element.className.toString().includes("select2-input")
+ );
+};
+
+const isReactSelectDropdown = (element) => {
+ return (
+ element.tagName.toLowerCase() === "input" &&
+ element.className.toString().includes("select__input") &&
+ element.getAttribute("role") === "combobox"
+ );
+};
+
+function isReadonlyInputDropdown(element) {
+ const className = element.className?.toString() ?? "";
+ return (
+ element.tagName.toLowerCase() === "input" &&
+ className.includes("custom-select") &&
+ isReadonlyElement(element)
+ );
+}
+
+function hasNgAttribute(element) {
+ if (!element.attributes[Symbol.iterator]) {
+ return false;
+ }
+
+ for (let attr of element.attributes) {
+ if (attr.name.startsWith("ng-")) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// TODO: it's a hack, should continue to optimize it
+function hasAngularClickEvent(element) {
+ const ctx = element.__ngContext__;
+ const tView = ctx && ctx[1];
+ if (!tView || !Array.isArray(tView.data)) {
+ return false;
+ }
+
+ const tagName = element.tagName.toLowerCase();
+ if (!tagName) {
+ _jsConsoleLog("Element has no tag name: ", element);
+ return false;
+ }
+
+ for (const tNode of tView.data) {
+ if (!tNode || typeof tNode !== "object") continue;
+ if (tNode.type !== 0 && tNode.type !== 2) continue; // 0: Element, 2: Container
+ if (tNode.value && tagName !== tNode.value.toLowerCase()) continue;
+ if (!Array.isArray(tNode.attrs)) continue;
+ if (tNode.attrs.includes("click")) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+function isAngularMaterial(element) {
+ if (!element.attributes[Symbol.iterator]) {
+ return false;
+ }
+
+ for (let attr of element.attributes) {
+ if (attr.name.startsWith("mat")) {
+ return true;
+ }
+ }
+ return false;
+}
+
+const isAngularDropdown = (element) => {
+ if (!hasNgAttribute(element)) {
+ return false;
+ }
+
+ if (element.type?.toLowerCase() === "search") {
+ return false;
+ }
+
+ const tagName = element.tagName.toLowerCase();
+ if (tagName === "input" || tagName === "span") {
+ const ariaLabel = element.hasAttribute("aria-label")
+ ? element.getAttribute("aria-label").toLowerCase()
+ : "";
+ return ariaLabel.includes("select") || ariaLabel.includes("choose");
+ }
+
+ return false;
+};
+
+const isAngularMaterialDatePicker = (element) => {
+ if (!isAngularMaterial(element)) {
+ return false;
+ }
+
+ const tagName = element.tagName.toLowerCase();
+ if (tagName !== "input") return false;
+
+ return (
+ (element.closest("mat-datepicker") ||
+ element.closest("mat-formio-date")) !== null
+ );
+};
+
+function getPseudoContent(element, pseudo) {
+ const pseudoStyle = getElementComputedStyle(element, pseudo);
+ if (!pseudoStyle) {
+ return null;
+ }
+ const content = pseudoStyle
+ .getPropertyValue("content")
+ .replace(/"/g, "")
+ .trim();
+
+ if (content === "none" || !content) {
+ return null;
+ }
+
+ return content;
+}
+
+function hasBeforeOrAfterPseudoContent(element) {
+ return (
+ getPseudoContent(element, "::before") != null ||
+ getPseudoContent(element, "::after") != null
+ );
+}
+
+const checkParentClass = (className) => {
+ const targetParentClasses = ["field", "entry"];
+ for (let i = 0; i < targetParentClasses.length; i++) {
+ if (className.includes(targetParentClasses[i])) {
+ return true;
+ }
+ }
+ return false;
+};
+
+function removeMultipleSpaces(str) {
+ // Optimization: check for empty values early
+ if (!str || str.length === 0) {
+ return str;
+ }
+
+ // Optimization: check if contains multiple spaces to avoid unnecessary regex replacement
+ if (
+ str.indexOf(" ") === -1 &&
+ str.indexOf("\t") === -1 &&
+ str.indexOf("\n") === -1
+ ) {
+ return str;
+ }
+
+ return str.replace(/\s+/g, " ");
+}
+
+function cleanupText(text) {
+ // Optimization: check for empty values early to avoid unnecessary processing
+ if (!text || text.length === 0) {
+ return "";
+ }
+
+ // Optimization: use more efficient string replacement
+ let cleanedText = text;
+
+ // Remove specific SVG error message
+ if (cleanedText.includes("SVGs not supported by this browser.")) {
+ cleanedText = cleanedText.replace(
+ "SVGs not supported by this browser.",
+ "",
+ );
+ }
+
+ // Optimization: combine space processing and trim operations
+ return removeMultipleSpaces(cleanedText).trim();
+}
+
+const checkStringIncludeRequire = (str) => {
+ return (
+ str.toLowerCase().includes("*") ||
+ str.toLowerCase().includes("✱") ||
+ str.toLowerCase().includes("require")
+ );
+};
+
+const checkRequiredFromStyle = (element) => {
+ const afterCustomStyle = getElementComputedStyle(element, "::after");
+ if (afterCustomStyle) {
+ const afterCustom = afterCustomStyle
+ .getPropertyValue("content")
+ .replace(/"/g, "");
+ if (checkStringIncludeRequire(afterCustom)) {
+ return true;
+ }
+ }
+
+ if (!element.className || typeof element.className !== "string") {
+ return false;
+ }
+
+ return element.className.toLowerCase().includes("require");
+};
+
+function checkDisabledFromStyle(element) {
+ const className = element.className.toString().toLowerCase();
+ if (className.includes("react-datepicker__day--disabled")) {
+ return true;
+ }
+ return false;
+}
+
+function getVisibleText(element) {
+ let visibleText = [];
+
+ function collectVisibleText(node) {
+ if (
+ node.nodeType === Node.TEXT_NODE &&
+ isElementVisible(node.parentElement)
+ ) {
+ const trimmedText = node.data.trim();
+ if (trimmedText.length > 0) {
+ visibleText.push(trimmedText);
+ }
+ } else if (node.nodeType === Node.ELEMENT_NODE && isElementVisible(node)) {
+ for (let child of node.childNodes) {
+ collectVisibleText(child);
+ }
+ }
+ }
+
+ collectVisibleText(element);
+ return visibleText.join(" ");
+}
+
+// only get text from element itself
+function getElementText(element) {
+ if (element.nodeType === Node.TEXT_NODE) {
+ return element.data.trim();
+ }
+
+ const childNodes = element.childNodes;
+ const childNodesLength = childNodes.length;
+
+ // If no child nodes, return empty string directly
+ if (childNodesLength === 0) {
+ return "";
+ }
+
+ const visibleText = [];
+ let hasText = false;
+
+ for (let i = 0; i < childNodesLength; i++) {
+ const node = childNodes[i];
+ if (node.nodeType === Node.TEXT_NODE) {
+ const nodeText = node.data.trim();
+ if (nodeText.length > 0) {
+ visibleText.push(nodeText);
+ hasText = true;
+ }
+ }
+ }
+
+ return hasText ? visibleText.join(";") : "";
+}
+
+function getSelectOptions(element) {
+ const options = Array.from(element.options);
+ const selectOptions = [];
+
+ for (const option of options) {
+ selectOptions.push({
+ optionIndex: option.index,
+ text: removeMultipleSpaces(option.textContent),
+ value: removeMultipleSpaces(option.value),
+ });
+ }
+
+ const selectedOption = element.querySelector("option:checked");
+ if (!selectedOption) {
+ return [selectOptions, ""];
+ }
+
+ return [selectOptions, removeMultipleSpaces(selectedOption.textContent)];
+}
+
+function getDOMElementBySkyvenElement(elementObj) {
+ // if element has shadowHost set, we need to find the shadowHost element first then find the element
+ if (elementObj.shadowHost) {
+ let shadowHostEle = document.querySelector(
+ `[unique_id="${elementObj.shadowHost}"]`,
+ );
+ if (!shadowHostEle) {
+ _jsConsoleLog(
+ "Could not find shadowHost element with unique_id: ",
+ elementObj.shadowHost,
+ );
+ return null;
+ }
+ return shadowHostEle.shadowRoot.querySelector(
+ `[unique_id="${elementObj.id}"]`,
+ );
+ }
+
+ return document.querySelector(`[unique_id="${elementObj.id}"]`);
+}
+
+if (window.elementIdCounter === undefined) {
+ window.elementIdCounter = new SafeCounter();
+}
+
+// generate a unique id for the element
+// length is 4, the first character is from the frame index, the last 3 characters are from the counter,
+async function uniqueId() {
+ const characters =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ const base = characters.length;
+
+ const extraCharacters = "~!@#$%^&*()-_+=";
+ const extraBase = extraCharacters.length;
+
+ let result = "";
+
+ if (
+ window.GlobalSkyvernFrameIndex === undefined ||
+ window.GlobalSkyvernFrameIndex < 0
+ ) {
+ const randomIndex = Math.floor(Math.random() * extraBase);
+ result += extraCharacters[randomIndex];
+ } else {
+ const c1 = window.GlobalSkyvernFrameIndex % base;
+ result += characters[c1];
+ }
+
+ const countPart =
+ (await window.elementIdCounter.add()) % (base * base * base);
+ const c2 = Math.floor(countPart / (base * base));
+ result += characters[c2];
+ const c3 = Math.floor(countPart / base) % base;
+ result += characters[c3];
+ const c4 = countPart % base;
+ result += characters[c4];
+
+ return result;
+}
+
+async function buildElementObject(
+ frame,
+ element,
+ interactable,
+ purgeable = false,
+) {
+ var element_id = element.getAttribute("unique_id") ?? (await uniqueId());
+ var elementTagNameLower = element.tagName.toLowerCase();
+ element.setAttribute("unique_id", element_id);
+
+ const attrs = {};
+ if (element.attributes[Symbol.iterator]) {
+ for (const attr of element.attributes) {
+ var attrValue = attr.value;
+ if (
+ attr.name === "required" ||
+ attr.name === "aria-required" ||
+ attr.name === "checked" ||
+ attr.name === "aria-checked" ||
+ attr.name === "selected" ||
+ attr.name === "aria-selected" ||
+ attr.name === "readonly" ||
+ attr.name === "aria-readonly" ||
+ attr.name === "disabled" ||
+ attr.name === "aria-disabled"
+ ) {
+ if (attrValue && attrValue.toLowerCase() === "false") {
+ attrValue = false;
+ } else {
+ attrValue = true;
+ }
+ }
+ attrs[attr.name] = attrValue;
+ }
+ } else {
+ _jsConsoleWarn(
+ "element.attributes is not iterable. element_id=" + element_id,
+ );
+ }
+
+ if (
+ checkDisabledFromStyle(element) &&
+ !attrs["disabled"] &&
+ !attrs["aria-disabled"]
+ ) {
+ attrs["disabled"] = true;
+ }
+
+ if (
+ checkRequiredFromStyle(element) &&
+ !attrs["required"] &&
+ !attrs["aria-required"]
+ ) {
+ attrs["required"] = true;
+ }
+
+ // check DOM property of required/checked/selected/readonly/disabled
+ // the value from the DOM property should be the top priority
+ if (element.required !== undefined) {
+ delete attrs["required"];
+ delete attrs["aria-required"];
+ if (element.required) {
+ attrs["required"] = true;
+ }
+ }
+ if (element.checked !== undefined) {
+ delete attrs["checked"];
+ delete attrs["aria-checked"];
+ if (element.checked) {
+ attrs["checked"] = true;
+ } else if (
+ elementTagNameLower === "input" &&
+ (element.type === "checkbox" || element.type === "radio")
+ ) {
+ // checked property always exists for checkbox and radio elements
+ attrs["checked"] = false;
+ }
+ }
+ if (element.selected !== undefined) {
+ delete attrs["selected"];
+ delete attrs["aria-selected"];
+ if (element.selected) {
+ attrs["selected"] = true;
+ }
+ }
+ if (element.readOnly !== undefined) {
+ delete attrs["readonly"];
+ delete attrs["aria-readonly"];
+ if (element.readOnly) {
+ attrs["readonly"] = true;
+ }
+ }
+ if (element.disabled !== undefined) {
+ delete attrs["disabled"];
+ delete attrs["aria-disabled"];
+ if (element.disabled) {
+ attrs["disabled"] = true;
+ }
+ }
+
+ if (elementTagNameLower === "input" || elementTagNameLower === "textarea") {
+ if (element.type === "password") {
+ attrs["value"] = element.value ? "*".repeat(element.value.length) : "";
+ } else {
+ attrs["value"] = element.value;
+ }
+ }
+
+ let elementObj = {
+ id: element_id,
+ frame: frame,
+ frame_index: window.GlobalSkyvernFrameIndex,
+ interactable: interactable,
+ hoverOnly: isHoverOnlyElement(element),
+ tagName: elementTagNameLower,
+ attributes: attrs,
+ beforePseudoText: getPseudoContent(element, "::before"),
+ text: getElementText(element),
+ afterPseudoText: getPseudoContent(element, "::after"),
+ children: [],
+ // if purgeable is True, which means this element is only used for building the tree relationship
+ purgeable: purgeable,
+ // don't trim any attr of this element if keepAllAttr=True
+ keepAllAttr:
+ elementTagNameLower === "svg" || element.closest("svg") !== null,
+ isSelectable:
+ elementTagNameLower === "select" ||
+ isDatePickerSelector(element) ||
+ isDivComboboxDropdown(element) ||
+ isDropdownButton(element) ||
+ isAngularDropdown(element) ||
+ isAngularMaterialDatePicker(element) ||
+ isSelect2Dropdown(element) ||
+ isSelect2MultiChoice(element) ||
+ isReadonlyInputDropdown(element),
+ };
+
+ let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;
+ if (isInShadowRoot) {
+ let shadowHostEle = element.getRootNode().host;
+ let shadowHostId = shadowHostEle.getAttribute("unique_id");
+ // assign shadowHostId to the shadowHost element if it doesn't have unique_id
+ if (!shadowHostId) {
+ shadowHostId = await uniqueId();
+ shadowHostEle.setAttribute("unique_id", shadowHostId);
+ }
+ elementObj.shadowHost = shadowHostId;
+ }
+
+ // get options for select element or for listbox element
+ let selectOptions = null;
+ let selectedValue = "";
+ if (elementTagNameLower === "select") {
+ [selectOptions, selectedValue] = getSelectOptions(element);
+ }
+
+ if (selectOptions) {
+ elementObj.options = selectOptions;
+ }
+ if (selectedValue) {
+ elementObj.attributes["selected"] = selectedValue;
+ }
+
+ return elementObj;
+}
+
+// build the element tree for the body
+async function buildTreeFromBody(
+ frame = "main.frame",
+ frame_index = undefined,
+ must_included_tags = [],
+) {
+ if (
+ window.GlobalSkyvernFrameIndex === undefined &&
+ frame_index !== undefined
+ ) {
+ window.GlobalSkyvernFrameIndex = frame_index;
+ }
+ const maxElementNumber = 15000;
+ const elementsAndResultArray = await buildElementTree(
+ document.documentElement,
+ frame,
+ false,
+ undefined,
+ maxElementNumber,
+ must_included_tags,
+ );
+ DomUtils.elementListCache = elementsAndResultArray[0];
+ return elementsAndResultArray;
+}
+
+async function buildElementTree(
+ starter = document.documentElement,
+ frame,
+ full_tree = false,
+ hoverStylesMap = undefined,
+ maxElementNumber = 0,
+ must_included_tags = [],
+) {
+ // Generate hover styles map at the start
+ if (hoverStylesMap === undefined) {
+ hoverStylesMap = await getHoverStylesMap();
+ }
+
+ if (must_included_tags.length > 0) {
+ _jsConsoleLog(
+ "full tree will be enabled as the must_included_tags is not empty",
+ );
+ full_tree = true;
+ }
+ if (window.GlobalEnableAllTextualElements === undefined) {
+ window.GlobalEnableAllTextualElements = false;
+ }
+
+ var elements = [];
+ var resultArray = [];
+
+ async function processElement(
+ element,
+ parentId,
+ parent_xpath,
+ current_node_index,
+ ) {
+ if (element === null) {
+ _jsConsoleLog("get a null element");
+ return;
+ }
+
+ if (maxElementNumber > 0 && elements.length >= maxElementNumber) {
+ _jsConsoleWarn(
+ "Max element number reached, aborting the element tree building",
+ );
+ return;
+ }
+
+ const tagName = element.tagName?.toLowerCase();
+ if (!tagName) {
+ _jsConsoleLog("get a null tagName");
+ return;
+ }
+
+ if (tagName === "head") {
+ return;
+ }
+
+ // skip processing option element as they are already added to the select.options
+ if (tagName === "option") {
+ return;
+ }
+
+ let current_xpath = null;
+ if (parent_xpath !== null) {
+ // ignore the namespace, otherwise the xpath sometimes won't find anything, specially for SVG elements
+ current_xpath =
+ parent_xpath +
+ "/" +
+ '*[name()="' +
+ tagName +
+ '"]' +
+ "[" +
+ current_node_index +
+ "]";
+ }
+
+ let shadowDOMchildren = [];
+ // sometimes the shadowRoot is not visible, but the elements in the shadowRoot are visible
+ if (element.shadowRoot) {
+ shadowDOMchildren = getChildElements(element.shadowRoot);
+ }
+ const isVisible = isElementVisible(element);
+ if (isVisible && !isHidden(element) && !isScriptOrStyle(element)) {
+ let interactable = isInteractable(element, hoverStylesMap);
+ let elementObj = null;
+ let isParentSVG = null;
+ if (must_included_tags.includes(tagName)) {
+ interactable = true;
+ }
+ if (interactable) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (
+ tagName === "frameset" ||
+ tagName === "iframe" ||
+ tagName === "frame"
+ ) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (element.shadowRoot) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (isTableRelatedElement(element)) {
+ // build all table related elements into skyvern element
+ // we need these elements to preserve the DOM structure
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (hasBeforeOrAfterPseudoContent(element)) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (tagName === "svg") {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (
+ (isParentSVG = element.closest("svg")) &&
+ isParentSVG.getAttribute("unique_id")
+ ) {
+ // if element is the children of the <svg> with an unique_id
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (tagName === "div" && isDOMNodeRepresentDiv(element)) {
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (
+ tagName === "embed" &&
+ element.getAttribute("type")?.toLowerCase() === "application/pdf"
+ ) {
+ elementObj = await buildElementObject(
+ frame,
+ element,
+ interactable,
+ true,
+ );
+ } else if (
+ getElementText(element).length > 0 &&
+ getElementText(element).length <= 5000
+ ) {
+ if (window.GlobalEnableAllTextualElements) {
+ // force all textual elements to be interactable
+ interactable = true;
+ }
+ elementObj = await buildElementObject(frame, element, interactable);
+ } else if (full_tree) {
+ // when building full tree, we only get text from element itself
+ // elements without text are purgeable
+ elementObj = await buildElementObject(
+ frame,
+ element,
+ interactable,
+ true,
+ );
+ if (elementObj.text.length > 0) {
+ elementObj.purgeable = false;
+ }
+ }
+
+ if (elementObj) {
+ elementObj.xpath = current_xpath;
+ elements.push(elementObj);
+ // If the element is interactable but has no interactable parent,
+ // then it starts a new tree, so add it to the result array
+ // and set its id as the interactable parent id for the next elements
+ // under it
+ if (parentId === null) {
+ resultArray.push(elementObj);
+ }
+ // If the element is interactable and has an interactable parent,
+ // then add it to the children of the parent
+ else {
+ // TODO: use dict/object so that we access these in O(1) instead
+ elements
+ .find((element) => element.id === parentId)
+ .children.push(elementObj);
+ }
+ parentId = elementObj.id;
+ }
+ }
+
+ const children = getChildElements(element);
+ const xpathMap = new Map();
+
+ for (let i = 0; i < children.length; i++) {
+ const childElement = children[i];
+ const tagName = childElement?.tagName?.toLowerCase();
+ if (!tagName) {
+ _jsConsoleLog("get a null tagName");
+ continue;
+ }
+ let current_node_index = xpathMap.get(tagName);
+ if (current_node_index == undefined) {
+ current_node_index = 1;
+ } else {
+ current_node_index = current_node_index + 1;
+ }
+ xpathMap.set(tagName, current_node_index);
+ await processElement(
+ childElement,
+ parentId,
+ current_xpath,
+ current_node_index,
+ );
+ }
+
+ // FIXME: xpath won't work when the element is in shadow DOM
+ for (let i = 0; i < shadowDOMchildren.length; i++) {
+ const childElement = shadowDOMchildren[i];
+ await processElement(childElement, parentId, null, 0);
+ }
+ return;
+ }
+
+ const trimDuplicatedText = (element) => {
+ if (element.children.length === 0 && !element.options) {
+ return;
+ }
+
+ // if the element has options, text will be duplicated with the option text
+ if (element.options) {
+ element.options.forEach((option) => {
+ element.text = element.text.replace(option.text, "");
+ });
+ }
+
+ // BFS to delete duplicated text
+ element.children.forEach((child) => {
+ // delete duplicated text in the tree
+ element.text = element.text.replace(child.text, "");
+ trimDuplicatedText(child);
+ });
+
+ // trim multiple ";"
+ element.text = element.text.replace(/;+/g, ";");
+ // trimleft and trimright ";"
+ element.text = element.text.replace(new RegExp(`^;+|;+$`, "g"), "");
+ };
+
+ // some elements without children nodes should be removed out, such as <label>
+ const removeOrphanNode = (results) => {
+ const trimmedResults = [];
+ for (let i = 0; i < results.length; i++) {
+ const element = results[i];
+ element.children = removeOrphanNode(element.children);
+ if (element.tagName === "label") {
+ const labelElement = document.querySelector(
+ element.tagName + '[unique_id="' + element.id + '"]',
+ );
+ if (
+ labelElement &&
+ labelElement.childElementCount === 0 &&
+ !labelElement.getAttribute("for") &&
+ !element.text
+ ) {
+ continue;
+ }
+ }
+ trimmedResults.push(element);
+ }
+ return trimmedResults;
+ };
+
+ let current_xpath = null;
+ if (starter === document.documentElement) {
+ current_xpath = "";
+ }
+
+ // setup before parsing the dom
+ await processElement(starter, null, current_xpath, 1);
+
+ for (var element of elements) {
+ if (
+ ((element.tagName === "input" && element.attributes["type"] === "text") ||
+ element.tagName === "textarea") &&
+ (element.attributes["required"] || element.attributes["aria-required"]) &&
+ element.attributes.value === ""
+ ) {
+ // TODO (kerem): we may want to pass these elements to the LLM as empty but required fields in the future
+ _jsConsoleLog(
+ "input element with required attribute and no value",
+ element,
+ );
+ }
+ }
+
+ resultArray = removeOrphanNode(resultArray);
+ resultArray.forEach((root) => {
+ trimDuplicatedText(root);
+ });
+
+ return [elements, resultArray];
+}
+
+function drawBoundingBoxes(elements) {
+ // draw a red border around the elements
+ DomUtils.clearVisibleClientRectCache();
+ elements.forEach((element) => {
+ const ele = getDOMElementBySkyvenElement(element);
+ element.rect = ele ? DomUtils.getVisibleClientRect(ele, true) : null;
+ });
+ var groups = groupElementsVisually(elements);
+ var hintMarkers = createHintMarkersForGroups(groups);
+ addHintMarkersToPage(hintMarkers);
+ DomUtils.clearVisibleClientRectCache();
+}
+
+async function buildElementsAndDrawBoundingBoxes(
+ frame = "main.frame",
+ frame_index = undefined,
+) {
+ if (DomUtils.elementListCache.length > 0) {
+ drawBoundingBoxes(DomUtils.elementListCache);
+ return;
+ }
+ _jsConsoleWarn("no element list cache, drawBoundingBoxes from scratch");
+ var elementsAndResultArray = await buildTreeFromBody(frame, frame_index);
+ drawBoundingBoxes(elementsAndResultArray[0]);
+}
+
+function captchaSolvedCallback() {
+ _jsConsoleLog("captcha solved");
+ if (!window["captchaSolvedCounter"]) {
+ window["captchaSolvedCounter"] = 0;
+ }
+ // For some reason this isn't being called.. TODO figure out why
+ window["captchaSolvedCounter"] = window["captchaSolvedCounter"] + 1;
+}
+
+function getCaptchaSolves() {
+ if (!window["captchaSolvedCounter"]) {
+ window["captchaSolvedCounter"] = 0;
+ }
+ return window["captchaSolvedCounter"];
+}
+
+function groupElementsVisually(elements) {
+ // Quadtree O(n log n)
+ const validElements = elements.filter((element) => element.rect);
+
+ if (validElements.length === 0) return [];
+
+ // Calculate bounds
+ const bounds = calculateBounds(validElements);
+
+ // Create quadtree
+ const quadTree = new QuadTreeNode(bounds);
+ validElements.forEach((element) => quadTree.insert(element));
+
+ const groups = [];
+ const processed = new Set();
+
+ for (const element of validElements) {
+ if (processed.has(element)) continue;
+
+ const group = { elements: [element], rect: null };
+ processed.add(element);
+
+ // Find all elements overlapping with current element
+ const overlapping = findOverlappingElements(
+ element,
+ validElements,
+ quadTree,
+ processed,
+ );
+
+ for (const overlappingElement of overlapping) {
+ group.elements.push(overlappingElement);
+ processed.add(overlappingElement);
+ }
+
+ group.rect = createRectangleForGroup(group);
+ groups.push(group);
+ }
+
+ return groups;
+}
+
+// Helper functions
+function calculateBounds(elements) {
+ const rects = elements.map((el) => el.rect);
+ const left = Math.min(...rects.map((r) => r.left));
+ const top = Math.min(...rects.map((r) => r.top));
+ const right = Math.max(...rects.map((r) => r.right));
+ const bottom = Math.max(...rects.map((r) => r.bottom));
+
+ return {
+ x: left,
+ y: top,
+ width: right - left,
+ height: bottom - top,
+ };
+}
+
+function findOverlappingElements(element, allElements, quadTree, processed) {
+ const result = [];
+ const queue = [element];
+
+ while (queue.length > 0) {
+ const current = queue.shift();
+
+ // Use quadtree to query nearby elements
+ const nearby = quadTree.query(current.rect);
+
+ for (const nearbyElement of nearby) {
+ if (
+ !processed.has(nearbyElement) &&
+ nearbyElement !== current &&
+ Rect.intersects(current.rect, nearbyElement.rect)
+ ) {
+ result.push(nearbyElement);
+ processed.add(nearbyElement);
+ queue.push(nearbyElement);
+ }
+ }
+ }
+ return result;
+}
+
+function createRectangleForGroup(group) {
+ const rects = group.elements.map((element) => element.rect);
+ const top = Math.min(...rects.map((rect) => rect.top));
+ const left = Math.min(...rects.map((rect) => rect.left));
+ const bottom = Math.max(...rects.map((rect) => rect.bottom));
+ const right = Math.max(...rects.map((rect) => rect.right));
+ return Rect.create(left, top, right, bottom);
+}
+
+function generateHintStrings(count) {
+ const hintCharacters = "sadfjklewcmpgh";
+ let hintStrings = [""];
+ let offset = 0;
+
+ while (hintStrings.length - offset < count || hintStrings.length === 1) {
+ const hintString = hintStrings[offset++];
+ for (const ch of hintCharacters) {
+ hintStrings.push(ch + hintString);
+ }
+ }
+ hintStrings = hintStrings.slice(offset, offset + count);
+
+ // Shuffle the hints so that they're scattered; hints starting with the same character and short
+ // hints are spread evenly throughout the array.
+ return hintStrings.sort(); // .map((str) => str.reverse())
+}
+
+function createHintMarkersForGroups(groups) {
+ if (groups.length === 0) {
+ _jsConsoleLog("No groups found, not adding hint markers to page.");
+ return [];
+ }
+
+ const hintMarkers = groups
+ .filter((group) => group.elements.some((element) => element.interactable))
+ .map((group) => createHintMarkerForGroup(group));
+ // fill in marker text
+ // const hintStrings = generateHintStrings(hintMarkers.length);
+ for (let i = 0; i < hintMarkers.length; i++) {
+ const hintMarker = hintMarkers[i];
+
+ let interactableElementFound = false;
+
+ for (let i = 0; i < hintMarker.group.elements.length; i++) {
+ if (hintMarker.group.elements[i].interactable) {
+ hintMarker.hintString = hintMarker.group.elements[i].id;
+ interactableElementFound = true;
+ break;
+ }
+ }
+
+ if (!interactableElementFound) {
+ hintMarker.hintString = "";
+ }
+
+ try {
+ hintMarker.element.innerHTML = hintMarker.hintString;
+ } catch (e) {
+ // Ensure trustedTypes is available
+ if (typeof trustedTypes !== "undefined") {
+ try {
+ const escapeHTMLPolicy = trustedTypes.createPolicy("hint-policy", {
+ createHTML: (string) => string,
+ });
+ hintMarker.element.innerHTML = escapeHTMLPolicy.createHTML(
+ hintMarker.hintString.toUpperCase(),
+ );
+ } catch (policyError) {
+ _jsConsoleWarn("Could not create trusted types policy:", policyError);
+ // Skip updating the hint marker if policy creation fails
+ }
+ } else {
+ _jsConsoleError("trustedTypes is not supported in this environment.");
+ }
+ }
+ }
+
+ return hintMarkers;
+}
+
+function createHintMarkerForGroup(group) {
+ // Calculate the position of the element relative to the document
+ var scrollTop = window.pageYOffset || document.documentElement.scrollTop;
+ var scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
+
+ const marker = {};
+ // yellow annotation box with string
+ const el = document.createElement("div");
+ el.style.position = "absolute";
+ el.style.left = group.rect.left + scrollLeft + "px";
+ el.style.top = group.rect.top + scrollTop + "px";
+ // Each group is assigned a different incremental z-index, we use the same z-index for the
+ // bounding box and the hint marker
+ el.style.zIndex = this.currentZIndex;
+
+ // The bounding box around the group of hints.
+ const boundingBox = document.createElement("div");
+
+ // Set styles for the bounding box
+ boundingBox.style.position = "absolute";
+ boundingBox.style.display = "display";
+ boundingBox.style.left = group.rect.left + scrollLeft + "px";
+ boundingBox.style.top = group.rect.top + scrollTop + "px";
+ boundingBox.style.width = group.rect.width + "px";
+ boundingBox.style.height = group.rect.height + "px";
+ boundingBox.style.bottom = boundingBox.style.top + boundingBox.style.height;
+ boundingBox.style.right = boundingBox.style.left + boundingBox.style.width;
+ boundingBox.style.border = "2px solid blue"; // Change the border color as needed
+ boundingBox.style.pointerEvents = "none"; // Ensures the box doesn't interfere with other interactions
+ boundingBox.style.zIndex = this.currentZIndex++;
+
+ return Object.assign(marker, {
+ element: el,
+ boundingBox: boundingBox,
+ group: group,
+ });
+}
+
+function addHintMarkersToPage(hintMarkers) {
+ const parent = document.createElement("div");
+ parent.id = "boundingBoxContainer";
+ for (const hintMarker of hintMarkers) {
+ parent.appendChild(hintMarker.element);
+ parent.appendChild(hintMarker.boundingBox);
+ }
+ document.documentElement.appendChild(parent);
+}
+
+function removeBoundingBoxes() {
+ var hintMarkerContainer = document.querySelector("#boundingBoxContainer");
+ if (hintMarkerContainer) {
+ hintMarkerContainer.remove();
+ }
+}
+
+function safeWindowScroll(x, y) {
+ if (typeof window.scroll === "function") {
+ window.scroll({ left: x, top: y, behavior: "instant" });
+ } else if (typeof window.scrollTo === "function") {
+ window.scrollTo({ left: x, top: y, behavior: "instant" });
+ } else {
+ _jsConsoleError("window.scroll and window.scrollTo are both not supported");
+ }
+}
+
+async function safeScrollToTop(
+ draw_boxes,
+ frame = "main.frame",
+ frame_index = undefined,
+) {
+ removeBoundingBoxes();
+ safeWindowScroll(0, 0);
+ if (draw_boxes) {
+ await buildElementsAndDrawBoundingBoxes(frame, frame_index);
+ }
+ return window.scrollY;
+}
+
+function getScrollWidthAndHeight() {
+ return [
+ document.documentElement.scrollWidth,
+ document.documentElement.scrollHeight,
+ ];
+}
+
+function getScrollXY() {
+ return [window.scrollX, window.scrollY];
+}
+
+function scrollToXY(x, y) {
+ safeWindowScroll(x, y);
+}
+
+async function scrollToNextPage(
+ draw_boxes,
+ frame = "main.frame",
+ frame_index = undefined,
+ need_overlap = true,
+) {
+ // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
+ // return true if there is a next page, false otherwise
+ removeBoundingBoxes();
+ window.scrollBy({
+ left: 0,
+ top: need_overlap ? window.innerHeight - 200 : window.innerHeight,
+ behavior: "instant",
+ });
+ if (draw_boxes) {
+ await buildElementsAndDrawBoundingBoxes(frame, frame_index);
+ }
+ return window.scrollY;
+}
+
+function isWindowScrollable() {
+ const documentBody = document.body;
+ const documentElement = document.documentElement;
+ if (!documentBody || !documentElement) {
+ return false;
+ }
+
+ // Check if the body's overflow style is set to hidden
+ const bodyOverflow = getElementComputedStyle(documentBody)?.overflow;
+ const htmlOverflow = getElementComputedStyle(documentElement)?.overflow;
+
+ // Check if the document height is greater than the window height
+ const isScrollable =
+ document.documentElement.scrollHeight > window.innerHeight;
+
+ // If the overflow is set to 'hidden' or there is no content to scroll, return false
+ if (bodyOverflow === "hidden" || htmlOverflow === "hidden" || !isScrollable) {
+ return false;
+ }
+
+ return true;
+}
+
+/**
+ * Find the nearest scrollable container relative to the given element and scroll it.
+ * Three strategies in priority order:
+ * 1) Walk up from element to find a scrollable ancestor (element is inside container)
+ * 2) Walk up the DOM checking siblings at each level (element is beside container)
+ * 3) Fall back to page-level scrolling (for pages where the body itself scrolls,
+ * e.g. T&C pages with no scrollable sub-container)
+ * Returns: true (sub-container scrolled), "page" (page-level scroll needed), or false (nothing scrollable).
+ */
+function scrollNearestScrollableContainer(element, direction) {
+ function isContainerScrollable(node) {
+ if (!node || node === document.documentElement || node === document.body)
+ return false;
+ const style = window.getComputedStyle(node);
+ const oy = style.overflowY;
+ return (
+ (oy === "auto" || oy === "scroll") &&
+ node.scrollHeight > node.clientHeight
+ );
+ }
+
+ // Strategy 1: walk up from element to find a scrollable ancestor
+ let target = element;
+ while (target && target !== document.documentElement) {
+ if (isContainerScrollable(target)) break;
+ target = target.parentElement;
+ }
+
+ // Strategy 2: walk up the DOM checking siblings at each level
+ if (!target || target === document.documentElement) {
+ target = null;
+ let level = element.parentElement;
+ while (level && level !== document.documentElement && !target) {
+ for (const child of level.children) {
+ if (isContainerScrollable(child)) {
+ target = child;
+ break;
+ }
+ }
+ level = level.parentElement;
+ }
+ }
+
+ // Scroll the sub-container if found
+ if (target) {
+ if (direction === "down") {
+ target.scrollTop = target.scrollHeight;
+ } else {
+ target.scrollTop = 0;
+ }
+ return true;
+ }
+
+ // Strategy 3: fall back to page-level scrolling when no sub-container exists.
+ // Many pages (e.g. T&C agreements) render content inline with the page body
+ // as the only scrollable area. Return "page" to signal the Python handler to
+ // use mouse.wheel events for native browser scrolling, which reliably triggers
+ // page JavaScript (IntersectionObserver, scroll listeners) that programmatic
+ // scrollTo/scrollTop methods may not.
+ if (isWindowScrollable()) {
+ return "page";
+ }
+
+ return false;
+}
+
+function scrollToElementBottom(element, page_by_page = false) {
+ const top = page_by_page
+ ? element.clientHeight + element.scrollTop
+ : element.scrollHeight;
+ element.scroll({
+ top: top,
+ left: 0,
+ behavior: "smooth",
+ });
+}
+
+function scrollToElementTop(element) {
+ element.scroll({
+ top: 0,
+ left: 0,
+ behavior: "instant",
+ });
+}
+
+/**
+ * Get all styles associated with :hover selectors
+ *
+ * Chrome doesn't allow you to compute these in run-time because hover is a protected attribute (from JS code)
+ *
+ * Instead of checking the hover state, we can look at the stylesheet and find all the :hover selectors
+ * and try to infer styles associated with them
+ *
+ * It's not 100% accurate, but it's a good start
+ *
+ * References:
+ * https://stackoverflow.com/questions/23040926/how-can-i-get-elementhover-style
+ * https://stackoverflow.com/questions/7013559/is-there-a-way-to-get-element-hover-style-while-the-element-not-in-hover-state
+ * https://stackoverflow.com/questions/17226676/how-to-simulate-a-mouseover-in-pure-javascript-that-activates-the-css-hover
+ */
+async function getHoverStylesMap() {
+ const hoverMap = new Map();
+ const sheets = [...document.styleSheets];
+
+ const parseCssSheet = (sheet) => {
+ const rules = sheet.cssRules || sheet.rules;
+ for (const rule of rules) {
+ if (rule.type === 1 && rule.selectorText) {
+ // Split multiple selectors (e.g., "a:hover, button:hover")
+ const selectors = rule.selectorText.split(",").map((s) => s.trim());
+
+ for (const selector of selectors) {
+ // Check if this is a hover rule
+ if (selector.includes(":hover")) {
+ // Get all parts of the selector
+ const parts = selector.split(/\s*[>+~]\s*/);
+
+ // Get the main hoverable element (the one with :hover)
+ const hoverPart = parts.find((part) => part.includes(":hover"));
+ if (!hoverPart) continue;
+
+ // Get base selector without :hover
+ const baseSelector = hoverPart.replace(/:hover/g, "").trim();
+
+ // Skip invalid selectors
+ if (!isValidCSSSelector(baseSelector)) {
+ continue;
+ }
+
+ // Get or create styles object for this selector
+ let styles = hoverMap.get(baseSelector) || {};
+
+ // Add all style properties
+ for (const prop of rule.style) {
+ styles[prop] = rule.style[prop];
+ }
+
+ // If this is a nested selector (like :hover > .something)
+ // store it in a special format
+ if (parts.length > 1) {
+ const fullSelector = selector;
+ styles["__nested__"] = styles["__nested__"] || [];
+ styles["__nested__"].push({
+ selector: fullSelector,
+ styles: Object.fromEntries(
+ [...rule.style].map((prop) => [prop, rule.style[prop]]),
+ ),
+ });
+ }
+
+ // only need the style which includes the cursor attribute.
+ if (!("cursor" in styles)) {
+ continue;
+ }
+ hoverMap.set(baseSelector, styles);
+ }
+ }
+ }
+ }
+ };
+
+ try {
+ await Promise.all(
+ sheets.map(async (sheet) => {
+ try {
+ parseCssSheet(sheet);
+ } catch (e) {
+ _jsConsoleWarn("Could not access stylesheet:", e);
+
+ if ((e.name !== "SecurityError" && e.code !== 18) || !sheet.href) {
+ return;
+ }
+
+ let newLink = null;
+ try {
+ const oldLink = sheet.ownerNode;
+ const url = new URL(sheet.href);
+ _jsConsoleLog("recreating the link element: ", sheet.href);
+ newLink = document.createElement("link");
+ newLink.rel = "stylesheet";
+ url.searchParams.set("v", Date.now());
+ newLink.href = url.toString();
+ newLink.crossOrigin = "anonymous";
+ // until the new link loaded, removing the old one
+ document.head.append(newLink);
+
+ // wait for a while until the sheet is fully loaded
+ await asyncSleepFor(1500);
+ const newSheets = [...document.styleSheets];
+ const refreshedSheet = newSheets.find(
+ (s) => s.href === newLink.href,
+ );
+ if (!refreshedSheet) {
+ newLink.remove();
+ return;
+ }
+ _jsConsoleLog("parsing recreated the link element: ", newLink.href);
+ parseCssSheet(refreshedSheet);
+ oldLink.remove();
+ } catch (e) {
+ _jsConsoleWarn("Error recreating the link element:", e);
+ if (newLink) {
+ newLink.remove();
+ }
+ }
+ }
+ }),
+ );
+ } catch (e) {
+ _jsConsoleError("Error processing stylesheets:", e);
+ }
+
+ return hoverMap;
+}
+
+// Helper method for debugging
+function findNodeById(arr, targetId, path = []) {
+ for (let i = 0; i < arr.length; i++) {
+ const currentPath = [...path, arr[i].id];
+ if (arr[i].id === targetId) {
+ _jsConsoleLog("Lineage:", currentPath.join(" -> "));
+ return arr[i];
+ }
+ if (arr[i].children && arr[i].children.length > 0) {
+ const result = findNodeById(arr[i].children, targetId, currentPath);
+ if (result) {
+ return result;
+ }
+ }
+ }
+ return null;
+}
+
+function getElementDomDepth(elementNode) {
+ let depth = 0;
+ const rootElement = elementNode.getRootNode().firstElementChild;
+ while (elementNode !== rootElement && elementNode.parentElement) {
+ depth++;
+ elementNode = elementNode.parentElement;
+ }
+ return depth;
+}
+
+if (window.globalOneTimeIncrementElements === undefined) {
+ window.globalOneTimeIncrementElements = [];
+}
+
+if (window.globalDomDepthMap === undefined) {
+ window.globalDomDepthMap = new Map();
+}
+
+function isClassNameIncludesHidden(className) {
+ // some hidden elements are with the classname like `class="select-items select-hide"` or `class="dropdown-container dropdown-invisible"`
+ return (
+ className.toLowerCase().includes("hide") ||
+ className.toLowerCase().includes("invisible") ||
+ className.toLowerCase().includes("closed")
+ );
+}
+
+function isClassNameIncludesActivatedStatus(className) {
+ // some elements are with the classname like `class="open"` or `class="active"` should be considered as activated by the click
+ return (
+ className.toLowerCase().includes("open") ||
+ className.toLowerCase().includes("active")
+ );
+}
+
+function waitForNextFrame() {
+ return new Promise((resolve) => {
+ requestAnimationFrame(() => resolve());
+ });
+}
+
+function asyncSleepFor(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function addIncrementalNodeToMap(parentNode, childrenNode) {
+ const maxParsedElement = 3000;
+ const maxElementToWait = 100;
+ if ((await window.globalParsedElementCounter.get()) > maxParsedElement) {
+ _jsConsoleWarn(
+ "Too many elements parsed, stopping the observer to parse the elements",
+ );
+ await window.globalParsedElementCounter.add();
+ return;
+ }
+
+ // make the dom parser async
+ await waitForNextFrame();
+ if (window.globalListnerFlag) {
+ // calculate the depth of targetNode element for sorting
+ const depth = getElementDomDepth(parentNode);
+ let newNodesTreeList = [];
+ if (window.globalDomDepthMap.has(depth)) {
+ newNodesTreeList = window.globalDomDepthMap.get(depth);
+ }
+
+ try {
+ for (const child of childrenNode) {
+ // sleep for a while until animation ends
+ if (
+ (await window.globalParsedElementCounter.get()) < maxElementToWait
+ ) {
+ await asyncSleepFor(300);
+ }
+ // Pass -1 as frame_index to indicate the frame number is not sensitive in this case
+ const [_, newNodeTree] = await buildElementTree(
+ child,
+ "",
+ true,
+ window.globalHoverStylesMap,
+ );
+ if (newNodeTree.length > 0) {
+ newNodesTreeList.push(...newNodeTree);
+ }
+ }
+ } catch (error) {
+ _jsConsoleError("Error building incremental element node:", error);
+ }
+ window.globalDomDepthMap.set(depth, newNodesTreeList);
+ }
+ await window.globalParsedElementCounter.add();
+}
+
+if (window.globalObserverForDOMIncrement === undefined) {
+ window.globalObserverForDOMIncrement = new MutationObserver(async function (
+ mutationsList,
+ observer,
+ ) {
+ // TODO: how to detect duplicated recreate element?
+ for (const mutation of mutationsList) {
+ const node = mutation.target;
+ if (node.nodeType === Node.TEXT_NODE) continue;
+ const tagName = node.tagName?.toLowerCase();
+
+ // ignore unique_id change to avoid infinite loop about DOM changes
+ if (mutation.attributeName === "unique_id") continue;
+
+ // if the changing element is dropdown related elements, we should consider
+ // they're the new element as long as the element is still visible on the page
+ if (
+ isDropdownRelatedElement(node) &&
+ getElementComputedStyle(node)?.display !== "none"
+ ) {
+ window.globalOneTimeIncrementElements.push({
+ targetNode: node,
+ newNodes: [node],
+ });
+ await addIncrementalNodeToMap(node, [node]);
+ continue;
+ }
+
+ // if they're not the dropdown related elements
+ // we detect the element based on the following rules
+ switch (mutation.type) {
+ case "attributes": {
+ switch (mutation.attributeName) {
+ case "hidden": {
+ if (!node.hidden) {
+ window.globalOneTimeIncrementElements.push({
+ targetNode: node,
+ newNodes: [node],
+ });
+ await addIncrementalNodeToMap(node, [node]);
+ }
+ break;
+ }
+ case "style": {
+ // TODO: need to confirm that elemnent is hidden previously
+ if (tagName === "body") continue;
+ if (getElementComputedStyle(node)?.display !== "none") {
+ window.globalOneTimeIncrementElements.push({
+ targetNode: node,
+ newNodes: [node],
+ });
+ await addIncrementalNodeToMap(node, [node]);
+ }
+ break;
+ }
+ case "class": {
+ if (tagName === "body") continue;
+ if (!mutation.oldValue) continue;
+ const currentClassName = node.className
+ ? node.className.toString()
+ : "";
+ if (
+ !isClassNameIncludesHidden(mutation.oldValue) &&
+ !isClassNameIncludesActivatedStatus(currentClassName) &&
+ !node.hasAttribute("data-menu-uid") && // google framework use this to trace dropdown menu
+ !mutation.oldValue.includes("select__items") &&
+ !(
+ node.hasAttribute("data-testid") &&
+ node.getAttribute("data-testid").includes("select-dropdown")
+ )
+ )
+ continue;
+ if (getElementComputedStyle(node)?.display !== "none") {
+ window.globalOneTimeIncrementElements.push({
+ targetNode: node,
+ newNodes: [node],
+ });
+ await addIncrementalNodeToMap(node, [node]);
+ }
+ break;
+ }
+ }
+ break;
+ }
+ case "childList": {
+ let changedNode = {
+ targetNode: node, // TODO: for future usage, when we want to parse new elements into a tree
+ };
+ let newNodes = [];
+ if (mutation.addedNodes && mutation.addedNodes.length > 0) {
+ for (const node of mutation.addedNodes) {
+ // skip the text nodes, they won't be interactable
+ if (node.nodeType === Node.TEXT_NODE) continue;
+ newNodes.push(node);
+ }
+ }
+ if (
+ newNodes.length == 0 &&
+ (tagName === "ul" ||
+ (tagName === "div" &&
+ node.hasAttribute("role") &&
+ node.getAttribute("role").toLowerCase() === "listbox"))
+ ) {
+ newNodes.push(node);
+ }
+
+ if (newNodes.length > 0) {
+ changedNode.newNodes = newNodes;
+ window.globalOneTimeIncrementElements.push(changedNode);
+ await addIncrementalNodeToMap(
+ changedNode.targetNode,
+ changedNode.newNodes,
+ );
+ }
+ break;
+ }
+ }
+ }
+ });
+}
+
+async function startGlobalIncrementalObserver(element = null) {
+ window.globalListnerFlag = true;
+ window.globalDomDepthMap = new Map();
+ window.globalOneTimeIncrementElements = [];
+ window.globalHoverStylesMap = await getHoverStylesMap();
+ window.globalParsedElementCounter = new SafeCounter();
+ window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
+ window.globalObserverForDOMIncrement.observe(document.body, {
+ attributes: true,
+ attributeOldValue: true,
+ childList: true,
+ subtree: true,
+ characterData: true,
+ });
+
+ // if the element is in shadow DOM, we need to observe the shadow DOM as well
+ if (element && element.getRootNode() instanceof ShadowRoot) {
+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {
+ attributes: true,
+ attributeOldValue: true,
+ childList: true,
+ subtree: true,
+ characterData: true,
+ });
+ }
+}
+
+async function stopGlobalIncrementalObserver() {
+ window.globalListnerFlag = false;
+ window.globalObserverForDOMIncrement.disconnect();
+ window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
+ while (
+ window.globalParsedElementCounter &&
+ window.globalOneTimeIncrementElements &&
+ (await window.globalParsedElementCounter.get()) <
+ window.globalOneTimeIncrementElements.length
+ ) {
+ await asyncSleepFor(100);
+ }
+ window.globalOneTimeIncrementElements = [];
+ window.globalDomDepthMap = new Map();
+}
+
+async function getIncrementElements(wait_until_finished = true) {
+ if (wait_until_finished) {
+ while (
+ (await window.globalParsedElementCounter.get()) <
+ window.globalOneTimeIncrementElements.length
+ ) {
+ await asyncSleepFor(100);
+ }
+ }
+
+ // cleanup the children tree, remove the duplicated element
+ // search starting from the shallowest node:
+ // 1. if deeper, the node could only be the children of the shallower one or no related one.
+ // 2. if depth is same, the node could only be duplicated one or no related one.
+ const idToElement = new Map();
+ const cleanedTreeList = [];
+ const sortedDepth = Array.from(window.globalDomDepthMap.keys()).sort(
+ (a, b) => a - b,
+ );
+ for (let idx = 0; idx < sortedDepth.length; idx++) {
+ const depth = sortedDepth[idx];
+ const treeList = window.globalDomDepthMap.get(depth);
+
+ const removeDupAndConcatChildren = async (element) => {
+ let children = element.children;
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
+ if (child.shadowHost) {
+ continue;
+ }
+ const domElement = document.querySelector(`[unique_id="${child.id}"]`);
+ // if the element is still on the page, we rebuild the element to update the information
+ if (domElement) {
+ let newChild = await buildElementObject(
+ "",
+ domElement,
+ child.interactable,
+ child.purgeable,
+ );
+ newChild.children = child.children;
+ children[i] = newChild;
+ } else {
+ children[i].interactable = false;
+ }
+ }
+
+ if (idToElement.has(element.id)) {
+ element = idToElement.get(element.id);
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ if (!idToElement.get(child.id)) {
+ element.children.push(child);
+ }
+ }
+ }
+ idToElement.set(element.id, element);
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ await removeDupAndConcatChildren(child);
+ }
+ };
+
+ for (let treeHeadElement of treeList) {
+ // FIXME: skip to update the element if it is in shadow DOM, since document.querySelector will not work
+ if (!treeHeadElement.shadowHost) {
+ const domElement = document.querySelector(
+ `[unique_id="${treeHeadElement.id}"]`,
+ );
+ // if the element is still on the page, we rebuild the element to update the information
+ if (domElement) {
+ let newHead = await buildElementObject(
+ "",
+ domElement,
+ treeHeadElement.interactable,
+ treeHeadElement.purgeable,
+ );
+ newHead.children = treeHeadElement.children;
+ treeHeadElement = newHead;
+ } else {
+ treeHeadElement.interactable = false;
+ }
+ }
+
+ // check if the element is existed
+ if (!idToElement.has(treeHeadElement.id)) {
+ cleanedTreeList.push(treeHeadElement);
+ }
+ await removeDupAndConcatChildren(treeHeadElement);
+ }
+ }
+
+ return [Array.from(idToElement.values()), cleanedTreeList];
+}
+
+function isAnimationFinished() {
+ const animations = document.getAnimations({ subtree: true });
+ const unfinishedAnimations = animations.filter(
+ (a) => a.playState !== "finished",
+ );
+ if (!unfinishedAnimations || unfinishedAnimations.length == 0) {
+ return true;
+ }
+ return false;
+}
+
+/**
+ * Remove unique_id attribute from all elements on the page.
+ * This includes elements in the main document and shadow DOM.
+ */
+function removeAllUniqueIds() {
+ // Function to recursively remove unique_id from an element and its children
+ const removeUniqueIdFromElement = (element) => {
+ if (element.hasAttribute("unique_id")) {
+ element.removeAttribute("unique_id");
+ }
+
+ // Process children in the main DOM
+ for (const child of Array.from(element.children)) {
+ removeUniqueIdFromElement(child);
+ }
+
+ // Process elements in shadow DOM if present
+ if (element.shadowRoot) {
+ for (const shadowChild of Array.from(element.shadowRoot.children)) {
+ removeUniqueIdFromElement(shadowChild);
+ }
+ }
+ };
+
+ // Start from document.documentElement to process the entire page
+ removeUniqueIdFromElement(document.documentElement);
+}
+
+/**
+
+// How to run the code:
+
+// Get all interactable elements and draw boxes
+buildElementsAndDrawBoundingBoxes();
+
+// Remove the boxes
+removeBoundingBoxes();
+
+// Get the element tree
+const [elements, tree] = buildTreeFromBody();
+_jsConsoleLog(elements); // All elements
+_jsConsoleLog(tree); // Tree structure
+
+// Test if a specific element is interactable
+const element = document.querySelector('button');
+const hoverMap = getHoverStylesMap();
+_jsConsoleLog(isInteractable(element, hoverMap));
+ */
diff --git a/skyvern/webeye/scraper/scraped_page.py b/skyvern/webeye/scraper/scraped_page.py
index 4036d32e0..460c1211b 100644
--- a/skyvern/webeye/scraper/scraped_page.py
+++ b/skyvern/webeye/scraper/scraped_page.py
@@ -2,6 +2,7 @@ import copy
import json
import typing
from abc import ABC, abstractmethod
+from collections import deque
from enum import StrEnum
from typing import Any, Awaitable, Callable, Self
@@ -9,6 +10,7 @@ import structlog
from playwright.async_api import Frame, Page
from pydantic import BaseModel, PrivateAttr
+from skyvern.config import settings
from skyvern.exceptions import UnknownElementTreeFormat
from skyvern.forge.sdk.api.crypto import calculate_sha256
from skyvern.forge.sdk.core import skyvern_context
@@ -33,10 +35,8 @@ def build_attribute(key: str, value: Any) -> str:
return f'{key}="{str(value)}"' if value else key
-def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
- """
- if element is flagged as dropped, the html format is empty
- """
+def _json_to_html_legacy(element: dict, need_skyvern_attrs: bool = True) -> str:
+ """Legacy: always deep-copies attributes (pre-V2)."""
tag = element["tagName"]
attributes: dict[str, Any] = copy.deepcopy(element.get("attributes", {}))
@@ -50,18 +50,13 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
context = skyvern_context.ensure_context()
- # FIXME: Theoretically, all href links with over 69(64+1+4) length could be hashed
- # but currently, just hash length>150 links to confirm the solution goes well
if "href" in attributes and len(attributes.get("href", "")) > 150:
href = attributes.get("href", "")
- # jinja style can't accept the variable name starts with number
- # adding "_" to make sure the variable name is valid.
hashed_href = "_" + calculate_sha256(href)
context.hashed_href_map[hashed_href] = href
attributes["href"] = "{{" + hashed_href + "}}"
if need_skyvern_attrs:
- # adding the node attribute to attributes
for attr in ELEMENT_NODE_ATTRIBUTES:
value = element.get(attr)
if value is None:
@@ -70,6 +65,92 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())
+ if element.get("isSelectable", False):
+ tag = "select"
+
+ text = element.get("text", "")
+ children_html = "".join(
+ _json_to_html_legacy(child, need_skyvern_attrs=need_skyvern_attrs) for child in element.get("children", [])
+ )
+ option_html = "".join(
+ f'<option index="{option.get("optionIndex")}">{option.get("text")}</option>'
+ if option.get("text")
+ else f'<option index="{option.get("optionIndex")}" value="{option.get("value")}">{option.get("text")}</option>'
+ for option in element.get("options", [])
+ )
+
+ if element.get("purgeable", False):
+ return children_html + option_html
+
+ before_pseudo_text = element.get("beforePseudoText") or ""
+ after_pseudo_text = element.get("afterPseudoText") or ""
+
+ if (
+ tag in ["img", "input", "br", "hr", "meta", "link"]
+ and not option_html
+ and not children_html
+ and not before_pseudo_text
+ and not after_pseudo_text
+ ):
+ return f"<{tag} {attributes_html}/>" if attributes_html else f"<{tag}/>"
+
+ return (
+ (f"<{tag} {attributes_html}>" if attributes_html else f"<{tag}>")
+ + before_pseudo_text
+ + text
+ + children_html
+ + option_html
+ + after_pseudo_text
+ + f"</{tag}>"
+ )
+
+
+def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
+ """
+ if element is flagged as dropped, the html format is empty
+ """
+ if not settings.ENABLE_DOM_PARSER_V2:
+ return _json_to_html_legacy(element, need_skyvern_attrs)
+
+ tag = element["tagName"]
+ original_attrs = element.get("attributes", {})
+
+ interactable = element.get("interactable", False)
+ if element.get("isDropped", False):
+ if not interactable:
+ return ""
+ else:
+ LOG.debug("Element is interactable. Trimmed all attributes instead of dropping it", element=element)
+ original_attrs = {}
+
+ context = skyvern_context.ensure_context()
+
+ # Only shallow-copy attributes when we actually need to mutate them.
+ # This avoids dict() allocation on every element — most elements don't need it.
+ attributes = original_attrs
+ href_val = original_attrs.get("href", "")
+
+ if href_val and len(href_val) > 150:
+ attributes = dict(original_attrs)
+ # jinja style can't accept the variable name starts with number
+ # adding "_" to make sure the variable name is valid.
+ hashed_href = "_" + calculate_sha256(href_val)
+ context.hashed_href_map[hashed_href] = href_val
+ attributes["href"] = "{{" + hashed_href + "}}"
+
+ if need_skyvern_attrs:
+ # adding the node attribute to attributes
+ has_skyvern_attrs = any(element.get(attr) is not None for attr in ELEMENT_NODE_ATTRIBUTES)
+ if has_skyvern_attrs:
+ if attributes is original_attrs:
+ attributes = dict(original_attrs)
+ for attr in ELEMENT_NODE_ATTRIBUTES:
+ value = element.get(attr)
+ if value is not None:
+ attributes[attr] = value
+
+ attributes_html = " ".join(build_attribute(key, value) for key, value in attributes.items())
+
if element.get("isSelectable", False):
tag = "select"
@@ -275,25 +356,45 @@ class ScrapedPage(BaseModel, ElementTreeBuilder):
raise UnknownElementTreeFormat(fmt=fmt)
- def _process_element_for_economy_tree(self, element: dict) -> dict | None:
- """
- Helper method to process an element for the economy tree using BFS.
- Removes SVG elements and their children.
- """
- # Skip SVG elements entirely
+ @staticmethod
+ def _process_element_for_economy_tree_legacy(element: dict) -> dict | None:
+ """Legacy: recursive SVG removal (pre-V2)."""
if element.get("tagName", "").lower() == "svg":
return None
- # Process children using BFS
if "children" in element:
new_children = []
for child in element["children"]:
- processed_child = self._process_element_for_economy_tree(child)
+ processed_child = ScrapedPage._process_element_for_economy_tree_legacy(child)
if processed_child:
new_children.append(processed_child)
element["children"] = new_children
return element
+ @staticmethod
+ def _process_element_for_economy_tree(element: dict) -> dict | None:
+ """
+ Process an element for the economy tree. V2 uses iterative BFS.
+ Removes SVG elements and their children.
+ """
+ if not settings.ENABLE_DOM_PARSER_V2:
+ return ScrapedPage._process_element_for_economy_tree_legacy(element)
+
+ if element.get("tagName", "").lower() == "svg":
+ return None
+
+ # BFS to filter SVG children at every level
+ queue: deque[dict] = deque([element])
+ while queue:
+ node = queue.popleft()
+ children = node.get("children")
+ if not children:
+ continue
+ filtered = [c for c in children if c.get("tagName", "").lower() != "svg"]
+ node["children"] = filtered
+ queue.extend(filtered)
+ return element
+
async def refresh(self, draw_boxes: bool = True, scroll: bool = True, max_retries: int = 0) -> Self:
refreshed_page = await self._browser_state.scrape_website(
url=self.url,
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 854a90be5..0ef2c08c6 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -1,7 +1,8 @@
import asyncio
import copy
import json
-from collections import defaultdict
+import time
+from collections import defaultdict, deque
import structlog
from playwright._impl._errors import TimeoutError
@@ -109,6 +110,7 @@ def clean_element_before_hashing(element: dict) -> dict:
def hash_element(element: dict) -> str:
hash_ready_element = clean_element_before_hashing(element)
# Sort the keys to ensure consistent ordering
+ # NOTE: Do NOT change separators — hashes are persisted to the database for cached action matching.
element_string = json.dumps(hash_ready_element, sort_keys=True)
return calculate_sha256(element_string)
@@ -131,7 +133,10 @@ def build_element_dict(
id_to_frame_dict[element_id] = element["frame"]
element_hash = hash_element(element)
id_to_element_hash[element_id] = element_hash
- hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]
+ if SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:
+ hash_to_element_ids.setdefault(element_hash, []).append(element_id)
+ else:
+ hash_to_element_ids[element_hash] = hash_to_element_ids.get(element_hash, []) + [element_id]
return id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids
@@ -296,6 +301,7 @@ async def scrape_web_unsafe(
# We check if the scroll_y_px_old is the same as scroll_y_px to determine if we have reached the end of the page.
# This also solves the issue where we can't scroll due to a popup.(e.g. geico first popup on the homepage after
# clicking start my quote)
+ scrape_start_time = time.time()
url = page.url
if url == "about:blank" and not support_empty_page:
# Allow scraping if the page has child frames with meaningful content
@@ -309,23 +315,46 @@ async def scrape_web_unsafe(
frame_count=len(meaningful_frames),
)
+ t0 = time.time()
skyvern_frame = await SkyvernFrame.create_instance(page)
+ js_inject_time = time.time() - t0
await skyvern_frame.safe_wait_for_animation_end()
if wait_seconds > 0:
LOG.info(f"Waiting for {wait_seconds} seconds before scraping the website.", wait_seconds=wait_seconds)
await asyncio.sleep(wait_seconds)
- elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
+ t0 = time.time()
+ elements, element_tree, main_frame_time, child_frames_time, num_child_frames = await get_interactable_element_tree(
+ page, scrape_exclude, must_included_tags
+ )
+ element_tree_time = time.time() - t0
if not elements and not support_empty_page:
LOG.warning("No elements found on the page, wait and retry")
await empty_page_retry_wait()
- elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
-
- element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))
+ t0 = time.time()
+ (
+ elements,
+ element_tree,
+ main_frame_time,
+ child_frames_time,
+ num_child_frames,
+ ) = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
+ element_tree_time = time.time() - t0
+
+ # Deep copy once for cleanup (mutates in-place), then copy the cleaned result for trim.
+ # The second copy of the cleaned tree is cheaper since cleanup may remove elements.
+ t0 = time.time()
+ element_tree_copy = copy.deepcopy(element_tree)
+ element_tree = await cleanup_element_tree(page, url, element_tree_copy)
+ cleanup_time = time.time() - t0
+
+ t0 = time.time()
element_tree_trimmed = trim_element_tree(copy.deepcopy(element_tree))
+ trim_time = time.time() - t0
screenshots = []
+ screenshot_time = 0.0
if take_screenshots:
element_tree_trimmed_html_str = "".join(
json_to_html(element, need_skyvern_attrs=False) for element in element_tree_trimmed
@@ -343,6 +372,7 @@ async def scrape_web_unsafe(
except Exception:
LOG.warning("Failed to get current x, y position of the page", exc_info=True)
+ t0 = time.time()
screenshots = await SkyvernFrame.take_split_screenshots(
page=page,
url=url,
@@ -350,21 +380,26 @@ async def scrape_web_unsafe(
max_number=max_screenshot_number,
scroll=scroll,
)
+ screenshot_time = time.time() - t0
# scroll back to the original x, y position of the page
if x is not None and y is not None:
await skyvern_frame.safe_scroll_to_x_y(x, y)
LOG.debug("Scrolled back to the original x, y position of the page after scraping", x=x, y=y)
+ t0 = time.time()
id_to_css_dict, id_to_element_dict, id_to_frame_dict, id_to_element_hash, hash_to_element_ids = build_element_dict(
elements
)
+ build_dict_time = time.time() - t0
# if there are no elements, fail the scraping unless support_empty_page is True
if not elements and not support_empty_page:
raise NoElementFound()
+ t0 = time.time()
text_content = await get_frame_text(page.main_frame)
+ text_extraction_time = time.time() - t0
html = ""
window_dimension = None
@@ -380,6 +415,25 @@ async def scrape_web_unsafe(
exc_info=True,
)
+ total_scrape_time = time.time() - scrape_start_time
+ LOG.debug(
+ "Scraping performance metrics",
+ url=url,
+ total_scrape_time=total_scrape_time,
+ js_inject_time=js_inject_time,
+ element_tree_time=element_tree_time,
+ main_frame_time=main_frame_time,
+ child_frames_time=child_frames_time,
+ num_child_frames=num_child_frames,
+ cleanup_time=cleanup_time,
+ trim_time=trim_time,
+ screenshot_time=screenshot_time,
+ build_dict_time=build_dict_time,
+ text_extraction_time=text_extraction_time,
+ num_elements=len(elements),
+ num_screenshots=len(screenshots),
+ )
+
return ScrapedPage(
elements=elements,
id_to_css_dict=id_to_css_dict,
@@ -480,17 +534,20 @@ async def get_interactable_element_tree(
page: Page,
scrape_exclude: ScrapeExcludeFunc | None = None,
must_included_tags: list[str] | None = None,
-) -> tuple[list[dict], list[dict]]:
+) -> tuple[list[dict], list[dict], float, float, int]:
"""
Get the element tree of the page, including all the elements that are interactable.
:param page: Page instance to get the element tree from.
- :return: Tuple containing the element tree and a map of element IDs to elements.
+ :return: Tuple of (elements, element_tree, main_frame_time, child_frames_time, num_child_frames).
"""
# main page index is 0
skyvern_page = await SkyvernFrame.create_instance(page)
+
+ t0 = time.time()
elements, element_tree = await skyvern_page.build_tree_from_body(
frame_name="main.frame", frame_index=0, must_included_tags=must_included_tags
)
+ main_frame_time = time.time() - t0
context = skyvern_context.ensure_context()
frames = await get_all_children_frames(page)
@@ -502,6 +559,7 @@ async def get_interactable_element_tree(
frame_index = len(context.frame_index_map) + 1
context.frame_index_map[frame] = frame_index
+ t0 = time.time()
for frame in frames:
frame_index = context.frame_index_map[frame]
elements, element_tree = await add_frame_interactable_elements(
@@ -511,8 +569,9 @@ async def get_interactable_element_tree(
element_tree,
must_included_tags,
)
+ child_frames_time = time.time() - t0
- return elements, element_tree
+ return elements, element_tree, main_frame_time, child_frames_time, len(frames)
class IncrementalScrapePage(ElementTreeBuilder):
@@ -557,7 +616,9 @@ class IncrementalScrapePage(ElementTreeBuilder):
self.elements = incremental_elements
- incremental_tree = await cleanup_element_tree(frame, frame.url, copy.deepcopy(incremental_tree))
+ incremental_tree_copy = copy.deepcopy(incremental_tree)
+ incremental_tree = await cleanup_element_tree(frame, frame.url, incremental_tree_copy)
+ # Second copy of cleaned tree is cheaper since cleanup may have removed elements
trimmed_element_tree = trim_element_tree(copy.deepcopy(incremental_tree))
self.element_tree = incremental_tree
@@ -699,7 +760,8 @@ def _should_keep_unique_id(element: dict) -> bool:
return element.get("interactable", False)
-def trim_element(element: dict) -> dict:
+def _trim_element_legacy(element: dict) -> dict:
+ """Legacy: list-based BFS with two-pass attribute filtering (pre-V2)."""
queue = [element]
while queue:
queue_ele = queue.pop(0)
@@ -725,7 +787,6 @@ def trim_element(element: dict) -> dict:
queue_ele["attributes"] = new_attributes
else:
del queue_ele["attributes"]
- # remove the tag, don't need it in the HTML tree
if "keepAllAttr" in queue_ele:
del queue_ele["keepAllAttr"]
@@ -754,6 +815,53 @@ def trim_element(element: dict) -> dict:
return element
+def trim_element(element: dict) -> dict:
+ if not SettingsManager.get_settings().ENABLE_DOM_PARSER_V2:
+ return _trim_element_legacy(element)
+
+ queue: deque = deque([element])
+ while queue:
+ queue_ele = queue.popleft()
+ queue_ele.pop("frame", None)
+ queue_ele.pop("frame_index", None)
+
+ if "id" in queue_ele and not _should_keep_unique_id(queue_ele):
+ del queue_ele["id"]
+
+ # Single-pass attribute filtering: remove base64 data AND whitelist in one pass
+ keep_all = queue_ele.get("keepAllAttr", False)
+ attributes = queue_ele.get("attributes")
+ if attributes:
+ new_attributes = _filter_attributes(attributes, keep_all)
+ if new_attributes:
+ queue_ele["attributes"] = new_attributes
+ else:
+ queue_ele.pop("attributes", None)
+
+ queue_ele.pop("keepAllAttr", None)
+
+ children = queue_ele.get("children")
+ if children:
+ queue.extend(children)
+ else:
+ queue_ele.pop("children", None)
+
+ text = queue_ele.get("text")
+ if text is not None and not str(text).strip():
+ del queue_ele["text"]
+
+ # Remove empty pseudo text fields using pop-and-reinsert pattern to avoid double lookup
+ before = queue_ele.pop("beforePseudoText", None)
+ if before:
+ queue_ele["beforePseudoText"] = before
+
+ after = queue_ele.pop("afterPseudoText", None)
+ if after:
+ queue_ele["afterPseudoText"] = after
+
+ return element
+
+
def trim_element_tree(elements: list[dict]) -> list[dict]:
for element in elements:
trim_element(element)
@@ -783,6 +891,24 @@ def _trimmed_attributes(attributes: dict) -> dict:
return new_attributes
+def _filter_attributes(attributes: dict, keep_all: bool) -> dict:
+ """Single-pass attribute filtering: removes base64 data, applies whitelist, and truncates long names."""
+ new_attributes: dict = {}
+ for key, value in attributes.items():
+ # Skip base64 data URIs
+ if key in BASE64_INCLUDE_ATTRIBUTES and isinstance(value, str) and "data:" in value:
+ continue
+ # Apply whitelist (unless keepAllAttr is True)
+ if keep_all or key in RESERVED_ATTRIBUTES:
+ # Truncate long name attributes in the same pass
+ if key == "name" and isinstance(value, str) and len(value) > 500:
+ value = value[:500]
+ new_attributes[key] = value
+ elif key == "role" and value in ("listbox", "option"):
+ new_attributes[key] = value
+ return new_attributes
+
+
def _remove_unique_id(element: dict) -> None:
if "attributes" not in element:
return
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 6b0a4b6c6..a0b172d92 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -20,12 +20,9 @@ from skyvern.forge.sdk.trace import traced
LOG = structlog.get_logger()
-def load_js_script() -> str:
- # TODO: Handle file location better. This is a hacky way to find the file location.
- path = f"{SKYVERN_DIR}/webeye/scraper/domUtils.js"
+def load_js_script(filename: str = "domUtils.js") -> str:
+ path = f"{SKYVERN_DIR}/webeye/scraper/{filename}"
try:
- # TODO: Implement TS of domUtils.js and use the complied JS file instead of the raw JS file.
- # This will allow our code to be type safe.
with open(path, encoding="utf-8") as f:
return f.read()
except FileNotFoundError as e:
@@ -33,7 +30,8 @@ def load_js_script() -> str:
raise e
-JS_FUNCTION_DEFS = load_js_script()
+JS_FUNCTION_DEFS = load_js_script("domUtils.js")
+JS_FUNCTION_DEFS_LEGACY = load_js_script("domUtils_legacy.js")
class ScreenshotMode(StrEnum):
@@ -379,8 +377,17 @@ class SkyvernFrame:
@classmethod
async def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:
instance = cls(frame=frame)
- await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)
- if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:
+ settings = SettingsManager.get_settings()
+ js_to_inject = JS_FUNCTION_DEFS if settings.ENABLE_DOM_PARSER_V2 else JS_FUNCTION_DEFS_LEGACY
+ t0 = time.time()
+ await cls.evaluate(frame=instance.frame, expression=js_to_inject)
+ js_inject_elapsed = time.time() - t0
+ LOG.debug(
+ "domUtils.js injection time",
+ elapsed_time=js_inject_elapsed,
+ dom_parser_v2=settings.ENABLE_DOM_PARSER_V2,
+ )
+ if settings.ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:
await instance.evaluate(
frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true"
)
@@ -543,12 +550,23 @@ class SkyvernFrame:
) -> tuple[list[dict], list[dict]]:
must_included_tags = must_included_tags or []
js_script = "async ([frame_name, frame_index, must_included_tags]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags)"
- return await self.evaluate(
+ t0 = time.time()
+ result = await self.evaluate(
frame=self.frame,
expression=js_script,
timeout_ms=timeout_ms,
arg=[frame_name, frame_index, must_included_tags],
)
+ elapsed = time.time() - t0
+ num_elements = len(result[0]) if result and len(result) > 0 else 0
+ LOG.debug(
+ "buildTreeFromBody JS execution time",
+ frame_name=frame_name,
+ frame_index=frame_index,
+ elapsed_time=elapsed,
+ num_elements=num_elements,
+ )
+ return result
@traced()
async def get_incremental_element_tree(
diff --git a/tests/unit/test_cli_bootstrap.py b/tests/unit/test_cli_bootstrap.py
new file mode 100644
index 000000000..bb29b7367
--- /dev/null
+++ b/tests/unit/test_cli_bootstrap.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import logging
+from types import SimpleNamespace
+
+import skyvern._cli_bootstrap as cli_bootstrap
+
+
+def test_bootstrap_defaults_to_warning_without_explicit_log_level(monkeypatch) -> None:
+ setup_calls: list[str] = []
+ fake_settings = SimpleNamespace(LOG_LEVEL="INFO", model_fields_set=set())
+
+ monkeypatch.setattr("skyvern.config.settings", fake_settings)
+ monkeypatch.setattr("skyvern.forge.sdk.forge_log.setup_logger", lambda: setup_calls.append("called"))
+
+ logger_names = ("", "skyvern", "httpx", "litellm", "playwright", "httpcore")
+ previous_levels = {name: logging.getLogger(name).level for name in logger_names}
+ try:
+ cli_bootstrap.configure_cli_bootstrap_logging()
+ assert setup_calls == ["called"]
+ assert fake_settings.LOG_LEVEL == "WARNING"
+ for name in logger_names:
+ assert logging.getLogger(name).level == logging.WARNING
+ finally:
+ for name, level in previous_levels.items():
+ logging.getLogger(name).setLevel(level)
+
+
+def test_bootstrap_honors_explicit_log_level(monkeypatch) -> None:
+ setup_calls: list[str] = []
+ fake_settings = SimpleNamespace(LOG_LEVEL="DEBUG", model_fields_set={"LOG_LEVEL"})
+
+ monkeypatch.setattr("skyvern.config.settings", fake_settings)
+ monkeypatch.setattr("skyvern.forge.sdk.forge_log.setup_logger", lambda: setup_calls.append("called"))
+
+ logger_names = ("", "skyvern", "httpx", "litellm", "playwright", "httpcore")
+ previous_levels = {name: logging.getLogger(name).level for name in logger_names}
+ try:
+ cli_bootstrap.configure_cli_bootstrap_logging()
+ assert setup_calls == ["called"]
+ assert fake_settings.LOG_LEVEL == "DEBUG"
+ for name in logger_names:
+ assert logging.getLogger(name).level == logging.DEBUG
+ finally:
+ for name, level in previous_levels.items():
+ logging.getLogger(name).setLevel(level)
diff --git a/tests/unit/test_cli_commands.py b/tests/unit/test_cli_commands.py
index 2dc5a3e68..a3472252b 100644
--- a/tests/unit/test_cli_commands.py
+++ b/tests/unit/test_cli_commands.py
@@ -452,6 +452,32 @@ class TestWorkflowCommands:
assert parsed["ok"] is False
assert parsed["error"]["code"] == "RUN_NOT_FOUND"
+ def test_workflow_status_cli_preserves_full_detail_behavior(
+ self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
+ ) -> None:
+ from skyvern.cli import workflow as workflow_cmd
+
+ tool = AsyncMock(
+ return_value={
+ "ok": True,
+ "action": "skyvern_workflow_status",
+ "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
+ "data": {"run_id": "wr_123", "recording_url": "https://example.com/recording"},
+ "artifacts": [],
+ "timing_ms": {},
+ "warnings": [],
+ "error": None,
+ }
+ )
+ monkeypatch.setattr(workflow_cmd, "tool_workflow_status", tool)
+
+ workflow_cmd.workflow_status(run_id="wr_123", json_output=True)
+
+ assert tool.await_args.kwargs == {"run_id": "wr_123", "verbosity": "full"}
+ parsed = json.loads(capsys.readouterr().out)
+ assert parsed["ok"] is True
+ assert parsed["data"]["recording_url"] == "https://example.com/recording"
+
def test_workflow_update_missing_definition_file_raises_bad_parameter(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
diff --git a/tests/unit/test_cli_commands_logging.py b/tests/unit/test_cli_commands_logging.py
index b714838d4..2d5aef1f6 100644
--- a/tests/unit/test_cli_commands_logging.py
+++ b/tests/unit/test_cli_commands_logging.py
@@ -1,28 +1,17 @@
from __future__ import annotations
-import logging
-
import skyvern.cli.commands as cli_commands
def test_configure_cli_logging_is_idempotent(monkeypatch) -> None:
setup_calls: list[str] = []
- monkeypatch.setattr(cli_commands, "_setup_logger", lambda: setup_calls.append("called"))
+ monkeypatch.setattr(cli_commands, "_configure_cli_bootstrap_logging", lambda: setup_calls.append("called"))
monkeypatch.setattr(cli_commands, "_cli_logging_configured", False)
+ cli_commands.configure_cli_logging()
+ assert setup_calls == ["called"]
- logger_names = ("skyvern", "httpx", "litellm", "playwright", "httpcore")
- previous_levels = {name: logging.getLogger(name).level for name in logger_names}
- try:
- cli_commands.configure_cli_logging()
- assert setup_calls == ["called"]
- for name in logger_names:
- assert logging.getLogger(name).level == logging.WARNING
-
- cli_commands.configure_cli_logging()
- assert setup_calls == ["called"]
- finally:
- for name, level in previous_levels.items():
- logging.getLogger(name).setLevel(level)
+ cli_commands.configure_cli_logging()
+ assert setup_calls == ["called"]
def test_cli_callback_configures_logging(monkeypatch) -> None:
diff --git a/tests/unit/test_dom_parser_js_benchmark.py b/tests/unit/test_dom_parser_js_benchmark.py
new file mode 100644
index 000000000..ca603f5e1
--- /dev/null
+++ b/tests/unit/test_dom_parser_js_benchmark.py
@@ -0,0 +1,255 @@
+"""
+Playwright-based benchmark for JavaScript DOM parsing (domUtils.js).
+
+Tests run a real browser to measure actual buildTreeFromBody() performance
+on a synthetic HTML page with many elements.
+
+These tests require playwright to be installed with browser binaries.
+They are automatically skipped if the browser binary is missing.
+Run `playwright install chromium` to enable these tests.
+"""
+
+import pytest
+
+from skyvern.webeye.scraper.scraper import JS_FUNCTION_DEFS
+
+# Only run if playwright is available
+playwright = pytest.importorskip("playwright")
+
+
+def _launch_browser():
+ """Try to launch a Chromium browser. Returns (browser, playwright_instance) or raises."""
+ from playwright.sync_api import sync_playwright
+
+ pw = sync_playwright().start()
+ try:
+ browser = pw.chromium.launch(headless=True)
+ return browser, pw
+ except Exception:
+ pw.stop()
+ raise
+
+
[email protected](scope="module")
+def browser_context():
+ """Launch a browser for JS benchmarks. Skips if browser binary is missing."""
+ try:
+ browser, pw = _launch_browser()
+ except Exception as e:
+ pytest.skip(f"Playwright browser not available: {e}")
+
+ context = browser.new_context()
+ yield context
+ context.close()
+ browser.close()
+ pw.stop()
+
+
+def _generate_test_html(num_elements: int) -> str:
+ """Generate an HTML page with the specified number of elements."""
+ elements = []
+ for i in range(num_elements):
+ tag = ["div", "span", "button", "input", "a"][i % 5]
+ if tag == "input":
+ elements.append(
+ f'<{tag} type="text" name="field_{i}" placeholder="Field {i}" '
+ f'class="form-control" aria-label="Field {i}">'
+ )
+ elif tag == "a":
+ elements.append(
+ f'<{tag} href="https://example.com/{i}" class="link-{i}" aria-label="Link {i}">Link text {i}</{tag}>'
+ )
+ elif tag == "button":
+ elements.append(f'<{tag} type="button" class="btn btn-{i}" aria-label="Button {i}">Click {i}</{tag}>')
+ else:
+ elements.append(f'<{tag} class="element-{i}" style="cursor: pointer;">Text content {i}</{tag}>')
+
+ body_content = "\n".join(elements)
+ return f"""
+ <!DOCTYPE html>
+ <html>
+ <head><title>Benchmark Page</title></head>
+ <body>
+ <div id="main-container">
+ {body_content}
+ </div>
+ </body>
+ </html>
+ """
+
+
+class TestDomUtilsJsBenchmark:
+ """Benchmark buildTreeFromBody() in a real browser."""
+
+ @pytest.mark.parametrize("num_elements", [500, 2000, 5000])
+ def test_build_tree_performance(self, browser_context, num_elements: int):
+ page = browser_context.new_page()
+ html = _generate_test_html(num_elements)
+ page.set_content(html)
+
+ # Load domUtils.js
+ page.evaluate(JS_FUNCTION_DEFS)
+
+ # Benchmark buildTreeFromBody
+ result = page.evaluate("""
+ async () => {
+ const start = performance.now();
+ const [elements, tree] = await buildTreeFromBody('main.frame', 0);
+ const elapsed = performance.now() - start;
+ return {
+ elapsed_ms: elapsed,
+ element_count: elements.length,
+ tree_root_count: tree.length,
+ };
+ }
+ """)
+
+ elapsed_ms = result["elapsed_ms"]
+ element_count = result["element_count"]
+ tree_root_count = result["tree_root_count"]
+
+ print(f"\nbuildTreeFromBody({num_elements} DOM nodes):")
+ print(f" Time: {elapsed_ms:.1f}ms")
+ print(f" Elements found: {element_count}")
+ print(f" Tree roots: {tree_root_count}")
+
+ # Should complete within reasonable time
+ assert elapsed_ms < 10000, f"buildTreeFromBody took {elapsed_ms:.1f}ms for {num_elements} elements"
+ assert element_count > 0, "Should find at least some elements"
+
+ page.close()
+
+ def test_hover_styles_map_performance(self, browser_context):
+ """Benchmark getHoverStylesMap() with many stylesheets."""
+ page = browser_context.new_page()
+
+ # Create a page with many CSS rules
+ css_rules = "\n".join([f".element-{i}:hover {{ cursor: pointer; background: #{i:06x}; }}" for i in range(500)])
+ html = f"""
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <style>{css_rules}</style>
+ <style>
+ .non-hover-1 {{ color: red; }}
+ .non-hover-2 {{ color: blue; }}
+ </style>
+ </head>
+ <body><div>Test</div></body>
+ </html>
+ """
+ page.set_content(html)
+ page.evaluate(JS_FUNCTION_DEFS)
+
+ result = page.evaluate("""
+ async () => {
+ const start = performance.now();
+ const hoverMap = await getHoverStylesMap();
+ const elapsed = performance.now() - start;
+ return {
+ elapsed_ms: elapsed,
+ hover_selectors: hoverMap.size,
+ };
+ }
+ """)
+
+ elapsed_ms = result["elapsed_ms"]
+ hover_selectors = result["hover_selectors"]
+
+ print("\ngetHoverStylesMap (500 hover rules + 2 non-hover rules):")
+ print(f" Time: {elapsed_ms:.1f}ms")
+ print(f" Hover selectors found: {hover_selectors}")
+
+ assert elapsed_ms < 5000, f"getHoverStylesMap took {elapsed_ms:.1f}ms"
+ assert hover_selectors > 0, "Should find hover selectors"
+
+ page.close()
+
+ def test_draw_bounding_boxes_performance(self, browser_context):
+ """Benchmark drawBoundingBoxes() with visual grouping."""
+ page = browser_context.new_page()
+ html = _generate_test_html(1000)
+ page.set_content(html)
+ page.evaluate(JS_FUNCTION_DEFS)
+
+ result = page.evaluate("""
+ async () => {
+ // First build the tree (populates cache)
+ await buildTreeFromBody('main.frame', 0);
+
+ const start = performance.now();
+ await buildElementsAndDrawBoundingBoxes('main.frame', 0);
+ const elapsed = performance.now() - start;
+
+ // Count bounding boxes added
+ const container = document.querySelector('#boundingBoxContainer');
+ const boxCount = container ? container.children.length : 0;
+
+ return {
+ elapsed_ms: elapsed,
+ box_count: boxCount,
+ };
+ }
+ """)
+
+ elapsed_ms = result["elapsed_ms"]
+ box_count = result["box_count"]
+
+ print("\nbuildElementsAndDrawBoundingBoxes (1000 elements):")
+ print(f" Time: {elapsed_ms:.1f}ms")
+ print(f" Bounding boxes: {box_count}")
+
+ assert elapsed_ms < 10000, f"drawBoundingBoxes took {elapsed_ms:.1f}ms"
+
+ page.close()
+
+ def test_tree_correctness(self, browser_context):
+ """Verify the tree structure is correct after optimizations."""
+ page = browser_context.new_page()
+ html = """
+ <!DOCTYPE html>
+ <html>
+ <body>
+ <div id="parent">
+ <button id="btn1">Click me</button>
+ <input type="text" name="field1" placeholder="Enter text">
+ <a href="https://example.com">Link</a>
+ <select name="dropdown">
+ <option value="a">Option A</option>
+ <option value="b">Option B</option>
+ </select>
+ </div>
+ </body>
+ </html>
+ """
+ page.set_content(html)
+ page.evaluate(JS_FUNCTION_DEFS)
+
+ result = page.evaluate("""
+ async () => {
+ const [elements, tree] = await buildTreeFromBody('main.frame', 0);
+ return {
+ element_count: elements.length,
+ tree_root_count: tree.length,
+ // Check that interactable elements were found
+ interactable_count: elements.filter(e => e.interactable).length,
+ // Check a button was found
+ has_button: elements.some(e => e.tagName === 'button'),
+ // Check an input was found
+ has_input: elements.some(e => e.tagName === 'input'),
+ // Check a link was found
+ has_link: elements.some(e => e.tagName === 'a'),
+ // Check select was found with options
+ has_select: elements.some(e => e.tagName === 'select' && e.options && e.options.length === 2),
+ };
+ }
+ """)
+
+ assert result["element_count"] > 0
+ assert result["interactable_count"] >= 4 # button, input, a, select
+ assert result["has_button"]
+ assert result["has_input"]
+ assert result["has_link"]
+ assert result["has_select"]
+
+ page.close()
diff --git a/tests/unit/test_dom_parser_performance.py b/tests/unit/test_dom_parser_performance.py
new file mode 100644
index 000000000..24a5c0534
--- /dev/null
+++ b/tests/unit/test_dom_parser_performance.py
@@ -0,0 +1,405 @@
+"""
+Benchmark and regression tests for DOM parser pipeline performance.
+
+Tests the Python-side processing: trim_element_tree, _filter_attributes,
+json_to_html, and cleanup traversal patterns.
+"""
+
+import copy
+import time
+from collections import deque
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.scraper.scraped_page import ScrapedPage, json_to_html
+from skyvern.webeye.scraper.scraper import (
+ _filter_attributes,
+ _trimmed_attributes,
+ _trimmed_base64_data,
+ build_element_dict,
+ trim_element,
+ trim_element_tree,
+)
+
+
[email protected](autouse=True)
+def _setup_skyvern_context():
+ """Ensure a SkyvernContext exists for tests that call json_to_html."""
+ ctx = SkyvernContext()
+ skyvern_context.set(ctx)
+ yield
+ skyvern_context.reset()
+
+
+def _make_element(
+ element_id: str,
+ tag: str = "div",
+ interactable: bool = True,
+ text: str = "sample text",
+ num_attrs: int = 10,
+ children: list | None = None,
+) -> dict:
+ """Generate a realistic element dict for testing."""
+ attrs = {
+ "class": f"cls-{element_id}",
+ "id": f"html-id-{element_id}",
+ "data-testid": f"test-{element_id}",
+ "style": "display: flex; align-items: center;",
+ "aria-label": f"Element {element_id}",
+ "role": "button",
+ "type": "button",
+ "name": f"name-{element_id}",
+ "value": f"val-{element_id}",
+ "placeholder": "Enter value...",
+ }
+ # Add extra non-reserved attributes to test filtering
+ for i in range(max(0, num_attrs - len(attrs))):
+ attrs[f"data-extra-{i}"] = f"extra-value-{i}"
+
+ return {
+ "id": element_id,
+ "tagName": tag,
+ "interactable": interactable,
+ "text": text,
+ "attributes": attrs,
+ "children": children or [],
+ "frame": "main.frame",
+ "frame_index": 0,
+ "keepAllAttr": False,
+ "beforePseudoText": "",
+ "afterPseudoText": "",
+ "purgeable": False,
+ "rect": {"top": 0, "left": 0, "bottom": 100, "right": 200, "width": 200, "height": 100},
+ }
+
+
+def _make_element_tree(num_elements: int, depth: int = 3) -> list[dict]:
+ """Generate a tree of elements for benchmarking."""
+ element_counter = 0
+ children_per_node = max(1, num_elements // (depth + 1))
+
+ def build_level(current_depth: int, remaining: int) -> list[dict]:
+ nonlocal element_counter
+ level_elements = []
+ while remaining > 0 and element_counter < num_elements:
+ element_counter += 1
+ children = []
+ if current_depth < depth and remaining > 1:
+ child_count = min(children_per_node, remaining - 1)
+ children = build_level(current_depth + 1, child_count)
+ remaining -= len(children)
+ el = _make_element(
+ element_id=f"el_{element_counter:04d}",
+ tag=["div", "span", "button", "input", "a"][element_counter % 5],
+ interactable=element_counter % 3 == 0,
+ text=f"Text content for element {element_counter}" if element_counter % 2 == 0 else "",
+ children=children,
+ )
+ level_elements.append(el)
+ remaining -= 1
+ return level_elements
+
+ return build_level(0, num_elements)
+
+
+class TestFilterAttributesMerged:
+ """Test that the new merged _filter_attributes produces the same output as the two-pass approach."""
+
+ def test_basic_whitelist(self):
+ attrs = {"class": "foo", "aria-label": "bar", "name": "baz", "data-x": "y"}
+ result = _filter_attributes(attrs, keep_all=False)
+ assert "aria-label" in result
+ assert "name" in result
+ assert "class" not in result
+ assert "data-x" not in result
+
+ def test_base64_removal(self):
+ attrs = {"href": "data:image/png;base64,abc123", "name": "test", "src": "data:text/html;base64,xyz"}
+ result = _filter_attributes(attrs, keep_all=False)
+ assert "href" not in result
+ assert "src" not in result
+ assert result["name"] == "test"
+
+ def test_keep_all_attr(self):
+ attrs = {"class": "foo", "aria-label": "bar", "data-x": "y"}
+ result = _filter_attributes(attrs, keep_all=True)
+ # keepAllAttr=True should keep everything except base64
+ assert "class" in result
+ assert "data-x" in result
+
+ def test_role_listbox_option(self):
+ attrs = {"role": "listbox", "class": "foo"}
+ result = _filter_attributes(attrs, keep_all=False)
+ assert result["role"] == "listbox"
+ assert "class" not in result
+
+ def test_name_truncation_in_filter(self):
+ """Name truncation should happen inside _filter_attributes (single pass)."""
+ attrs = {"name": "x" * 1000, "aria-label": "test"}
+ result = _filter_attributes(attrs, keep_all=False)
+ assert len(result["name"]) == 500
+ assert result["aria-label"] == "test"
+
+ def test_equivalence_with_old_approach(self):
+ """The new _filter_attributes should produce the same result as the old two-pass approach."""
+ attrs = {
+ "class": "foo",
+ "aria-label": "bar",
+ "name": "baz",
+ "href": "data:image/png;base64,abc",
+ "src": "https://example.com/image.png",
+ "role": "option",
+ "data-x": "y",
+ "type": "button",
+ }
+ # Old approach: two passes
+ old_pass1 = _trimmed_base64_data(attrs)
+ old_result = _trimmed_attributes(old_pass1)
+
+ # New approach: single pass
+ new_result = _filter_attributes(attrs, keep_all=False)
+
+ assert old_result == new_result
+
+
+class TestTrimElement:
+ """Test trim_element correctness after optimization."""
+
+ def test_removes_frame_fields(self):
+ el = _make_element("test1")
+ trim_element(el)
+ assert "frame" not in el
+ assert "frame_index" not in el
+
+ def test_removes_keep_all_attr(self):
+ el = _make_element("test2")
+ trim_element(el)
+ assert "keepAllAttr" not in el
+
+ def test_removes_empty_text(self):
+ el = _make_element("test3", text="")
+ trim_element(el)
+ assert "text" not in el
+
+ def test_removes_empty_pseudo_text(self):
+ el = _make_element("test4")
+ el["beforePseudoText"] = ""
+ el["afterPseudoText"] = ""
+ trim_element(el)
+ assert "beforePseudoText" not in el
+ assert "afterPseudoText" not in el
+
+ def test_keeps_interactable_id(self):
+ el = _make_element("test5", interactable=True)
+ trim_element(el)
+ assert "id" in el
+
+ def test_removes_non_interactable_id(self):
+ el = _make_element("test6", interactable=False)
+ el["attributes"].pop("disabled", None)
+ el["attributes"].pop("aria-disabled", None)
+ el["attributes"].pop("readonly", None)
+ el["attributes"].pop("aria-readonly", None)
+ el.pop("hoverOnly", None)
+ trim_element(el)
+ assert "id" not in el
+
+ def test_filters_attributes(self):
+ el = _make_element("test7")
+ trim_element(el)
+ attrs = el.get("attributes", {})
+ # Non-reserved attributes should be removed
+ assert "class" not in attrs
+ assert "data-testid" not in attrs
+ assert "style" not in attrs
+ # Reserved attributes should remain
+ assert "aria-label" in attrs
+ assert "type" in attrs
+
+ def test_truncates_long_name(self):
+ el = _make_element("test8")
+ el["attributes"]["name"] = "x" * 1000
+ trim_element(el)
+ assert len(el["attributes"]["name"]) == 500
+
+ def test_processes_children(self):
+ child = _make_element("child1")
+ parent = _make_element("parent1", children=[child])
+ trim_element(parent)
+ assert "frame" not in child
+ assert "keepAllAttr" not in child
+
+
+class TestTrimElementTreePerformance:
+ """Benchmark tests for trim_element_tree at various scales."""
+
+ @pytest.mark.parametrize("num_elements", [100, 1000, 5000])
+ def test_trim_performance(self, num_elements: int):
+ tree = _make_element_tree(num_elements)
+ tree_copy = copy.deepcopy(tree)
+
+ start = time.perf_counter()
+ trim_element_tree(tree_copy)
+ elapsed = time.perf_counter() - start
+
+ # Log timing for visibility
+ print(f"\ntrim_element_tree({num_elements} elements): {elapsed:.4f}s")
+ # Should complete in reasonable time (< 1s for 5000 elements)
+ assert elapsed < 2.0, f"trim_element_tree took too long: {elapsed:.4f}s for {num_elements} elements"
+
+
+class TestJsonToHtmlPerformance:
+ """Benchmark json_to_html at various scales."""
+
+ @pytest.mark.parametrize("num_elements", [100, 1000, 5000])
+ def test_json_to_html_performance(self, num_elements: int):
+ tree = _make_element_tree(num_elements)
+ # Trim first (like the real pipeline)
+ tree = trim_element_tree(copy.deepcopy(tree))
+
+ start = time.perf_counter()
+ result = "".join(json_to_html(element) for element in tree)
+ elapsed = time.perf_counter() - start
+
+ print(f"\njson_to_html({num_elements} elements): {elapsed:.4f}s, output: {len(result)} chars")
+ assert elapsed < 2.0, f"json_to_html took too long: {elapsed:.4f}s for {num_elements} elements"
+ assert len(result) > 0
+
+ def test_json_to_html_correctness(self):
+ """Verify basic HTML output structure."""
+ el = {
+ "tagName": "button",
+ "id": "btn1",
+ "interactable": True,
+ "text": "Click me",
+ "attributes": {"type": "submit", "aria-label": "Submit"},
+ "children": [],
+ }
+ html = json_to_html(el)
+ assert "<button" in html
+ assert "Click me" in html
+ assert 'type="submit"' in html
+ assert "</button>" in html
+
+
+class TestDequeVsList:
+ """Verify deque.popleft() is faster than list.pop(0) for BFS."""
+
+ def test_deque_faster_than_list(self):
+ n = 10000
+ items = list(range(n))
+
+ # List pop(0)
+ lst = list(items)
+ start = time.perf_counter()
+ while lst:
+ lst.pop(0)
+ list_time = time.perf_counter() - start
+
+ # Deque popleft
+ dq = deque(items)
+ start = time.perf_counter()
+ while dq:
+ dq.popleft()
+ deque_time = time.perf_counter() - start
+
+ print(f"\nlist.pop(0): {list_time:.6f}s, deque.popleft(): {deque_time:.6f}s")
+ # deque should be significantly faster
+ assert deque_time < list_time
+
+
+class TestBuildElementDict:
+ """Test build_element_dict correctness and hash collision handling."""
+
+ def test_basic_dict_building(self):
+ elements = [
+ {"id": "e1", "tagName": "button", "frame": "main", "attributes": {"type": "submit"}},
+ {"id": "e2", "tagName": "input", "frame": "main", "attributes": {"name": "email"}},
+ ]
+ css_dict, elem_dict, frame_dict, hash_dict, hash_to_ids = build_element_dict(elements)
+
+ assert "e1" in css_dict
+ assert "e2" in css_dict
+ assert elem_dict["e1"]["tagName"] == "button"
+ assert frame_dict["e1"] == "main"
+ assert "e1" in hash_dict
+ assert "e2" in hash_dict
+
+ def test_hash_collision_uses_append(self):
+ """Verify that hash_to_element_ids uses list append (not concat) for collisions."""
+ # Two identical elements (same tag, same attrs) will have the same hash
+ el1 = {"id": "e1", "tagName": "div", "frame": "main", "attributes": {}}
+ el2 = {"id": "e2", "tagName": "div", "frame": "main", "attributes": {}}
+ _, _, _, hash_dict, hash_to_ids = build_element_dict([el1, el2])
+
+ h1 = hash_dict["e1"]
+ h2 = hash_dict["e2"]
+ assert h1 == h2, "Identical elements should produce the same hash"
+ assert hash_to_ids[h1] == ["e1", "e2"]
+
+
+class TestEconomyTreeProcessing:
+ """Test the economy tree SVG filtering logic."""
+
+ def test_filters_svg_root(self):
+ svg_el = {"tagName": "svg", "children": [{"tagName": "path"}]}
+ result = ScrapedPage._process_element_for_economy_tree(svg_el)
+ assert result is None
+
+ def test_filters_svg_children(self):
+ tree = {
+ "tagName": "div",
+ "children": [
+ {"tagName": "button", "children": []},
+ {"tagName": "svg", "children": [{"tagName": "path"}]},
+ {"tagName": "span", "children": []},
+ ],
+ }
+ result = ScrapedPage._process_element_for_economy_tree(tree)
+ assert result is not None
+ children = result["children"]
+ assert len(children) == 2
+ assert children[0]["tagName"] == "button"
+ assert children[1]["tagName"] == "span"
+
+ def test_filters_nested_svg(self):
+ tree = {
+ "tagName": "div",
+ "children": [
+ {
+ "tagName": "span",
+ "children": [
+ {"tagName": "svg", "children": []},
+ {"tagName": "a", "children": []},
+ ],
+ },
+ ],
+ }
+ result = ScrapedPage._process_element_for_economy_tree(tree)
+ assert result is not None
+ inner = result["children"][0]
+ assert len(inner["children"]) == 1
+ assert inner["children"][0]["tagName"] == "a"
+
+ def test_filters_svg_case_insensitive(self):
+ """SVG filtering is case-insensitive via .lower()."""
+ tree = {
+ "tagName": "div",
+ "children": [
+ {"tagName": "SVG", "children": []},
+ {"tagName": "Svg", "children": []},
+ {"tagName": "button", "children": []},
+ ],
+ }
+ result = ScrapedPage._process_element_for_economy_tree(tree)
+ assert result is not None
+ assert len(result["children"]) == 1
+ assert result["children"][0]["tagName"] == "button"
+
+ def test_no_children(self):
+ el = {"tagName": "input"}
+ result = ScrapedPage._process_element_for_economy_tree(el)
+ assert result is not None
+ assert result["tagName"] == "input"
diff --git a/tests/unit/test_mcp_concise_responses.py b/tests/unit/test_mcp_concise_responses.py
index 4522cf07a..48535fb0f 100644
--- a/tests/unit/test_mcp_concise_responses.py
+++ b/tests/unit/test_mcp_concise_responses.py
@@ -2,10 +2,12 @@
from __future__ import annotations
+import json
from collections.abc import Iterator
import pytest
+import skyvern.cli.mcp_tools.workflow as workflow_tools
from skyvern.cli.core.result import Artifact, BrowserContext, make_result, set_concise_responses
@@ -183,3 +185,70 @@ def test_verbose_returns_all_fields() -> None:
assert result["data"]["resolved_selector"] is not None
assert result["artifacts"] == []
assert result["warnings"] == []
+
+
+def test_workflow_status_summary_stays_bounded_for_heavy_payload() -> None:
+ long_url = "https://artifacts.skyvern.example/" + ("x" * 1450)
+ heavy_run = {
+ "workflow_run_id": "wr_heavy",
+ "status": "terminated",
+ "failure_reason": "Execution terminated after repeated navigation failures",
+ "workflow_title": "Heavy workflow",
+ "recording_url": long_url,
+ "screenshot_urls": [f"{long_url}-{idx}" for idx in range(6)],
+ "downloaded_files": [{"url": f"{long_url}-download", "filename": "case-export.csv"}],
+ "outputs": {
+ "collect_customer_data": {
+ "task_screenshot_artifact_ids": [f"art_task_{idx}" for idx in range(12)],
+ "workflow_screenshot_artifact_ids": [f"art_workflow_{idx}" for idx in range(12)],
+ "task_screenshots": [f"{long_url}-task-{idx}" for idx in range(4)],
+ "workflow_screenshots": [f"{long_url}-workflow-{idx}" for idx in range(4)],
+ "extracted_information": [{"account_id": "acct_123", "status": "terminated"}],
+ },
+ "submit_case": [
+ {
+ "task_screenshot_artifact_ids": [f"art_nested_{idx}" for idx in range(6)],
+ "workflow_screenshot_artifact_ids": [f"art_followup_{idx}" for idx in range(6)],
+ "task_screenshots": [f"{long_url}-nested-task-{idx}" for idx in range(3)],
+ "workflow_screenshots": [f"{long_url}-nested-workflow-{idx}" for idx in range(4)],
+ }
+ ],
+ "extracted_information": [{"duplicated_rollup": True}],
+ },
+ "run_with": "code",
+ }
+
+ full_payload = workflow_tools._serialize_run_full(heavy_run)
+ summary_payload = workflow_tools._serialize_run_summary(heavy_run)
+
+ assert len(json.dumps(full_payload)) > 20_000
+ assert len(json.dumps(summary_payload)) < 2_000
+
+ result = make_result("skyvern_workflow_status", data=summary_payload)
+ assert len(json.dumps(result)) < 2_200
+ assert "recording_url" not in result["data"]
+ assert "output" not in result["data"]
+ assert result["data"]["artifact_summary"]["artifact_id_count"] == 36
+
+
+def test_workflow_status_summary_shrinks_simple_payload() -> None:
+ long_url = "https://artifacts.skyvern.example/" + ("y" * 1450)
+ simple_run = {
+ "workflow_run_id": "wr_simple",
+ "status": "completed",
+ "workflow_title": "Simple workflow",
+ "recording_url": long_url,
+ "screenshot_urls": [f"{long_url}-shot"],
+ "outputs": {
+ "result": "success",
+ "order_id": "ord_123",
+ },
+ "run_with": "code",
+ }
+
+ full_payload = workflow_tools._serialize_run_full(simple_run)
+ summary_payload = workflow_tools._serialize_run_summary(simple_run)
+
+ assert len(json.dumps(full_payload)) > 1_500
+ assert len(json.dumps(summary_payload)) < 800
+ assert summary_payload["output_summary"]["scalar_preview"] == {"result": "success", "order_id": "ord_123"}
diff --git a/tests/unit/test_mcp_workflow_tools.py b/tests/unit/test_mcp_workflow_tools.py
index 2ec94b0c3..50db933b7 100644
--- a/tests/unit/test_mcp_workflow_tools.py
+++ b/tests/unit/test_mcp_workflow_tools.py
@@ -6,8 +6,10 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
+from fastmcp import Client
import skyvern.cli.mcp_tools.workflow as workflow_tools
+from skyvern.cli.mcp_tools import mcp
def _fake_workflow_response() -> SimpleNamespace:
@@ -26,6 +28,64 @@ def _fake_workflow_response() -> SimpleNamespace:
)
+def _fake_http_response(payload: dict[str, object], status_code: int = 200) -> SimpleNamespace:
+ return SimpleNamespace(
+ status_code=status_code,
+ json=lambda: payload,
+ text=json.dumps(payload),
+ )
+
+
+def _heavy_workflow_run_payload(*, include_expanded_outputs: bool = True) -> dict[str, object]:
+ long_url = "https://artifacts.skyvern.example/" + ("x" * 1450)
+ screenshot_urls = [f"{long_url}-{idx}" for idx in range(6)]
+ task_artifact_ids = [f"art_task_{idx}" for idx in range(12)]
+ workflow_artifact_ids = [f"art_workflow_{idx}" for idx in range(12)]
+ outputs: dict[str, object] = {
+ "collect_customer_data": {
+ "task_screenshot_artifact_ids": task_artifact_ids,
+ "workflow_screenshot_artifact_ids": workflow_artifact_ids,
+ "extracted_information": [{"account_id": "acct_123", "status": "terminated"}],
+ "status": "terminated",
+ },
+ "submit_case": [
+ {
+ "workflow_screenshot_artifact_ids": [f"art_followup_{idx}" for idx in range(6)],
+ "task_screenshot_artifact_ids": [f"art_nested_{idx}" for idx in range(6)],
+ "result": "retry_required",
+ }
+ ],
+ }
+ if include_expanded_outputs:
+ outputs["collect_customer_data"] |= {
+ "task_screenshots": screenshot_urls[:4],
+ "workflow_screenshots": screenshot_urls[2:6],
+ }
+ outputs["submit_case"][0] |= {
+ "task_screenshots": screenshot_urls[:3],
+ "workflow_screenshots": screenshot_urls[1:5],
+ }
+ outputs["extracted_information"] = [{"duplicated_rollup": True}]
+
+ return {
+ "workflow_id": "wpid_heavy",
+ "workflow_run_id": "wr_heavy",
+ "status": "terminated",
+ "failure_reason": "Execution terminated after repeated navigation failures",
+ "workflow_title": "Heavy workflow",
+ "recording_url": long_url,
+ "screenshot_urls": screenshot_urls,
+ "downloaded_files": [
+ {
+ "url": f"{long_url}-download",
+ "filename": "case-export.csv",
+ }
+ ],
+ "outputs": outputs,
+ "run_with": "code",
+ }
+
+
@pytest.mark.asyncio
async def test_workflow_create_normalizes_invalid_text_prompt_llm_key(monkeypatch: pytest.MonkeyPatch) -> None:
fake_client = SimpleNamespace(create_workflow=AsyncMock(return_value=_fake_workflow_response()))
@@ -439,3 +499,134 @@ async def test_mcp_text_prompt_without_llm_key_stays_null(monkeypatch: pytest.Mo
assert result["ok"] is True
assert sent_block.llm_key is None
assert sent_block.model is None
+
+
+async def test_workflow_status_uses_workflow_run_route_for_wr_ids(monkeypatch: pytest.MonkeyPatch) -> None:
+ payload = _heavy_workflow_run_payload(include_expanded_outputs=False)
+ request = AsyncMock(return_value=_fake_http_response(payload))
+ fake_client = SimpleNamespace(
+ get_run=AsyncMock(),
+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),
+ )
+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)
+
+ result = await workflow_tools.skyvern_workflow_status(run_id="wr_heavy")
+
+ assert result["ok"] is True
+ request.assert_awaited_once_with(
+ "api/v1/workflows/runs/wr_heavy",
+ method="GET",
+ params={"include_output_details": False},
+ )
+ fake_client.get_run.assert_not_awaited()
+ data = result["data"]
+ assert data["run_id"] == "wr_heavy"
+ assert data["run_type"] == "workflow_run"
+ assert "recording_url" not in data
+ assert "output" not in data
+ assert data["artifact_summary"]["recording_available"] is True
+ assert data["artifact_summary"]["artifact_id_count"] == 36
+ assert data["output_summary"]["nested_screenshot_count"] == 0
+ assert data["output_summary"]["has_extracted_information"] is True
+
+
+async def test_workflow_status_full_preserves_expanded_workflow_details(monkeypatch: pytest.MonkeyPatch) -> None:
+ payload = _heavy_workflow_run_payload(include_expanded_outputs=True)
+ request = AsyncMock(return_value=_fake_http_response(payload))
+ fake_client = SimpleNamespace(
+ get_run=AsyncMock(),
+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),
+ )
+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)
+
+ result = await workflow_tools.skyvern_workflow_status(run_id="wr_heavy", verbosity="full")
+
+ assert result["ok"] is True
+ request.assert_awaited_once_with(
+ "api/v1/workflows/runs/wr_heavy",
+ method="GET",
+ params={"include_output_details": True},
+ )
+ data = result["data"]
+ assert data["run_id"] == "wr_heavy"
+ assert data["recording_url"] == payload["recording_url"]
+ assert data["output"] == payload["outputs"]
+ assert data["workflow_title"] == "Heavy workflow"
+
+
+async def test_workflow_status_task_runs_still_use_get_run(monkeypatch: pytest.MonkeyPatch) -> None:
+ task_run = SimpleNamespace(
+ run_id="tsk_v2_123",
+ status="completed",
+ run_type="task_v2",
+ output={"answer": "42"},
+ failure_reason=None,
+ step_count=4,
+ recording_url=None,
+ app_url=None,
+ browser_session_id=None,
+ run_with=None,
+ created_at=None,
+ modified_at=None,
+ started_at=None,
+ finished_at=None,
+ queued_at=None,
+ )
+ fake_client = SimpleNamespace(get_run=AsyncMock(return_value=task_run))
+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)
+
+ result = await workflow_tools.skyvern_workflow_status(run_id="tsk_v2_123")
+
+ fake_client.get_run.assert_awaited_once_with("tsk_v2_123")
+ data = result["data"]
+ assert data["run_id"] == "tsk_v2_123"
+ assert data["step_count"] == 4
+ assert data["output_summary"]["scalar_preview"] == {"answer": "42"}
+
+
+async def test_workflow_status_summary_via_mcp_client(monkeypatch: pytest.MonkeyPatch) -> None:
+ payload = _heavy_workflow_run_payload(include_expanded_outputs=False)
+ request = AsyncMock(return_value=_fake_http_response(payload))
+ fake_client = SimpleNamespace(
+ get_run=AsyncMock(),
+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),
+ )
+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)
+
+ async with Client(mcp) as client:
+ result = await client.call_tool("skyvern_workflow_status", {"run_id": "wr_heavy"})
+
+ assert result.is_error is False
+ assert isinstance(result.data, dict)
+ assert result.data["ok"] is True
+ data = result.data["data"]
+ assert data["run_id"] == "wr_heavy"
+ assert data["artifact_summary"]["artifact_id_count"] == 36
+ assert "recording_url" not in data
+ assert "output" not in data
+
+
+async def test_workflow_status_full_via_mcp_client(monkeypatch: pytest.MonkeyPatch) -> None:
+ payload = _heavy_workflow_run_payload(include_expanded_outputs=True)
+ request = AsyncMock(return_value=_fake_http_response(payload))
+ fake_client = SimpleNamespace(
+ get_run=AsyncMock(),
+ _client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request)),
+ )
+ monkeypatch.setattr(workflow_tools, "get_skyvern", lambda: fake_client)
+
+ async with Client(mcp) as client:
+ result = await client.call_tool("skyvern_workflow_status", {"run_id": "wr_heavy", "verbosity": "full"})
+
+ assert result.is_error is False
+ assert isinstance(result.data, dict)
+ assert result.data["ok"] is True
+ data = result.data["data"]
+ assert data["run_id"] == "wr_heavy"
+ assert data["recording_url"] == payload["recording_url"]
+ assert data["output"] == payload["outputs"]