Skip to content

Automate browser based workflows with AI

fix(SKY-11499): keep horizontally scrolled-off cells in element tree (#6893)

LawyZheng4d14cb17b9b1
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 92e15650f..40df1480e 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -449,6 +449,28 @@ function isHoverOnlyElement(element) {
return false;
}
+// Walks ancestors looking for one whose own scrollLeft is non-zero AND that
+// is itself a horizontal scroll container (overflow-x: auto or scroll). Used
+// by isElementVisible to keep cells of horizontally scrolled data grids in
+// the element tree even when the current scroll position has pushed them
+// outside the viewport horizontally.
+function hasHorizontallyScrolledAncestor(element) {
+ let parent = element.parentElement;
+ while (parent) {
+ if (parent.scrollLeft && parent.scrollLeft !== 0) {
+ const parentStyle = getElementComputedStyle(parent);
+ if (parentStyle) {
+ const overflowX = parentStyle.overflowX;
+ if (overflowX === "auto" || overflowX === "scroll") {
+ return true;
+ }
+ }
+ }
+ parent = parent.parentElement;
+ }
+ return false;
+}
+
// from playwright: https://github.com/microsoft/playwright/blob/1b65f26f0287c0352e76673bc5f85bc36c934b55/packages/playwright-core/src/server/injected/domUtils.ts#L100-L119
// NOTE: According this logic, some elements with aria-hidden won't be considered as invisible. And the result shows they are indeed interactable.
function isElementVisible(element, seen) {
@@ -554,6 +576,15 @@ function isElementVisible(element, seen) {
// FIXME: sometimes there could be an overflow element blocking the default scrolling, making Y coordinate be wrong. So we currently only check for X
const center_x = (rect.left + rect.width) / 2 + window.scrollX;
if (center_x < 0) {
+ // The element's center lies left of the viewport. If it sits inside a
+ // horizontally scrolled container, it is still part of the page — just
+ // outside the current scroll window. Keeping it in the element tree
+ // lets the LLM see columns that the user could reveal by scrolling. We
+ // still drop elements without a scrolled ancestor (e.g. accessibility
+ // skip links positioned at left: -9999px).
+ if (hasHorizontallyScrolledAncestor(element)) {
+ return true;
+ }
return false;
}
// const center_y = (rect.top + rect.height) / 2 + window.scrollY;