Skip to content

Automate browser based workflows with AI

Task V3 observe: stop listing an unnamed control no scroll can reach, on a per-run arm (SKY-16501) (#8605)

pedrohsdb55702a579ade
diff --git a/skyvern/config.py b/skyvern/config.py
index 8a2a344ce..1dfbf69e1 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -552,6 +552,9 @@ class Settings(BaseSettings):
# actuation without the probe realm gives working actions whose readbacks answer about the main
# document instead of the element's own -- a verdict reported without being measured.
TASK_V3_FRAME_PERCEPTION: bool = False
+ # Force-on term only: the arm is randomized per run by the flag of the same name and read through
+ # run_arm_enabled() in skyvern/forge/taskv3/run_arms.py (SKY-16501).
+ TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED: bool = False
# Which browser surface the v3 loop offers: today's action tools ("off"), those plus a code
# tool ("add"), or the code tool instead of them ("replace"). Three states rather than a boolean
# because the benchmark separated add from replace on speed alone, not on success. The code tool
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 9f3caf936..9b92d07a9 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -180,6 +180,7 @@ from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, Wo
from skyvern.forge.taskv3.frame_perception import frame_perception_enabled, resolve_frame_perception
from skyvern.forge.taskv3.loop import LoopOutcome, RoundAction
from skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot
+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, resolve_run_arm
from skyvern.forge.taskv3.target_label import compose_target_intention
from skyvern.forge.validation_evidence_router import (
ValidationRouterMode,
@@ -1851,6 +1852,13 @@ class ForgeAgent:
distinct_id=task.workflow_run_id or task.task_id,
organization_id=task.organization_id,
)
+ await resolve_run_arm(
+ context,
+ OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,
+ distinct_id=task.workflow_run_id or task.task_id,
+ organization_id=task.organization_id,
+ forced=settings.TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED,
+ )
offer_error_codes = False
if task.error_code_mapping:
try:
diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.py
index df095fdde..5a6c0544b 100644
--- a/skyvern/forge/sdk/core/skyvern_context.py
+++ b/skyvern/forge/sdk/core/skyvern_context.py
@@ -9,7 +9,7 @@ from contextlib import contextmanager
from contextvars import ContextVar, Token
from dataclasses import dataclass, field
from enum import StrEnum
-from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypedDict
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Literal, TypedDict
from zoneinfo import ZoneInfo
import structlog
@@ -239,6 +239,9 @@ class DialogEntry(TypedDict):
count: int
+RunArm = Literal["treatment", "control", "unrandomized"]
+
+
class EnrichTreeMode(StrEnum):
CONTROL = "control"
ENRICHED_TREE = "enriched_tree"
@@ -418,6 +421,10 @@ class SkyvernContext:
frame_perception_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
enrich_tree_mode: EnrichTreeMode = EnrichTreeMode.CONTROL
step_retry_index: int = 0
+ # Task V3 run arms by flag: (run id the arm was resolved for, arm). Same pin contract as frame perception
+ # above; written and read only through skyvern.forge.taskv3.run_arms.
+ run_arms: dict[str, tuple[str, RunArm]] = field(default_factory=dict)
+ run_arms_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
# Run-level SLIM_LLM_OUTPUT_PROMPTS assignment, resolved once by slim_llm_output.
# The lock makes first-use resolution single-flight under parallel prompt builds.
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
new file mode 100644
index 000000000..8e4bdba63
--- /dev/null
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -0,0 +1,86 @@
+"""Per-run arms for Task V3 experiments that change what one run does (SKY-16501).
+
+Each flag is multivariate so that "randomized to control" and "never randomized" stay apart: a run outside
+the rollout, or with no flag at all, gets no variant, and an analyst reading the arm log must not pool it
+with control. An env setting per flag can only force the arm on.
+"""
+
+from __future__ import annotations
+
+import structlog
+
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import RunArm
+from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider
+
+LOG = structlog.get_logger()
+
+OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG = "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED"
+
+
+def _pinned_arm(context: skyvern_context.SkyvernContext, flag: str, distinct_id: str) -> RunArm | None:
+ pin = context.run_arms.get(flag)
+ return pin[1] if pin is not None and pin[0] == distinct_id else None
+
+
+async def resolve_run_arm(
+ context: skyvern_context.SkyvernContext,
+ flag: str,
+ *,
+ distinct_id: str,
+ organization_id: str | None,
+ forced: bool,
+) -> RunArm:
+ """Resolve ``flag`` once per run and pin it on the context; every failure resolves unrandomized (off)."""
+ arm = _pinned_arm(context, flag, distinct_id)
+ if arm is not None:
+ return arm
+ async with context.run_arms_lock:
+ arm = _pinned_arm(context, flag, distinct_id)
+ if arm is not None:
+ return arm
+ variant: str | None = None
+ try:
+ provider = app.EXPERIMENTATION_PROVIDER
+ if not isinstance(provider, NoOpExperimentationProvider):
+ variant = await provider.get_value_cached(
+ flag,
+ distinct_id,
+ properties={"organization_id": organization_id},
+ )
+ except Exception:
+ LOG.warning("Failed to resolve Task V3 run arm", flag=flag, exc_info=True)
+ variant = None
+ if variant == "treatment":
+ arm = "treatment"
+ elif variant == "control":
+ arm = "control"
+ else:
+ # An unknown variant string is a misconfigured flag, not a randomized control run.
+ arm = "unrandomized"
+ # Rebound rather than mutated, so a shallow copy of the context keeps the pins it was copied with.
+ context.run_arms = {**context.run_arms, flag: (distinct_id, arm)}
+ LOG.info(
+ "Resolved Task V3 run arm",
+ flag=flag,
+ distinct_id=distinct_id,
+ organization_id=organization_id,
+ arm=arm,
+ variant=variant,
+ # The arm is the randomization; this is what the run did, which an env force can override.
+ enabled=forced or arm == "treatment",
+ forced_by_env=forced,
+ )
+ return arm
+
+
+def run_arm_enabled(flag: str, forced: bool) -> bool:
+ """Whether the current run is on for ``flag``. Sound only inside ``_execute_task_v3``, after resolution."""
+ if forced:
+ return True
+ context = skyvern_context.current()
+ if context is None:
+ return False
+ pin = context.run_arms.get(flag)
+ return pin is not None and pin[1] == "treatment"
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 527e5f0e0..ef3d1bcd8 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -67,6 +67,7 @@ from skyvern.forge.taskv3.loop import (
set_driver_timeout_predicate,
)
from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_action
+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, run_arm_enabled
from skyvern.forge.taskv3.target_label import TARGET_KIND_TOKENS, TARGET_NAME_CAP
from skyvern.webeye.actions.key_names import normalize_key_chord
from skyvern.webeye.browser_driver_errors import is_driver_timeout_error
@@ -5177,6 +5178,7 @@ async () => {
// Field text is retained at this width and masked, then capped for display, in Python. Substituted
// per call from the payload refs: any minted URL that starts inside a display window fits whole.
const _RETAIN_WIDTH = __OBSERVE_RETAIN_WIDTH__;
+ const _DROP_OFFVIEWPORT_UNNAMED = __OBSERVE_DROP_OFFVIEWPORT_UNNAMED__;
const _GROUP_TEXT_TOTAL_CAP = """
+ str(OBSERVE_GROUP_TEXT_TOTAL_CAP)
+ r""";
@@ -5197,6 +5199,14 @@ async () => {
};
const _parentOf = _getter(Node.prototype, 'parentElement');
const _scrollLeftOf = _getter(Element.prototype, 'scrollLeft');
+ const _assignedSlotOf = _getter(Element.prototype, 'assignedSlot');
+ const _tagNameOf = _getter(Element.prototype, 'tagName');
+ const _localNameOf = _getter(Element.prototype, 'localName');
+ const _shadowRootOf = _getter(Element.prototype, 'shadowRoot');
+ const _scrollHeightOf = _getter(Element.prototype, 'scrollHeight');
+ const _scrollWidthOf = _getter(Element.prototype, 'scrollWidth');
+ const _clientHeightOf = _getter(Element.prototype, 'clientHeight');
+ const _clientWidthOf = _getter(Element.prototype, 'clientWidth');
const _prevOf = _getter(Node.prototype, 'previousSibling');
const _nextOf = _getter(Node.prototype, 'nextSibling');
const _firstChildOf = _getter(Node.prototype, 'firstChild');
@@ -6041,6 +6051,9 @@ async () => {
let hiddenDroppedOffCanvas = 0;
let hiddenDroppedVisibility = 0;
let hiddenDroppedZeroRect = 0;
+ let hiddenDroppedOffViewport = 0;
+ let offViewportUnreachableUnnamed = 0;
+ let offViewportUnnamedHostExempt = 0;
let phantomDropped = 0;
let truncated = 0;
let truncatedInComponents = 0;
@@ -6078,6 +6091,106 @@ async () => {
}
return false;
};
+ // The state the record reports for a control, computed once and used by both the record below and
+ // the off-viewport gate. Asked in one place on purpose: two enumerations of "carries state" drift,
+ // and the gate's copy drifting is a silently dropped control.
+ const _stateFields = (el) => {
+ const out = {};
+ const role = el.getAttribute('role');
+ const secret = el.type === 'password' || isOtpInputValueSecret(el);
+ if (secret) { if (el.value) out.value = '(hidden)'; }
+ else if (el.tagName === 'SELECT' && el.multiple === true) {
+ const picked = Array.from(el.selectedOptions || []);
+ out.selectedOptions = picked.slice(0, 60).map((o) => (o.value + '|' + o.text).slice(0, _RETAIN_WIDTH));
+ out.selectedTotal = picked.length;
+ }
+ else if (el.value) out.value = String(el.value).slice(0, _RETAIN_WIDTH);
+ else if (_isAutocomplete(el)) {
+ const sv = ownCommittedSurface(el);
+ if (sv) out.value = String(sv).slice(0, _RETAIN_WIDTH);
+ }
+ if (el.type === 'checkbox' || el.type === 'radio') out.checked = !!el.checked;
+ else if (role === 'checkbox' || role === 'radio' || role === 'switch') {
+ const ck = el.getAttribute('aria-checked');
+ if (ck === 'true' || ck === 'false') out.checked = ck === 'true';
+ }
+ const selected = el.getAttribute('aria-selected');
+ if ((role === 'tab' || role === 'option') && (selected === 'true' || selected === 'false')) out.selected = selected === 'true';
+ if (role === 'spinbutton' && !secret) {
+ const now = el.getAttribute('aria-valuenow');
+ if (now !== null && !out.value) out.value = String(now).slice(0, _RETAIN_WIDTH);
+ }
+ return out;
+ };
+ // Fails towards reporting state, so a control whose state cannot be read is never dropped for being empty.
+ const _reportsState = (el) => {
+ try {
+ for (const k in _stateFields(el)) return true;
+ return false;
+ } catch (e) {
+ return true;
+ }
+ };
+ const _outsideViewport = (b) => b.right <= 0 || b.bottom <= 0 || b.left >= window.innerWidth || b.top >= window.innerHeight;
+ const _SCROLLS = /^(?:auto|scroll|overlay|hidden)$/;
+ // Walks the flat tree: a slotted node is laid out inside its slot, so a scroll container wrapping
+ // the slot in the host's shadow tree moves it; and a scroll container or fixed ancestor outside
+ // the control's own tree still decides whether a scroll can move it.
+ const _layoutParentOf = (n) => {
+ let slot = null;
+ try { slot = _assignedSlotOf.call(n); } catch (e) { slot = null; }
+ if (slot) return slot;
+ const p = _parentOf.call(n);
+ if (p) return p;
+ let r = null;
+ try { r = Node.prototype.getRootNode.call(n); } catch (e) { r = null; }
+ return r instanceof ShadowRoot ? r.host : null;
+ };
+ // A registered custom element whose shadow root reads null has one that is closed or none at all:
+ // the two cannot be told apart without attaching a root, which would mutate the page. Either way a
+ // slot and the scroll container wrapping it may be unreadable, so nothing observable proves a box
+ // below such a host unreachable. Kept out of _scrollReachable, which answers a question about
+ // layout; this one is about what the walk can see.
+ const _unreadableRootHostAbove = (node) => {
+ for (let p = _layoutParentOf(node); p; p = _layoutParentOf(p)) {
+ try {
+ // Read through the prototype: a form exposes its named controls as its own properties, so an
+ // <input name="tagName"> between the control and the host would otherwise hide the host.
+ const tag = String(_tagNameOf.call(p) || '');
+ if (tag.includes('-') && !!customElements.get(_localNameOf.call(p)) && _shadowRootOf.call(p) === null) return true;
+ } catch (e) {
+ continue;
+ }
+ }
+ return false;
+ };
+ // Whether some scroll could bring box `b` of `node` into the viewport. Errs towards true: any
+ // scroll container with overflow above the node counts, whichever way it scrolls. A fixed box does
+ // not move with the ancestors above it unless one of them re-anchors it (any transform, filter,
+ // perspective or containment, including their will-change hints); otherwise the document scrolls
+ // to any box inside its scroll extent.
+ const _scrollReachable = (node, b) => {
+ let pinned = false;
+ for (let p = node; p; p = _layoutParentOf(p)) {
+ const cs = window.getComputedStyle(p);
+ if (pinned) {
+ const anchors = [cs.transform, cs.translate, cs.rotate, cs.scale, cs.perspective, cs.filter, cs.backdropFilter].some((v) => v && v !== 'none')
+ || /paint|layout|strict|content/.test(cs.contain || '') || cs.contentVisibility === 'auto'
+ || /transform|translate|rotate|scale|perspective|filter/.test(cs.willChange || '');
+ if (!anchors) continue;
+ pinned = false;
+ }
+ if (p !== node) {
+ if (_SCROLLS.test(cs.overflowY) && _scrollHeightOf.call(p) > _clientHeightOf.call(p)) return true;
+ if (_SCROLLS.test(cs.overflowX) && _scrollWidthOf.call(p) > _clientWidthOf.call(p)) return true;
+ }
+ if (cs.position === 'fixed') pinned = true;
+ }
+ if (pinned) return false;
+ const se = document.scrollingElement || document.documentElement;
+ return b.bottom + window.scrollY > 0 && b.top + window.scrollY < _scrollHeightOf.call(se)
+ && b.right + window.scrollX > 0 && b.left + window.scrollX < _scrollWidthOf.call(se);
+ };
// v1 isElementVisible (domUtils.js) force-marks a native form control inside an open shadow root
// as visible even when CSS hides it: web-component libraries hide the native input via
// visibility:hidden / off-canvas positioning behind a styled overlay the user actually clicks.
@@ -6223,6 +6336,28 @@ async () => {
phantomDropped++;
continue;
}
+ // An unnamed control no scroll can bring on screen is one the model can neither identify nor
+ // click: an action on it waits out the whole timeout. Named controls stay, off screen or not, and
+ // so does a file input, which takes files without being visible. Counted in every arm, so the
+ // exposed SET can be compared across arms; only the drop is gated. The per-call count cannot: a
+ // drop does not consume the element budget, so a truncating call in treatment examines further
+ // down the page than the same call in control.
+ if (ownGated && !hidden && unnamed && !_reportsState(el)
+ && !(el.tagName === 'INPUT' && String(el.type || '').toLowerCase() === 'file')
+ && gr.width !== 0 && gr.height !== 0 && _outsideViewport(gr) && !_scrollReachable(gateEl, gr)) {
+ // Kept, and counted apart from the exposed set: an exempt control is not at risk of a wrong
+ // drop, and a page that exempts every candidate must not read as a page that had none.
+ if (_unreadableRootHostAbove(gateEl)) {
+ offViewportUnnamedHostExempt++;
+ } else {
+ offViewportUnreachableUnnamed++;
+ if (_DROP_OFFVIEWPORT_UNNAMED) {
+ hiddenDropped++;
+ hiddenDroppedOffViewport++;
+ continue;
+ }
+ }
+ }
let selector = naturalSelector(el);
if (!selector) {
// We do not write inside a shadow root. Setting a marker there is a mutation of the
@@ -6291,42 +6426,12 @@ async () => {
// element line for a selector that does not exist.
if (role && _WIDGET_ROLES.indexOf(String(role)) !== -1) rec.role = String(role);
if (el.tagName === 'SELECT') rec.options = Array.from(el.options).map((o) => o.value + '|' + o.text).slice(0, 60);
- // el.value on a <select multiple> is the FIRST selected option only: a control holding nine
- // reads as holding one, so an overwrite and an accumulation look identical. Report the set
- // instead -- the scalar is a false readout here, not a partial one. Single-select is untouched.
- const _multiSelect = el.tagName === 'SELECT' && el.multiple === true;
- if (secretValue) { if (el.value) rec.value = '(hidden)'; }
- else if (_multiSelect) {
- const _picked = Array.from(el.selectedOptions || []);
- // Retained per item at the same width as the scalar branch below: this list rides in the
- // persistent conversation prefix, so an uncapped label is paid for on every later turn.
- rec.selectedOptions = _picked.slice(0, 60).map((o) => (o.value + '|' + o.text).slice(0, _RETAIN_WIDTH));
- // The size actually held, not the size retained. Python renders the truncation marker off
- // this: a list that lost its tail silently reads as the whole selection.
- rec.selectedTotal = _picked.length;
- }
- else if (el.value) rec.value = String(el.value).slice(0, _RETAIN_WIDTH);
- // React-Select-style commit: the widget clears el.value and moves the label into its own surface
- // (D3). Only when el.value is empty, so a field still holding its own text is never overridden.
- else if (_isAutocomplete(el)) {
- const sv = ownCommittedSurface(el);
- if (sv) rec.value = String(sv).slice(0, _RETAIN_WIDTH);
- }
- // ARIA defines switch as a checkbox variant carrying the same aria-checked, so it belongs here.
- if (el.type === 'checkbox' || el.type === 'radio') rec.checked = !!el.checked;
- else if (role === 'checkbox' || role === 'radio' || role === 'switch') {
- // Presence-gated like `selected` below: an absent aria-checked, or "mixed", is a state the
- // page never stated, and reporting checked=False for an ON switch is the exact wrong-way
- // toggle this enumeration exists to prevent.
- const ck = el.getAttribute('aria-checked');
- if (ck === 'true' || ck === 'false') rec.checked = ck === 'true';
- }
- const selected = el.getAttribute('aria-selected');
- if ((role === 'tab' || role === 'option') && (selected === 'true' || selected === 'false')) rec.selected = selected === 'true';
- if (role === 'spinbutton' && !secretValue) {
- const now = el.getAttribute('aria-valuenow');
- if (now !== null && !rec.value) rec.value = String(now).slice(0, _RETAIN_WIDTH);
- }
+ // el.value on a <select multiple> is the FIRST selected option only, a React-Select commit moves
+ // the label off el.value into the widget's own surface, and a widget role carries its state in
+ // aria-checked / aria-selected / aria-valuenow. All of that lives in _stateFields, which the
+ // off-viewport gate asks the same question of. Retained at the scalar width per item: this rides
+ // in the persistent conversation prefix, so an uncapped label is paid for on every later turn.
+ Object.assign(rec, _stateFields(el));
if (el.getAttribute('aria-required') === 'true' || el.required) rec.required = true;
const isChoice = el.type === 'checkbox' || el.type === 'radio' || role === 'checkbox' || role === 'radio';
// Read .validity, never checkValidity(): that dispatches an 'invalid' event and perception must
@@ -6912,7 +7017,7 @@ async () => {
}
rec.ref = typeof r === 'number' ? r : null;
}
- const payload = JSON.stringify({ refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });
+ const payload = JSON.stringify({ refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });
return __OBSERVE_RETURN__;
}
"""
@@ -6926,6 +7031,13 @@ def _observe_js_returning(expression: str, retain_width: int) -> str:
key = f'"__tv3el_{secrets.token_hex(8)}"'
return (
_OBSERVE_JS_TEMPLATE.replace("__OBSERVE_RETAIN_WIDTH__", str(int(retain_width)), 1)
+ .replace(
+ "__OBSERVE_DROP_OFFVIEWPORT_UNNAMED__",
+ "true"
+ if run_arm_enabled(OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, settings.TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED)
+ else "false",
+ 1,
+ )
.replace("__OBSERVE_RETURN__", expression, 1)
.replace("__OBSERVE_EL_KEY__", key)
)
@@ -6942,6 +7054,8 @@ def observe_handles_js(retain_width: int = OBSERVE_RETAIN_WIDTH_MIN) -> str:
return _observe_js_returning("{ json: payload, els: outEls }", retain_width)
+# Frozen at import, with no context, so the run-arm terms in it read off forever. Tests only: the
+# production path rebuilds the script per call. Never assert arm-sensitive behaviour against this.
_OBSERVE_JS = observe_js()
@@ -7806,6 +7920,9 @@ _OBSERVE_SUMMED_KEYS = (
"hiddenDroppedOffCanvas",
"hiddenDroppedVisibility",
"hiddenDroppedZeroRect",
+ "hiddenDroppedOffViewport",
+ "offViewportUnreachableUnnamed",
+ "offViewportUnnamedHostExempt",
"phantomDropped",
"markersMinted",
"markersReused",
@@ -8604,6 +8721,9 @@ def build_browser_tools(
"hidden_dropped_off_canvas": int(data.get("hiddenDroppedOffCanvas") or 0),
"hidden_dropped_visibility": int(data.get("hiddenDroppedVisibility") or 0),
"hidden_dropped_zero_rect": int(data.get("hiddenDroppedZeroRect") or 0),
+ "hidden_dropped_off_viewport": int(data.get("hiddenDroppedOffViewport") or 0),
+ "off_viewport_unreachable_unnamed": int(data.get("offViewportUnreachableUnnamed") or 0),
+ "off_viewport_unnamed_host_exempt": int(data.get("offViewportUnnamedHostExempt") or 0),
"phantom_dropped": phantom_dropped,
"iframes_in_component_roots": iframe_info.get("inComponents") or 0,
"undiscovered_roots": data.get("undiscoveredRoots") or 0,
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 7f4ceffd0..12bed8202 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -53,6 +53,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.taskv3.engine import MIN_ACTION_STEPS
from skyvern.forge.taskv3.frame_perception import FRAME_PERCEPTION_FLAG, frame_perception_enabled
from skyvern.forge.taskv3.loop import LoopOutcome, RoundAction
+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, run_arm_enabled
from skyvern.schemas.workflows import BlockStatus, BlockType
from skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDER
from skyvern.webeye.actions.actions import (
@@ -114,6 +115,9 @@ async def _run_execute_task_v3(
loop_mock.context = context
loop_mock.active_credential_parameter_key_during_loop = context.active_credential_parameter_key
loop_mock.frame_perception_enabled_during_loop = frame_perception_enabled()
+ loop_mock.observe_offviewport_drop_during_loop = run_arm_enabled(
+ OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, forced=False
+ )
cb = kwargs.get("on_action_round")
if cb is not None and action_rounds:
for i, round_actions in enumerate(action_rounds):
@@ -206,6 +210,34 @@ async def _run_execute_task_v3(
return out_step, out_task, loop_mock, post_step_mock
+async def test_execute_task_v3_pins_the_observe_offviewport_arm_before_the_loop(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)
+ reader = AsyncMock(
+ side_effect=lambda flag, *_a, **_k: "treatment" if flag == OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG else None
+ )
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
+
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ workflow_run_id="wr_offviewport_reach",
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ # Read inside the loop: the context is the same object after the run, so a pin written late would
+ # still be visible here -- and every observe call builds its script from the arm as it stands then.
+ assert loop_mock.observe_offviewport_drop_during_loop is True
+ assert loop_mock.context.run_arms[OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG] == (task.workflow_run_id, "treatment")
+ assert [c.args[1] for c in reader.await_args_list if c.args[0] == OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG] == [
+ task.workflow_run_id
+ ]
+
+
@pytest.mark.asyncio
async def test_execute_task_v3_resolves_frame_perception_before_the_loop(monkeypatch: pytest.MonkeyPatch) -> None:
"""`_execute_task_v3` must actually call `resolve_frame_perception` with the run's real
diff --git a/tests/unit/test_taskv3_run_arms.py b/tests/unit/test_taskv3_run_arms.py
new file mode 100644
index 000000000..81e49f4c9
--- /dev/null
+++ b/tests/unit/test_taskv3_run_arms.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from unittest.mock import AsyncMock
+
+import pytest
+from structlog.testing import capture_logs
+
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.taskv3.run_arms import resolve_run_arm, run_arm_enabled
+
+_ARM_LOG = "Resolved Task V3 run arm"
+_FLAG = "TASK_V3_EXAMPLE_ARM"
+_OTHER_FLAG = "TASK_V3_OTHER_ARM"
+
+
+def scoped_context() -> Iterator[SkyvernContext]:
+ context = SkyvernContext()
+ skyvern_context.set(context)
+ try:
+ yield context
+ finally:
+ skyvern_context.reset()
+
+
+ ("reader", "arm", "enabled"),
+ [
+ (AsyncMock(return_value="treatment"), "treatment", True),
+ (AsyncMock(return_value="control"), "control", False),
+ (AsyncMock(return_value=None), "unrandomized", False),
+ (AsyncMock(return_value="true"), "unrandomized", False),
+ (AsyncMock(side_effect=RuntimeError("flag service down")), "unrandomized", False),
+ ],
+)
+async def test_arm_log_tells_randomized_control_from_never_randomized(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, reader: AsyncMock, arm: str, enabled: bool
+) -> None:
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
+
+ with capture_logs() as logs:
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+
+ assert run_arm_enabled(_FLAG, forced=False) is enabled
+ [line] = [log for log in logs if log["event"] == _ARM_LOG]
+ assert (line["flag"], line["distinct_id"], line["arm"], line["enabled"], line["forced_by_env"]) == (
+ _FLAG,
+ "wr_1",
+ arm,
+ enabled,
+ False,
+ )
+
+
+async def test_env_setting_forces_on_but_the_log_keeps_the_randomized_arm(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", AsyncMock(return_value="control"))
+
+ with capture_logs() as logs:
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=True)
+
+ assert run_arm_enabled(_FLAG, forced=True) is True
+ assert run_arm_enabled(_FLAG, forced=False) is False
+ [line] = [log for log in logs if log["event"] == _ARM_LOG]
+ assert (line["arm"], line["enabled"], line["forced_by_env"]) == ("control", True, True)
+
+
+async def test_later_blocks_of_a_run_keep_the_first_resolution(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ reader = AsyncMock(side_effect=["treatment", "control", "control"])
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
+
+ with capture_logs() as logs:
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+ assert run_arm_enabled(_FLAG, forced=False) is True
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_2", organization_id="o_1", forced=False)
+
+ assert run_arm_enabled(_FLAG, forced=False) is False
+ assert [log["distinct_id"] for log in logs if log["event"] == _ARM_LOG] == ["wr_1", "wr_2"]
+
+
+async def test_each_flag_keeps_its_own_arm_within_one_run(
+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ reader = AsyncMock(side_effect=lambda flag, *_a, **_k: "treatment" if flag == _FLAG else "control")
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
+
+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+ await resolve_run_arm(scoped_context, _OTHER_FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+
+ assert (run_arm_enabled(_FLAG, forced=False), run_arm_enabled(_OTHER_FLAG, forced=False)) == (True, False)
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 6b3f056d2..d110222f2 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -34,7 +34,7 @@ import skyvern.forge.taskv3.loop as taskv3_loop
import skyvern.forge.taskv3.tools as taskv3_tools
from skyvern.config import settings
from skyvern.forge.sdk.core import skyvern_context
-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.core.skyvern_context import RunArm, SkyvernContext
from skyvern.forge.taskv3.code_surface import (
CodeToolSurface,
apply_surface,
@@ -1814,6 +1814,9 @@ async def test_observe_result_carries_count_only_summary_for_the_call_record() -
"hidden_dropped_off_canvas",
"hidden_dropped_visibility",
"hidden_dropped_zero_rect",
+ "hidden_dropped_off_viewport",
+ "off_viewport_unreachable_unnamed",
+ "off_viewport_unnamed_host_exempt",
"phantom_dropped",
"iframes_in_component_roots",
"undiscovered_roots",
@@ -22764,10 +22767,15 @@ async def test_a_frames_hidden_drops_are_summed_into_the_page_split(monkeypatch:
"<button style='position:absolute;left:-9999px'>FOff</button>"
"<button style='visibility:hidden'>FVis</button>"
"<button style='display:none'>FZ1</button><button style='display:none'>FZ2</button>"
+ "<button style='position:fixed;left:10px;top:calc(100vh + 200px);width:40px;height:20px'></button>"
+ "<script>customElements.define('x-lite', class extends HTMLElement {});</script>"
+ "<x-lite><button style='position:fixed;left:60px;top:calc(100vh + 200px);width:40px;height:20px'>"
+ "</button></x-lite>"
"<button>Frame Live</button>"
)
html = (
'<button style="display:none">MainZ</button><button>Main Live</button>'
+ '<button style="position:fixed;left:10px;top:calc(100vh + 200px);width:40px;height:20px"></button>'
f'<iframe srcdoc="{frame}" width="300" height="120"></iframe>'
)
async with _live_page(html) as page:
@@ -22782,6 +22790,9 @@ async def test_a_frames_hidden_drops_are_summed_into_the_page_split(monkeypatch:
"hidden_dropped_zero_rect": 3,
}, summary
assert summary["hidden_dropped"] == 5, summary
+ # Counted in every arm and in every realm: this one is armed off, and each frame holds one.
+ assert summary["off_viewport_unreachable_unnamed"] == 2, summary
+ assert summary["off_viewport_unnamed_host_exempt"] == 1, summary
# The shape SKY-15662 was diagnosed on: many live, visible per-row controls whose accessible name is
@@ -25911,3 +25922,158 @@ async def test_press_key_accepts_the_key_names_v1_accepts() -> None:
r = await _tool(tools, "press_key").handler({"key": key, "selector": "#field"})
assert r.status == "ok", (key, r.content)
assert expected in await page.evaluate("window.__keys"), key
+
+
+# Controls placed past the bottom and past the right edge: the one-sided off-canvas gate (center-x left
+# of the page) reaches neither. Each carries a distinct tag/type so its digest line is identifiable
+# without a name. A fixed box does not move with the document, so no scroll brings either on screen.
+_OFFVIEWPORT_UNNAMED_HTML = (
+ "<!doctype html><html><head><title>Portal</title><style>"
+ ".away { position: fixed; width: 40px; height: 40px; }"
+ "</style></head><body>"
+ '<button id="go">Go</button>'
+ '<button type="submit" class="away" style="left:40px;top:calc(100vh + 200px)"></button>'
+ '<button type="reset" class="away" style="top:40px;left:calc(100vw + 200px)"></button>'
+ # Must-drop: a hyphenated tag that is not an upgraded custom element has no root that could hide a
+ # scroller, so it must not exempt its children.
+ '<app-shell><div role="option" tabindex="0" class="away" style="left:40px;top:calc(100vh + 200px)"></div></app-shell>'
+ # Must-not-drop: named, same off-screen geometry as the first.
+ '<button type="button" class="away" aria-label="Close notice" style="left:40px;top:calc(100vh + 200px)"></button>'
+ # Must-not-drop: unnamed, off screen, but its scroll container brings it into view. The container
+ # scrolls further than the document does, so only the container can be what keeps it.
+ '<div style="height:200px;overflow:auto"><div style="height:6000px"></div>'
+ '<a href="#in-scroller" style="display:inline-block;width:40px;height:40px"></a></div>'
+ # Must-not-drop: the same, but slotted into a component whose shadow scroll container wraps the slot.
+ '<x-scroller><div style="height:6000px"></div>'
+ '<div role="switch" aria-checked="false" tabindex="0" style="width:40px;height:40px"></div></x-scroller>'
+ # Must-not-drop: the same, slotted into a CLOSED root, whose scroll container is unreadable.
+ '<x-sealed><div style="height:6000px"></div>'
+ '<div role="menuitem" tabindex="0" style="width:40px;height:40px"></div></x-sealed>'
+ # Must-not-drop: the same closed root, with a form between that clobbers the host's own properties.
+ '<x-sealed><form><img name="tagName" alt=""><img name="shadowRoot" alt="">'
+ '<div role="menuitem" tabindex="0" class="away" style="left:40px;top:calc(100vh + 200px)"></div>'
+ "</form></x-sealed>"
+ # Must-not-drop: fixed, but a translated wrapper re-anchors it inside a scroll container.
+ '<div style="height:200px;overflow:auto"><div style="height:6000px"></div><div style="translate:0 0">'
+ '<div role="radio" aria-checked="false" tabindex="0" class="away" style="left:0;top:0"></div></div></div>'
+ # Must-not-drop: state carried the way a widget role carries it, which has no el.value at all.
+ '<div role="tab" aria-selected="true" tabindex="0" class="away"'
+ ' style="left:120px;top:calc(100vh + 200px)"></div>'
+ '<div role="switch" aria-checked="true" tabindex="0" class="away"'
+ ' style="left:180px;top:calc(100vh + 200px)"></div>'
+ # Must-not-drop: value-bearing in the two forms that reach the record after this gate runs -- a
+ # committed autocomplete surface, and a spinbutton's aria-valuenow.
+ '<div class="away" style="left:40px;top:calc(100vh + 400px);width:120px;height:40px">'
+ '<div class="single-value">Paris</div>'
+ '<input role="combobox" aria-autocomplete="list" aria-expanded="false"></div>'
+ '<div role="spinbutton" aria-valuenow="3" tabindex="0" class="away"'
+ ' style="left:40px;top:calc(100vh + 500px);width:40px;height:40px"></div>'
+ # Must-not-drop: a file input takes files without being visible.
+ '<input type="file" class="away" style="left:40px;top:calc(100vh + 200px)">'
+ # Must-not-drop: unnamed, below the fold of a document that scrolls to it.
+ '<div style="height:3000px"></div>'
+ '<div role="checkbox" aria-checked="false" tabindex="0" style="width:40px;height:40px"></div>'
+ "<script>customElements.define('x-scroller', class extends HTMLElement { constructor() { super();"
+ " this.attachShadow({ mode: 'open' }).innerHTML ="
+ " '<div style=\"height:200px;overflow-y:auto\"><slot></slot></div>'; } });"
+ "customElements.define('x-sealed', class extends HTMLElement { constructor() { super();"
+ " this.attachShadow({ mode: 'closed' }).innerHTML ="
+ " '<div style=\"height:200px;overflow-y:auto\"><slot></slot></div>'; } });</script>"
+ "</body></html>"
+)
+
+
+async def _observe_offviewport_fixture(monkeypatch: pytest.MonkeyPatch, arm: RunArm) -> Any:
+ # Driven by the run's pinned arm, not the env force, so the observe tool is shown to read the arm.
+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)
+ context = skyvern_context.SkyvernContext(
+ run_arms={"TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED": ("wr_offviewport", arm)}
+ )
+ skyvern_context.set(context)
+ try:
+ async with _content_page(_OFFVIEWPORT_UNNAMED_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ finally:
+ skyvern_context.reset()
+ assert r.status == "ok" and r.data is not None, r.content
+ return r
+
+
+@_skip_no_browser
+async def test_observe_drops_an_unnamed_control_no_scroll_can_bring_into_the_viewport(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ r = await _observe_offviewport_fixture(monkeypatch, arm="treatment")
+
+ assert "button/submit ''" not in r.content, r.content
+ assert "button/reset ''" not in r.content, r.content
+ assert "div/option ''" not in r.content, r.content
+ assert "Close notice" in r.content, r.content
+ assert re.search(r"ref=\S+ a ''", r.content), r.content
+ for kept in ("div/checkbox", "div/switch", "div/menuitem", "div/radio", "input/file", "div/spinbutton"):
+ assert re.search(rf"ref=\S+ {kept} ''", r.content), (kept, r.content)
+ assert "Paris" in r.content, r.content
+ assert "(13 interactive elements)" in r.content, r.content
+ assert r.data["summary"]["hidden_dropped_off_viewport"] == 3, r.data["summary"]
+ assert r.data["summary"]["off_viewport_unreachable_unnamed"] == 3, r.data["summary"]
+ # The closed-root host's controls are kept and counted apart, not as at-risk candidates.
+ assert r.data["summary"]["off_viewport_unnamed_host_exempt"] == 2, r.data["summary"]
+ assert r.data["summary"]["hidden_dropped"] == 3, r.data["summary"]
+
+
+async def _observe_records(page: Any, arm: RunArm) -> dict[str, Any]:
+ from skyvern.forge.taskv3.tools import observe_js # noqa: PLC0415
+
+ context = skyvern_context.SkyvernContext(
+ run_arms={"TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED": ("wr_offviewport", arm)}
+ )
+ skyvern_context.set(context)
+ try:
+ return json.loads(await page.evaluate(observe_js()))
+ finally:
+ skyvern_context.reset()
+
+
+@_skip_no_browser
+async def test_the_drop_never_removes_a_control_whose_record_reports_state(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The property, over whatever the page holds: a control the arm removes must not be one whose
+ # record carries state. Asserting the class rather than the three forms a field list happens to
+ # enumerate -- that list is what drifted before (SKY-16501).
+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)
+ async with _content_page(_OFFVIEWPORT_UNNAMED_HTML) as page:
+ listed_off = await _observe_records(page, "control")
+ listed_on = await _observe_records(page, "treatment")
+
+ def _identity(e: dict[str, Any]) -> str:
+ return str(e.get("selector") or f"i={e.get('i')}")
+
+ state_keys = ("value", "checked", "selected", "selectedOptions")
+ stateful = {_identity(e) for e in listed_off["elements"] if any(k in e for k in state_keys)}
+ kept = {_identity(e) for e in listed_on["elements"]}
+ dropped = {_identity(e) for e in listed_off["elements"]} - kept
+
+ assert dropped, listed_off["elements"]
+ assert stateful & dropped == set(), sorted(stateful & dropped)
+
+
+@_skip_no_browser
[email protected]("arm", ["control", "unrandomized"])
+async def test_observe_lists_unnamed_offviewport_controls_outside_treatment(
+ monkeypatch: pytest.MonkeyPatch, arm: RunArm
+) -> None:
+ r = await _observe_offviewport_fixture(monkeypatch, arm=arm)
+
+ assert "button/submit ''" in r.content, r.content
+ assert "button/reset ''" in r.content, r.content
+ assert "div/option ''" in r.content, r.content
+ assert "(16 interactive elements)" in r.content, r.content
+ assert r.data["summary"]["hidden_dropped_off_viewport"] == 0, r.data["summary"]
+ # Exposure is measured in every arm, so treatment and control can be compared on the same set.
+ assert r.data["summary"]["off_viewport_unreachable_unnamed"] == 3, r.data["summary"]
+ assert r.data["summary"]["off_viewport_unnamed_host_exempt"] == 2, r.data["summary"]