Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Task V3 observe: stop listing an unnamed control no scroll can reach, on a per-run arm (SKY-16501) (#8605)
pedrohsdb55702a579ade
diff --git a/skyvern/config.py b/skyvern/config.pyindex 8a2a344ce..1dfbf69e1 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -552,6 +552,9 @@ class Settings(BaseSettings):# actuation without the probe realm gives working actions whose readbacks answer about the main# document instead of the element's own -- a verdict reported without being measured.TASK_V3_FRAME_PERCEPTION: bool = False+ # Force-on term only: the arm is randomized per run by the flag of the same name and read through+ # run_arm_enabled() in skyvern/forge/taskv3/run_arms.py (SKY-16501).+ TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED: bool = False# Which browser surface the v3 loop offers: today's action tools ("off"), those plus a code# tool ("add"), or the code tool instead of them ("replace"). Three states rather than a boolean# because the benchmark separated add from replace on speed alone, not on success. The code tooldiff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 9f3caf936..9b92d07a9 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -180,6 +180,7 @@ from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, Wofrom skyvern.forge.taskv3.frame_perception import frame_perception_enabled, resolve_frame_perceptionfrom skyvern.forge.taskv3.loop import LoopOutcome, RoundActionfrom skyvern.forge.taskv3.pre_submit_capture import PreSubmitCaptureRing, is_run_sampled, pre_submit_screenshot+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, resolve_run_armfrom skyvern.forge.taskv3.target_label import compose_target_intentionfrom skyvern.forge.validation_evidence_router import (ValidationRouterMode,@@ -1851,6 +1852,13 @@ class ForgeAgent:distinct_id=task.workflow_run_id or task.task_id,organization_id=task.organization_id,)+ await resolve_run_arm(+ context,+ OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG,+ distinct_id=task.workflow_run_id or task.task_id,+ organization_id=task.organization_id,+ forced=settings.TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED,+ )offer_error_codes = Falseif task.error_code_mapping:try:diff --git a/skyvern/forge/sdk/core/skyvern_context.py b/skyvern/forge/sdk/core/skyvern_context.pyindex df095fdde..5a6c0544b 100644--- a/skyvern/forge/sdk/core/skyvern_context.py+++ b/skyvern/forge/sdk/core/skyvern_context.py@@ -9,7 +9,7 @@ from contextlib import contextmanagerfrom contextvars import ContextVar, Tokenfrom dataclasses import dataclass, fieldfrom enum import StrEnum-from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypedDict+from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Literal, TypedDictfrom zoneinfo import ZoneInfoimport structlog@@ -239,6 +239,9 @@ class DialogEntry(TypedDict):count: int+RunArm = Literal["treatment", "control", "unrandomized"]++class EnrichTreeMode(StrEnum):CONTROL = "control"ENRICHED_TREE = "enriched_tree"@@ -418,6 +421,10 @@ class SkyvernContext:frame_perception_lock: asyncio.Lock = field(default_factory=asyncio.Lock)enrich_tree_mode: EnrichTreeMode = EnrichTreeMode.CONTROLstep_retry_index: int = 0+ # Task V3 run arms by flag: (run id the arm was resolved for, arm). Same pin contract as frame perception+ # above; written and read only through skyvern.forge.taskv3.run_arms.+ run_arms: dict[str, tuple[str, RunArm]] = field(default_factory=dict)+ run_arms_lock: asyncio.Lock = field(default_factory=asyncio.Lock)# Run-level SLIM_LLM_OUTPUT_PROMPTS assignment, resolved once by slim_llm_output.# The lock makes first-use resolution single-flight under parallel prompt builds.diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.pynew file mode 100644index 000000000..8e4bdba63--- /dev/null+++ b/skyvern/forge/taskv3/run_arms.py@@ -0,0 +1,86 @@+"""Per-run arms for Task V3 experiments that change what one run does (SKY-16501).++Each flag is multivariate so that "randomized to control" and "never randomized" stay apart: a run outside+the rollout, or with no flag at all, gets no variant, and an analyst reading the arm log must not pool it+with control. An env setting per flag can only force the arm on.+"""++from __future__ import annotations++import structlog++from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import RunArm+from skyvern.forge.sdk.experimentation.providers import NoOpExperimentationProvider++LOG = structlog.get_logger()++OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG = "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED"+++def _pinned_arm(context: skyvern_context.SkyvernContext, flag: str, distinct_id: str) -> RunArm | None:+ pin = context.run_arms.get(flag)+ return pin[1] if pin is not None and pin[0] == distinct_id else None+++async def resolve_run_arm(+ context: skyvern_context.SkyvernContext,+ flag: str,+ *,+ distinct_id: str,+ organization_id: str | None,+ forced: bool,+) -> RunArm:+ """Resolve ``flag`` once per run and pin it on the context; every failure resolves unrandomized (off)."""+ arm = _pinned_arm(context, flag, distinct_id)+ if arm is not None:+ return arm+ async with context.run_arms_lock:+ arm = _pinned_arm(context, flag, distinct_id)+ if arm is not None:+ return arm+ variant: str | None = None+ try:+ provider = app.EXPERIMENTATION_PROVIDER+ if not isinstance(provider, NoOpExperimentationProvider):+ variant = await provider.get_value_cached(+ flag,+ distinct_id,+ properties={"organization_id": organization_id},+ )+ except Exception:+ LOG.warning("Failed to resolve Task V3 run arm", flag=flag, exc_info=True)+ variant = None+ if variant == "treatment":+ arm = "treatment"+ elif variant == "control":+ arm = "control"+ else:+ # An unknown variant string is a misconfigured flag, not a randomized control run.+ arm = "unrandomized"+ # Rebound rather than mutated, so a shallow copy of the context keeps the pins it was copied with.+ context.run_arms = {**context.run_arms, flag: (distinct_id, arm)}+ LOG.info(+ "Resolved Task V3 run arm",+ flag=flag,+ distinct_id=distinct_id,+ organization_id=organization_id,+ arm=arm,+ variant=variant,+ # The arm is the randomization; this is what the run did, which an env force can override.+ enabled=forced or arm == "treatment",+ forced_by_env=forced,+ )+ return arm+++def run_arm_enabled(flag: str, forced: bool) -> bool:+ """Whether the current run is on for ``flag``. Sound only inside ``_execute_task_v3``, after resolution."""+ if forced:+ return True+ context = skyvern_context.current()+ if context is None:+ return False+ pin = context.run_arms.get(flag)+ return pin is not None and pin[1] == "treatment"diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex 527e5f0e0..ef3d1bcd8 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -67,6 +67,7 @@ from skyvern.forge.taskv3.loop import (set_driver_timeout_predicate,)from skyvern.forge.taskv3.preflight import PREFLIGHT_TOOL_NAMES, preflight_tool_action+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, run_arm_enabledfrom skyvern.forge.taskv3.target_label import TARGET_KIND_TOKENS, TARGET_NAME_CAPfrom skyvern.webeye.actions.key_names import normalize_key_chordfrom skyvern.webeye.browser_driver_errors import is_driver_timeout_error@@ -5177,6 +5178,7 @@ async () => {// Field text is retained at this width and masked, then capped for display, in Python. Substituted// per call from the payload refs: any minted URL that starts inside a display window fits whole.const _RETAIN_WIDTH = __OBSERVE_RETAIN_WIDTH__;+ const _DROP_OFFVIEWPORT_UNNAMED = __OBSERVE_DROP_OFFVIEWPORT_UNNAMED__;const _GROUP_TEXT_TOTAL_CAP = """+ str(OBSERVE_GROUP_TEXT_TOTAL_CAP)+ r""";@@ -5197,6 +5199,14 @@ async () => {};const _parentOf = _getter(Node.prototype, 'parentElement');const _scrollLeftOf = _getter(Element.prototype, 'scrollLeft');+ const _assignedSlotOf = _getter(Element.prototype, 'assignedSlot');+ const _tagNameOf = _getter(Element.prototype, 'tagName');+ const _localNameOf = _getter(Element.prototype, 'localName');+ const _shadowRootOf = _getter(Element.prototype, 'shadowRoot');+ const _scrollHeightOf = _getter(Element.prototype, 'scrollHeight');+ const _scrollWidthOf = _getter(Element.prototype, 'scrollWidth');+ const _clientHeightOf = _getter(Element.prototype, 'clientHeight');+ const _clientWidthOf = _getter(Element.prototype, 'clientWidth');const _prevOf = _getter(Node.prototype, 'previousSibling');const _nextOf = _getter(Node.prototype, 'nextSibling');const _firstChildOf = _getter(Node.prototype, 'firstChild');@@ -6041,6 +6051,9 @@ async () => {let hiddenDroppedOffCanvas = 0;let hiddenDroppedVisibility = 0;let hiddenDroppedZeroRect = 0;+ let hiddenDroppedOffViewport = 0;+ let offViewportUnreachableUnnamed = 0;+ let offViewportUnnamedHostExempt = 0;let phantomDropped = 0;let truncated = 0;let truncatedInComponents = 0;@@ -6078,6 +6091,106 @@ async () => {}return false;};+ // The state the record reports for a control, computed once and used by both the record below and+ // the off-viewport gate. Asked in one place on purpose: two enumerations of "carries state" drift,+ // and the gate's copy drifting is a silently dropped control.+ const _stateFields = (el) => {+ const out = {};+ const role = el.getAttribute('role');+ const secret = el.type === 'password' || isOtpInputValueSecret(el);+ if (secret) { if (el.value) out.value = '(hidden)'; }+ else if (el.tagName === 'SELECT' && el.multiple === true) {+ const picked = Array.from(el.selectedOptions || []);+ out.selectedOptions = picked.slice(0, 60).map((o) => (o.value + '|' + o.text).slice(0, _RETAIN_WIDTH));+ out.selectedTotal = picked.length;+ }+ else if (el.value) out.value = String(el.value).slice(0, _RETAIN_WIDTH);+ else if (_isAutocomplete(el)) {+ const sv = ownCommittedSurface(el);+ if (sv) out.value = String(sv).slice(0, _RETAIN_WIDTH);+ }+ if (el.type === 'checkbox' || el.type === 'radio') out.checked = !!el.checked;+ else if (role === 'checkbox' || role === 'radio' || role === 'switch') {+ const ck = el.getAttribute('aria-checked');+ if (ck === 'true' || ck === 'false') out.checked = ck === 'true';+ }+ const selected = el.getAttribute('aria-selected');+ if ((role === 'tab' || role === 'option') && (selected === 'true' || selected === 'false')) out.selected = selected === 'true';+ if (role === 'spinbutton' && !secret) {+ const now = el.getAttribute('aria-valuenow');+ if (now !== null && !out.value) out.value = String(now).slice(0, _RETAIN_WIDTH);+ }+ return out;+ };+ // Fails towards reporting state, so a control whose state cannot be read is never dropped for being empty.+ const _reportsState = (el) => {+ try {+ for (const k in _stateFields(el)) return true;+ return false;+ } catch (e) {+ return true;+ }+ };+ const _outsideViewport = (b) => b.right <= 0 || b.bottom <= 0 || b.left >= window.innerWidth || b.top >= window.innerHeight;+ const _SCROLLS = /^(?:auto|scroll|overlay|hidden)$/;+ // Walks the flat tree: a slotted node is laid out inside its slot, so a scroll container wrapping+ // the slot in the host's shadow tree moves it; and a scroll container or fixed ancestor outside+ // the control's own tree still decides whether a scroll can move it.+ const _layoutParentOf = (n) => {+ let slot = null;+ try { slot = _assignedSlotOf.call(n); } catch (e) { slot = null; }+ if (slot) return slot;+ const p = _parentOf.call(n);+ if (p) return p;+ let r = null;+ try { r = Node.prototype.getRootNode.call(n); } catch (e) { r = null; }+ return r instanceof ShadowRoot ? r.host : null;+ };+ // A registered custom element whose shadow root reads null has one that is closed or none at all:+ // the two cannot be told apart without attaching a root, which would mutate the page. Either way a+ // slot and the scroll container wrapping it may be unreadable, so nothing observable proves a box+ // below such a host unreachable. Kept out of _scrollReachable, which answers a question about+ // layout; this one is about what the walk can see.+ const _unreadableRootHostAbove = (node) => {+ for (let p = _layoutParentOf(node); p; p = _layoutParentOf(p)) {+ try {+ // Read through the prototype: a form exposes its named controls as its own properties, so an+ // <input name="tagName"> between the control and the host would otherwise hide the host.+ const tag = String(_tagNameOf.call(p) || '');+ if (tag.includes('-') && !!customElements.get(_localNameOf.call(p)) && _shadowRootOf.call(p) === null) return true;+ } catch (e) {+ continue;+ }+ }+ return false;+ };+ // Whether some scroll could bring box `b` of `node` into the viewport. Errs towards true: any+ // scroll container with overflow above the node counts, whichever way it scrolls. A fixed box does+ // not move with the ancestors above it unless one of them re-anchors it (any transform, filter,+ // perspective or containment, including their will-change hints); otherwise the document scrolls+ // to any box inside its scroll extent.+ const _scrollReachable = (node, b) => {+ let pinned = false;+ for (let p = node; p; p = _layoutParentOf(p)) {+ const cs = window.getComputedStyle(p);+ if (pinned) {+ const anchors = [cs.transform, cs.translate, cs.rotate, cs.scale, cs.perspective, cs.filter, cs.backdropFilter].some((v) => v && v !== 'none')+ || /paint|layout|strict|content/.test(cs.contain || '') || cs.contentVisibility === 'auto'+ || /transform|translate|rotate|scale|perspective|filter/.test(cs.willChange || '');+ if (!anchors) continue;+ pinned = false;+ }+ if (p !== node) {+ if (_SCROLLS.test(cs.overflowY) && _scrollHeightOf.call(p) > _clientHeightOf.call(p)) return true;+ if (_SCROLLS.test(cs.overflowX) && _scrollWidthOf.call(p) > _clientWidthOf.call(p)) return true;+ }+ if (cs.position === 'fixed') pinned = true;+ }+ if (pinned) return false;+ const se = document.scrollingElement || document.documentElement;+ return b.bottom + window.scrollY > 0 && b.top + window.scrollY < _scrollHeightOf.call(se)+ && b.right + window.scrollX > 0 && b.left + window.scrollX < _scrollWidthOf.call(se);+ };// v1 isElementVisible (domUtils.js) force-marks a native form control inside an open shadow root// as visible even when CSS hides it: web-component libraries hide the native input via// visibility:hidden / off-canvas positioning behind a styled overlay the user actually clicks.@@ -6223,6 +6336,28 @@ async () => {phantomDropped++;continue;}+ // An unnamed control no scroll can bring on screen is one the model can neither identify nor+ // click: an action on it waits out the whole timeout. Named controls stay, off screen or not, and+ // so does a file input, which takes files without being visible. Counted in every arm, so the+ // exposed SET can be compared across arms; only the drop is gated. The per-call count cannot: a+ // drop does not consume the element budget, so a truncating call in treatment examines further+ // down the page than the same call in control.+ if (ownGated && !hidden && unnamed && !_reportsState(el)+ && !(el.tagName === 'INPUT' && String(el.type || '').toLowerCase() === 'file')+ && gr.width !== 0 && gr.height !== 0 && _outsideViewport(gr) && !_scrollReachable(gateEl, gr)) {+ // Kept, and counted apart from the exposed set: an exempt control is not at risk of a wrong+ // drop, and a page that exempts every candidate must not read as a page that had none.+ if (_unreadableRootHostAbove(gateEl)) {+ offViewportUnnamedHostExempt++;+ } else {+ offViewportUnreachableUnnamed++;+ if (_DROP_OFFVIEWPORT_UNNAMED) {+ hiddenDropped++;+ hiddenDroppedOffViewport++;+ continue;+ }+ }+ }let selector = naturalSelector(el);if (!selector) {// We do not write inside a shadow root. Setting a marker there is a mutation of the@@ -6291,42 +6426,12 @@ async () => {// element line for a selector that does not exist.if (role && _WIDGET_ROLES.indexOf(String(role)) !== -1) rec.role = String(role);if (el.tagName === 'SELECT') rec.options = Array.from(el.options).map((o) => o.value + '|' + o.text).slice(0, 60);- // el.value on a <select multiple> is the FIRST selected option only: a control holding nine- // reads as holding one, so an overwrite and an accumulation look identical. Report the set- // instead -- the scalar is a false readout here, not a partial one. Single-select is untouched.- const _multiSelect = el.tagName === 'SELECT' && el.multiple === true;- if (secretValue) { if (el.value) rec.value = '(hidden)'; }- else if (_multiSelect) {- const _picked = Array.from(el.selectedOptions || []);- // Retained per item at the same width as the scalar branch below: this list rides in the- // persistent conversation prefix, so an uncapped label is paid for on every later turn.- rec.selectedOptions = _picked.slice(0, 60).map((o) => (o.value + '|' + o.text).slice(0, _RETAIN_WIDTH));- // The size actually held, not the size retained. Python renders the truncation marker off- // this: a list that lost its tail silently reads as the whole selection.- rec.selectedTotal = _picked.length;- }- else if (el.value) rec.value = String(el.value).slice(0, _RETAIN_WIDTH);- // React-Select-style commit: the widget clears el.value and moves the label into its own surface- // (D3). Only when el.value is empty, so a field still holding its own text is never overridden.- else if (_isAutocomplete(el)) {- const sv = ownCommittedSurface(el);- if (sv) rec.value = String(sv).slice(0, _RETAIN_WIDTH);- }- // ARIA defines switch as a checkbox variant carrying the same aria-checked, so it belongs here.- if (el.type === 'checkbox' || el.type === 'radio') rec.checked = !!el.checked;- else if (role === 'checkbox' || role === 'radio' || role === 'switch') {- // Presence-gated like `selected` below: an absent aria-checked, or "mixed", is a state the- // page never stated, and reporting checked=False for an ON switch is the exact wrong-way- // toggle this enumeration exists to prevent.- const ck = el.getAttribute('aria-checked');- if (ck === 'true' || ck === 'false') rec.checked = ck === 'true';- }- const selected = el.getAttribute('aria-selected');- if ((role === 'tab' || role === 'option') && (selected === 'true' || selected === 'false')) rec.selected = selected === 'true';- if (role === 'spinbutton' && !secretValue) {- const now = el.getAttribute('aria-valuenow');- if (now !== null && !rec.value) rec.value = String(now).slice(0, _RETAIN_WIDTH);- }+ // el.value on a <select multiple> is the FIRST selected option only, a React-Select commit moves+ // the label off el.value into the widget's own surface, and a widget role carries its state in+ // aria-checked / aria-selected / aria-valuenow. All of that lives in _stateFields, which the+ // off-viewport gate asks the same question of. Retained at the scalar width per item: this rides+ // in the persistent conversation prefix, so an uncapped label is paid for on every later turn.+ Object.assign(rec, _stateFields(el));if (el.getAttribute('aria-required') === 'true' || el.required) rec.required = true;const isChoice = el.type === 'checkbox' || el.type === 'radio' || role === 'checkbox' || role === 'radio';// Read .validity, never checkValidity(): that dispatches an 'invalid' event and perception must@@ -6912,7 +7017,7 @@ async () => {}rec.ref = typeof r === 'number' ? r : null;}- const payload = JSON.stringify({ refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });+ const payload = JSON.stringify({ refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });return __OBSERVE_RETURN__;}"""@@ -6926,6 +7031,13 @@ def _observe_js_returning(expression: str, retain_width: int) -> str:key = f'"__tv3el_{secrets.token_hex(8)}"'return (_OBSERVE_JS_TEMPLATE.replace("__OBSERVE_RETAIN_WIDTH__", str(int(retain_width)), 1)+ .replace(+ "__OBSERVE_DROP_OFFVIEWPORT_UNNAMED__",+ "true"+ if run_arm_enabled(OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, settings.TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED)+ else "false",+ 1,+ ).replace("__OBSERVE_RETURN__", expression, 1).replace("__OBSERVE_EL_KEY__", key))@@ -6942,6 +7054,8 @@ def observe_handles_js(retain_width: int = OBSERVE_RETAIN_WIDTH_MIN) -> str:return _observe_js_returning("{ json: payload, els: outEls }", retain_width)+# Frozen at import, with no context, so the run-arm terms in it read off forever. Tests only: the+# production path rebuilds the script per call. Never assert arm-sensitive behaviour against this._OBSERVE_JS = observe_js()@@ -7806,6 +7920,9 @@ _OBSERVE_SUMMED_KEYS = ("hiddenDroppedOffCanvas","hiddenDroppedVisibility","hiddenDroppedZeroRect",+ "hiddenDroppedOffViewport",+ "offViewportUnreachableUnnamed",+ "offViewportUnnamedHostExempt","phantomDropped","markersMinted","markersReused",@@ -8604,6 +8721,9 @@ def build_browser_tools("hidden_dropped_off_canvas": int(data.get("hiddenDroppedOffCanvas") or 0),"hidden_dropped_visibility": int(data.get("hiddenDroppedVisibility") or 0),"hidden_dropped_zero_rect": int(data.get("hiddenDroppedZeroRect") or 0),+ "hidden_dropped_off_viewport": int(data.get("hiddenDroppedOffViewport") or 0),+ "off_viewport_unreachable_unnamed": int(data.get("offViewportUnreachableUnnamed") or 0),+ "off_viewport_unnamed_host_exempt": int(data.get("offViewportUnnamedHostExempt") or 0),"phantom_dropped": phantom_dropped,"iframes_in_component_roots": iframe_info.get("inComponents") or 0,"undiscovered_roots": data.get("undiscoveredRoots") or 0,diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.pyindex 7f4ceffd0..12bed8202 100644--- a/tests/unit/test_agent_task_v3.py+++ b/tests/unit/test_agent_task_v3.py@@ -53,6 +53,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.forge.taskv3.engine import MIN_ACTION_STEPSfrom skyvern.forge.taskv3.frame_perception import FRAME_PERCEPTION_FLAG, frame_perception_enabledfrom skyvern.forge.taskv3.loop import LoopOutcome, RoundAction+from skyvern.forge.taskv3.run_arms import OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, run_arm_enabledfrom skyvern.schemas.workflows import BlockStatus, BlockTypefrom skyvern.utils.secret_redaction import REDACTED_SECRET_PLACEHOLDERfrom skyvern.webeye.actions.actions import (@@ -114,6 +115,9 @@ async def _run_execute_task_v3(loop_mock.context = contextloop_mock.active_credential_parameter_key_during_loop = context.active_credential_parameter_keyloop_mock.frame_perception_enabled_during_loop = frame_perception_enabled()+ loop_mock.observe_offviewport_drop_during_loop = run_arm_enabled(+ OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG, forced=False+ )cb = kwargs.get("on_action_round")if cb is not None and action_rounds:for i, round_actions in enumerate(action_rounds):@@ -206,6 +210,34 @@ async def _run_execute_task_v3(return out_step, out_task, loop_mock, post_step_mock+async def test_execute_task_v3_pins_the_observe_offviewport_arm_before_the_loop(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)+ reader = AsyncMock(+ side_effect=lambda flag, *_a, **_k: "treatment" if flag == OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG else None+ )+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)++ outcome = LoopOutcome(status="completed", reason="done", billable_actions=[])+ _step, task, loop_mock, _post = await _run_execute_task_v3(+ monkeypatch,+ outcome,+ workflow_run_id="wr_offviewport_reach",+ data_extraction_goal=None,+ extracted_information_schema=None,+ )++ # Read inside the loop: the context is the same object after the run, so a pin written late would+ # still be visible here -- and every observe call builds its script from the arm as it stands then.+ assert loop_mock.observe_offviewport_drop_during_loop is True+ assert loop_mock.context.run_arms[OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG] == (task.workflow_run_id, "treatment")+ assert [c.args[1] for c in reader.await_args_list if c.args[0] == OBSERVE_DROP_OFFVIEWPORT_UNNAMED_FLAG] == [+ task.workflow_run_id+ ]++@pytest.mark.asyncioasync def test_execute_task_v3_resolves_frame_perception_before_the_loop(monkeypatch: pytest.MonkeyPatch) -> None:"""`_execute_task_v3` must actually call `resolve_frame_perception` with the run's realdiff --git a/tests/unit/test_taskv3_run_arms.py b/tests/unit/test_taskv3_run_arms.pynew file mode 100644index 000000000..81e49f4c9--- /dev/null+++ b/tests/unit/test_taskv3_run_arms.py@@ -0,0 +1,101 @@+from __future__ import annotations++from collections.abc import Iterator+from unittest.mock import AsyncMock++import pytest+from structlog.testing import capture_logs++from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.taskv3.run_arms import resolve_run_arm, run_arm_enabled++_ARM_LOG = "Resolved Task V3 run arm"+_FLAG = "TASK_V3_EXAMPLE_ARM"+_OTHER_FLAG = "TASK_V3_OTHER_ARM"+++def scoped_context() -> Iterator[SkyvernContext]:+ context = SkyvernContext()+ skyvern_context.set(context)+ try:+ yield context+ finally:+ skyvern_context.reset()+++ ("reader", "arm", "enabled"),+ [+ (AsyncMock(return_value="treatment"), "treatment", True),+ (AsyncMock(return_value="control"), "control", False),+ (AsyncMock(return_value=None), "unrandomized", False),+ (AsyncMock(return_value="true"), "unrandomized", False),+ (AsyncMock(side_effect=RuntimeError("flag service down")), "unrandomized", False),+ ],+)+async def test_arm_log_tells_randomized_control_from_never_randomized(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch, reader: AsyncMock, arm: str, enabled: bool+) -> None:+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)++ with capture_logs() as logs:+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)++ assert run_arm_enabled(_FLAG, forced=False) is enabled+ [line] = [log for log in logs if log["event"] == _ARM_LOG]+ assert (line["flag"], line["distinct_id"], line["arm"], line["enabled"], line["forced_by_env"]) == (+ _FLAG,+ "wr_1",+ arm,+ enabled,+ False,+ )+++async def test_env_setting_forces_on_but_the_log_keeps_the_randomized_arm(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", AsyncMock(return_value="control"))++ with capture_logs() as logs:+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=True)++ assert run_arm_enabled(_FLAG, forced=True) is True+ assert run_arm_enabled(_FLAG, forced=False) is False+ [line] = [log for log in logs if log["event"] == _ARM_LOG]+ assert (line["arm"], line["enabled"], line["forced_by_env"]) == ("control", True, True)+++async def test_later_blocks_of_a_run_keep_the_first_resolution(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ reader = AsyncMock(side_effect=["treatment", "control", "control"])+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)++ with capture_logs() as logs:+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)+ assert run_arm_enabled(_FLAG, forced=False) is True+ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_2", organization_id="o_1", forced=False)++ assert run_arm_enabled(_FLAG, forced=False) is False+ assert [log["distinct_id"] for log in logs if log["event"] == _ARM_LOG] == ["wr_1", "wr_2"]+++async def test_each_flag_keeps_its_own_arm_within_one_run(+ scoped_context: SkyvernContext, monkeypatch: pytest.MonkeyPatch+) -> None:+ reader = AsyncMock(side_effect=lambda flag, *_a, **_k: "treatment" if flag == _FLAG else "control")+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)++ await resolve_run_arm(scoped_context, _FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)+ await resolve_run_arm(scoped_context, _OTHER_FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)++ assert (run_arm_enabled(_FLAG, forced=False), run_arm_enabled(_OTHER_FLAG, forced=False)) == (True, False)diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 6b3f056d2..d110222f2 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -34,7 +34,7 @@ import skyvern.forge.taskv3.loop as taskv3_loopimport skyvern.forge.taskv3.tools as taskv3_toolsfrom skyvern.config import settingsfrom skyvern.forge.sdk.core import skyvern_context-from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.core.skyvern_context import RunArm, SkyvernContextfrom skyvern.forge.taskv3.code_surface import (CodeToolSurface,apply_surface,@@ -1814,6 +1814,9 @@ async def test_observe_result_carries_count_only_summary_for_the_call_record() -"hidden_dropped_off_canvas","hidden_dropped_visibility","hidden_dropped_zero_rect",+ "hidden_dropped_off_viewport",+ "off_viewport_unreachable_unnamed",+ "off_viewport_unnamed_host_exempt","phantom_dropped","iframes_in_component_roots","undiscovered_roots",@@ -22764,10 +22767,15 @@ async def test_a_frames_hidden_drops_are_summed_into_the_page_split(monkeypatch:"<button style='position:absolute;left:-9999px'>FOff</button>""<button style='visibility:hidden'>FVis</button>""<button style='display:none'>FZ1</button><button style='display:none'>FZ2</button>"+ "<button style='position:fixed;left:10px;top:calc(100vh + 200px);width:40px;height:20px'></button>"+ "<script>customElements.define('x-lite', class extends HTMLElement {});</script>"+ "<x-lite><button style='position:fixed;left:60px;top:calc(100vh + 200px);width:40px;height:20px'>"+ "</button></x-lite>""<button>Frame Live</button>")html = ('<button style="display:none">MainZ</button><button>Main Live</button>'+ '<button style="position:fixed;left:10px;top:calc(100vh + 200px);width:40px;height:20px"></button>'f'<iframe srcdoc="{frame}" width="300" height="120"></iframe>')async with _live_page(html) as page:@@ -22782,6 +22790,9 @@ async def test_a_frames_hidden_drops_are_summed_into_the_page_split(monkeypatch:"hidden_dropped_zero_rect": 3,}, summaryassert summary["hidden_dropped"] == 5, summary+ # Counted in every arm and in every realm: this one is armed off, and each frame holds one.+ assert summary["off_viewport_unreachable_unnamed"] == 2, summary+ assert summary["off_viewport_unnamed_host_exempt"] == 1, summary# The shape SKY-15662 was diagnosed on: many live, visible per-row controls whose accessible name is@@ -25911,3 +25922,158 @@ async def test_press_key_accepts_the_key_names_v1_accepts() -> None:r = await _tool(tools, "press_key").handler({"key": key, "selector": "#field"})assert r.status == "ok", (key, r.content)assert expected in await page.evaluate("window.__keys"), key+++# Controls placed past the bottom and past the right edge: the one-sided off-canvas gate (center-x left+# of the page) reaches neither. Each carries a distinct tag/type so its digest line is identifiable+# without a name. A fixed box does not move with the document, so no scroll brings either on screen.+_OFFVIEWPORT_UNNAMED_HTML = (+ "<!doctype html><html><head><title>Portal</title><style>"+ ".away { position: fixed; width: 40px; height: 40px; }"+ "</style></head><body>"+ '<button id="go">Go</button>'+ '<button type="submit" class="away" style="left:40px;top:calc(100vh + 200px)"></button>'+ '<button type="reset" class="away" style="top:40px;left:calc(100vw + 200px)"></button>'+ # Must-drop: a hyphenated tag that is not an upgraded custom element has no root that could hide a+ # scroller, so it must not exempt its children.+ '<app-shell><div role="option" tabindex="0" class="away" style="left:40px;top:calc(100vh + 200px)"></div></app-shell>'+ # Must-not-drop: named, same off-screen geometry as the first.+ '<button type="button" class="away" aria-label="Close notice" style="left:40px;top:calc(100vh + 200px)"></button>'+ # Must-not-drop: unnamed, off screen, but its scroll container brings it into view. The container+ # scrolls further than the document does, so only the container can be what keeps it.+ '<div style="height:200px;overflow:auto"><div style="height:6000px"></div>'+ '<a href="#in-scroller" style="display:inline-block;width:40px;height:40px"></a></div>'+ # Must-not-drop: the same, but slotted into a component whose shadow scroll container wraps the slot.+ '<x-scroller><div style="height:6000px"></div>'+ '<div role="switch" aria-checked="false" tabindex="0" style="width:40px;height:40px"></div></x-scroller>'+ # Must-not-drop: the same, slotted into a CLOSED root, whose scroll container is unreadable.+ '<x-sealed><div style="height:6000px"></div>'+ '<div role="menuitem" tabindex="0" style="width:40px;height:40px"></div></x-sealed>'+ # Must-not-drop: the same closed root, with a form between that clobbers the host's own properties.+ '<x-sealed><form><img name="tagName" alt=""><img name="shadowRoot" alt="">'+ '<div role="menuitem" tabindex="0" class="away" style="left:40px;top:calc(100vh + 200px)"></div>'+ "</form></x-sealed>"+ # Must-not-drop: fixed, but a translated wrapper re-anchors it inside a scroll container.+ '<div style="height:200px;overflow:auto"><div style="height:6000px"></div><div style="translate:0 0">'+ '<div role="radio" aria-checked="false" tabindex="0" class="away" style="left:0;top:0"></div></div></div>'+ # Must-not-drop: state carried the way a widget role carries it, which has no el.value at all.+ '<div role="tab" aria-selected="true" tabindex="0" class="away"'+ ' style="left:120px;top:calc(100vh + 200px)"></div>'+ '<div role="switch" aria-checked="true" tabindex="0" class="away"'+ ' style="left:180px;top:calc(100vh + 200px)"></div>'+ # Must-not-drop: value-bearing in the two forms that reach the record after this gate runs -- a+ # committed autocomplete surface, and a spinbutton's aria-valuenow.+ '<div class="away" style="left:40px;top:calc(100vh + 400px);width:120px;height:40px">'+ '<div class="single-value">Paris</div>'+ '<input role="combobox" aria-autocomplete="list" aria-expanded="false"></div>'+ '<div role="spinbutton" aria-valuenow="3" tabindex="0" class="away"'+ ' style="left:40px;top:calc(100vh + 500px);width:40px;height:40px"></div>'+ # Must-not-drop: a file input takes files without being visible.+ '<input type="file" class="away" style="left:40px;top:calc(100vh + 200px)">'+ # Must-not-drop: unnamed, below the fold of a document that scrolls to it.+ '<div style="height:3000px"></div>'+ '<div role="checkbox" aria-checked="false" tabindex="0" style="width:40px;height:40px"></div>'+ "<script>customElements.define('x-scroller', class extends HTMLElement { constructor() { super();"+ " this.attachShadow({ mode: 'open' }).innerHTML ="+ " '<div style=\"height:200px;overflow-y:auto\"><slot></slot></div>'; } });"+ "customElements.define('x-sealed', class extends HTMLElement { constructor() { super();"+ " this.attachShadow({ mode: 'closed' }).innerHTML ="+ " '<div style=\"height:200px;overflow-y:auto\"><slot></slot></div>'; } });</script>"+ "</body></html>"+)+++async def _observe_offviewport_fixture(monkeypatch: pytest.MonkeyPatch, arm: RunArm) -> Any:+ # Driven by the run's pinned arm, not the env force, so the observe tool is shown to read the arm.+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)+ context = skyvern_context.SkyvernContext(+ run_arms={"TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED": ("wr_offviewport", arm)}+ )+ skyvern_context.set(context)+ try:+ async with _content_page(_OFFVIEWPORT_UNNAMED_HTML) as page:+ tools = build_browser_tools(_fixed_page_provider(page))+ r = await _tool(tools, "observe").handler({})+ finally:+ skyvern_context.reset()+ assert r.status == "ok" and r.data is not None, r.content+ return r+++@_skip_no_browser+async def test_observe_drops_an_unnamed_control_no_scroll_can_bring_into_the_viewport(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ r = await _observe_offviewport_fixture(monkeypatch, arm="treatment")++ assert "button/submit ''" not in r.content, r.content+ assert "button/reset ''" not in r.content, r.content+ assert "div/option ''" not in r.content, r.content+ assert "Close notice" in r.content, r.content+ assert re.search(r"ref=\S+ a ''", r.content), r.content+ for kept in ("div/checkbox", "div/switch", "div/menuitem", "div/radio", "input/file", "div/spinbutton"):+ assert re.search(rf"ref=\S+ {kept} ''", r.content), (kept, r.content)+ assert "Paris" in r.content, r.content+ assert "(13 interactive elements)" in r.content, r.content+ assert r.data["summary"]["hidden_dropped_off_viewport"] == 3, r.data["summary"]+ assert r.data["summary"]["off_viewport_unreachable_unnamed"] == 3, r.data["summary"]+ # The closed-root host's controls are kept and counted apart, not as at-risk candidates.+ assert r.data["summary"]["off_viewport_unnamed_host_exempt"] == 2, r.data["summary"]+ assert r.data["summary"]["hidden_dropped"] == 3, r.data["summary"]+++async def _observe_records(page: Any, arm: RunArm) -> dict[str, Any]:+ from skyvern.forge.taskv3.tools import observe_js # noqa: PLC0415++ context = skyvern_context.SkyvernContext(+ run_arms={"TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED": ("wr_offviewport", arm)}+ )+ skyvern_context.set(context)+ try:+ return json.loads(await page.evaluate(observe_js()))+ finally:+ skyvern_context.reset()+++@_skip_no_browser+async def test_the_drop_never_removes_a_control_whose_record_reports_state(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # The property, over whatever the page holds: a control the arm removes must not be one whose+ # record carries state. Asserting the class rather than the three forms a field list happens to+ # enumerate -- that list is what drifted before (SKY-16501).+ monkeypatch.setattr(settings, "TASK_V3_OBSERVE_DROP_OFFVIEWPORT_UNNAMED", False)+ async with _content_page(_OFFVIEWPORT_UNNAMED_HTML) as page:+ listed_off = await _observe_records(page, "control")+ listed_on = await _observe_records(page, "treatment")++ def _identity(e: dict[str, Any]) -> str:+ return str(e.get("selector") or f"i={e.get('i')}")++ state_keys = ("value", "checked", "selected", "selectedOptions")+ stateful = {_identity(e) for e in listed_off["elements"] if any(k in e for k in state_keys)}+ kept = {_identity(e) for e in listed_on["elements"]}+ dropped = {_identity(e) for e in listed_off["elements"]} - kept++ assert dropped, listed_off["elements"]+ assert stateful & dropped == set(), sorted(stateful & dropped)+++@_skip_no_browser[email protected]("arm", ["control", "unrandomized"])+async def test_observe_lists_unnamed_offviewport_controls_outside_treatment(+ monkeypatch: pytest.MonkeyPatch, arm: RunArm+) -> None:+ r = await _observe_offviewport_fixture(monkeypatch, arm=arm)++ assert "button/submit ''" in r.content, r.content+ assert "button/reset ''" in r.content, r.content+ assert "div/option ''" in r.content, r.content+ assert "(16 interactive elements)" in r.content, r.content+ assert r.data["summary"]["hidden_dropped_off_viewport"] == 0, r.data["summary"]+ # Exposure is measured in every arm, so treatment and control can be compared on the same set.+ assert r.data["summary"]["off_viewport_unreachable_unnamed"] == 3, r.data["summary"]+ assert r.data["summary"]["off_viewport_unnamed_host_exempt"] == 2, r.data["summary"]