Skip to content

Automate browser based workflows with AI

Say a bank payment is still clearing and send canceled subscriptions to checkout (SKY-17328) (#8782)

Shuchang Zhengff9bae8b6d05
diff --git a/skyvern/config.py b/skyvern/config.py
index e8f7e1e55..ddbf6602a 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -547,6 +547,10 @@ class Settings(BaseSettings):
# year-only segment groups to the segment path (SKY-17013). Force-on term only: runs are randomized per
# run by the flag of the same name, read through run_arm_enabled(DATE_SEGMENT_AIM_FLAG, ...).
TASK_V3_DATE_SEGMENT_AIM: bool = False
+ # Move the pointer onto an input or click target before acting, as v1 does, and take the click and typing
+ # pre-snapshots without writing an attribute to every visible element. Force-on term only: runs are randomized per run by the
+ # flag of the same name, read through run_arm_enabled(POINTER_PARITY_FLAG, ...).
+ TASK_V3_POINTER_PARITY: bool = False
# Render the previous block's outcome (status / finish reason / final URL) and whether this is the
# last block into a v3 block's goal. Costs prompt tokens on every turn of the block, so it is
# measured via taskv3_block_context_tokens before it earns default-on. The outcome itself is
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 3ab1a0e73..0c1694751 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -152,6 +152,7 @@ from skyvern.forge.sdk.core.skyvern_context import (
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.exceptions import NotFoundError
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
@@ -198,6 +199,7 @@ from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
GOAL_CHECK_ENFORCE_FLAG,
GOAL_CHECK_FLAG,
+ POINTER_PARITY_FLAG,
resolve_run_arm,
run_arm_enabled,
)
@@ -2329,6 +2331,20 @@ class ForgeAgent:
organization_id=task.organization_id,
forced=settings.TASK_V3_DATE_SEGMENT_AIM,
)
+ await resolve_run_arm(
+ context,
+ POINTER_PARITY_FLAG,
+ distinct_id=task.workflow_run_id or task.task_id,
+ organization_id=task.organization_id,
+ forced=settings.TASK_V3_POINTER_PARITY,
+ properties={"workflow_permanent_id": task.workflow_permanent_id or context.workflow_permanent_id or ""},
+ )
+ if run_arm_enabled(POINTER_PARITY_FLAG, settings.TASK_V3_POINTER_PARITY):
+ # The dose: the run's registered cursor strategy draws the moves, or a plain move does without one.
+ LOG.info(
+ "Task V3 pointer parity cursor",
+ cursor_strategy=type(EventStrategyFactory.get_cursor_strategy()).__name__,
+ )
# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.
goal_judge_shot: list[bytes] = []
page_free_validation = bool(
diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.py
index bd493aab1..e0cac3e13 100644
--- a/skyvern/forge/taskv3/input_dispatch.py
+++ b/skyvern/forge/taskv3/input_dispatch.py
@@ -1,14 +1,23 @@
"""The one place Task V3 sends pointer and keyboard input to a page.
-Each gesture runs the Playwright call each tool made before this module existed, argument for argument.
+Control runs the Playwright call each tool made before this module existed, argument for argument. The
+TASK_V3_POINTER_PARITY treatment first moves the pointer onto the input or click target, as v1's action setup does.
"""
from __future__ import annotations
from typing import Any, TypeAlias, cast
+import structlog
from playwright.async_api import ElementHandle, FileChooser, Frame, Locator, Page
+from skyvern.config import settings
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
+from skyvern.forge.taskv3.run_arms import POINTER_PARITY_FLAG, run_arm_enabled
+from skyvern.webeye.browser_object_predicates import is_page_like
+
+LOG = structlog.get_logger()
+
Realm: TypeAlias = Page | Frame
Target: TypeAlias = str | Locator | ElementHandle
@@ -18,6 +27,26 @@ def _opts(**options: Any) -> dict[str, Any]:
return {key: value for key, value in options.items() if value is not None}
+def parity() -> bool:
+ """Whether this run is on the TASK_V3_POINTER_PARITY treatment."""
+ return run_arm_enabled(POINTER_PARITY_FLAG, settings.TASK_V3_POINTER_PARITY)
+
+
+async def _pointer_onto(realm: Realm, target: Target, timeout: float | None) -> None:
+ """Treatment: move the pointer onto ``target`` through the run's cursor strategy, as v1 does before it
+ types or clicks. The strategy's own position then ends on the target the native gesture lands on."""
+ if not parity():
+ return
+ try:
+ page = cast(Page, realm) if is_page_like(realm) else cast(Frame, realm).page
+ locator = realm.locator(target).first if isinstance(target, str) else target
+ await locator.scroll_into_view_if_needed(timeout=min(2000, timeout or 2000))
+ # An ElementHandle answers bounding_box() as a Locator does, which is all the strategies read.
+ await EventStrategyFactory.move_to_element(page, cast(Locator, locator))
+ except Exception:
+ LOG.info("taskv3 pointer move skipped", exc_info=True)
+
+
async def click(
realm: Realm,
target: str | Locator,
@@ -27,6 +56,7 @@ async def click(
position: dict[str, float] | None = None,
) -> None:
options = _opts(timeout=timeout, force=force, position=position)
+ await _pointer_onto(realm, target, timeout)
if isinstance(target, str):
await realm.click(target, **options)
else:
@@ -41,14 +71,21 @@ async def click_handle(
force: bool | None = None,
position: dict[str, float] | None = None,
) -> None:
+ await _pointer_onto(realm, handle, timeout)
await handle.click(**_opts(timeout=timeout, force=force, position=position))
async def click_at(page: Page, x: float, y: float) -> None:
+ if parity():
+ try:
+ await EventStrategyFactory.move_cursor(page, x, y)
+ except Exception:
+ LOG.info("taskv3 pointer move skipped", exc_info=True)
await page.mouse.click(x, y)
async def hover(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
+ await _pointer_onto(realm, selector, timeout)
await realm.hover(selector, **_opts(timeout=timeout))
@@ -71,6 +108,7 @@ async def focus(realm: Realm, target: str | Locator, *, timeout: float | None =
async def fill(realm: Realm, target: Target, value: str, *, timeout: float | None = None) -> None:
+ await _pointer_onto(realm, target, timeout)
if isinstance(target, str):
await realm.fill(target, value, **_opts(timeout=timeout))
else:
@@ -78,6 +116,7 @@ async def fill(realm: Realm, target: Target, value: str, *, timeout: float | Non
async def clear(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
+ await _pointer_onto(realm, selector, timeout)
await realm.fill(selector, "", **_opts(timeout=timeout))
@@ -92,6 +131,8 @@ async def type_keys(
"""Key events for ``text``. ``target`` None types into whatever holds focus, with no focus of its own;
``realm`` is then the page."""
options = _opts(delay=delay, timeout=timeout)
+ if target is not None:
+ await _pointer_onto(realm, target, timeout)
if target is None:
await cast(Page, realm).keyboard.type(text, **options)
elif isinstance(target, str):
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index 29abc3e39..b94f91780 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -19,10 +19,11 @@ LOG = structlog.get_logger()
DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"
GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"
GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"
+POINTER_PARITY_FLAG = "TASK_V3_POINTER_PARITY"
# Person properties a flag is evaluated with beyond organization_id; resolve_run_arm drops any other key. The
# PostHog preflight (scripts/check_run_arm_flags.py) reads this mapping to accept release conditions on them.
-RUN_ARM_EXTRA_PROPERTIES: dict[str, tuple[str, ...]] = {}
+RUN_ARM_EXTRA_PROPERTIES: dict[str, tuple[str, ...]] = {POINTER_PARITY_FLAG: ("workflow_permanent_id",)}
def _pinned_arm(context: skyvern_context.SkyvernContext, flag: str, distinct_id: str) -> RunArm | None:
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 5279b5bce..cd41cd462 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -1864,8 +1864,11 @@ _PIERCED_QUERY_JS = (
// a static list as a reaction. The WeakSet is the best carrier that costs no mutation, at one
// disclosed price -- a component that re-creates its own content by cloning reads as all-new.
// Absent (a navigation cleared window) means "no snapshot", never "everything is new".
- const preMark = (el, inShadow) => {
+ // A run that must write nothing (weak) keeps the light DOM in the WeakSet too, at the same disclosed
+ // price: a container the page re-creates by cloning reads as all-new.
+ const preMark = (el, inShadow, weak) => {
if (inShadow) window.__tv3_pre.add(el);
+ else if (weak) window.__tv3_pre.add(el);
else el.setAttribute('data-tv3-pre', '1');
};
// instanceof, not truthiness: a page that pre-defines __tv3_pre as an accessor keeps its own
@@ -1931,17 +1934,23 @@ _INSTRUCTION_CLAUSE_RE = re.compile(
)
_PRESNAPSHOT_JS = (
- r"""() => {"""
+ r"""(weak) => {"""
+ _PIERCED_QUERY_JS
+ r"""
preReset();
pScopeEach((el, inShadow) => {
const r = el.getBoundingClientRect();
- if (r.width > 0 && r.height > 0) preMark(el, inShadow);
+ if (r.width > 0 && r.height > 0) preMark(el, inShadow, !!weak);
});
}"""
)
+
+def _pre_weak() -> bool:
+ """Whether this run's pre-snapshots use only the WeakSet carrier, writing no attribute to the page."""
+ return input_dispatch.parity()
+
+
# Behavioral, site-agnostic suggestion finder. After the caller types a value (with a pre-snapshot taken
# first), this looks for the suggestion list the typeahead rendered IN REACTION: a small, visible,
# leaf-ish row that did NOT exist/show before typing (not `data-tv3-pre`), sits in the dropdown region
@@ -2591,7 +2600,7 @@ _FOCUS_SNAPSHOT_JS = (
if (preHas(el) && !inOpenOwn) return;
const r = el.getBoundingClientRect();
if (r.width <= 0 || r.height <= 0) return;
- if (inOptionList(el)) focusMark(el, inShadow); else preMark(el, inShadow);
+ if (inOptionList(el)) focusMark(el, inShadow); else preMark(el, inShadow, !!arg.weak);
});
// Record what the list offered NOW: a widget that filters by re-rendering unmounts the rows the
// typed value does not match, so a later read would find nothing to name on an honest no-match.
@@ -3595,7 +3604,8 @@ _WATCH_REWRITTEN_ROWS_JS = r"""() => {
const row = t.closest(ROW);
if (row) {
unmark(row);
- row.querySelectorAll('[data-tv3-pre]').forEach(unmark);
+ // Every descendant, not only stamped ones: a run that writes nothing carries its marks in the WeakSet.
+ row.querySelectorAll('*').forEach(unmark);
} else if (rec.type === 'childList' && t.matches(LIST)) {
t.querySelectorAll(ROW).forEach(unmark);
}
@@ -6716,7 +6726,7 @@ _CLICK_PRECHECK_JS = (
}
}
preReset();
- pScopeEach((el, inShadow) => { if (vis(el)) preMark(el, inShadow); });
+ pScopeEach((el, inShadow) => { if (vis(el)) preMark(el, inShadow, !!arg.weak); });
return { menuOpen: openRows.length > 0, isOption, containsMenu, optText, optState, optSel, optKids, optH, optVis };
}"""
)
@@ -13307,6 +13317,9 @@ def build_browser_tools(
element = None
return {"sel": selector, "el": element}
+ async def _snapshot_arg(page: Any, selector: str) -> dict[str, Any]:
+ return {**await _probe_arg(page, selector), "weak": _pre_weak()}
+
def _with_target_label(handler: ToolHandler) -> ToolHandler:
"""Capture the target's page-visible name and kind BEFORE the handler runs and carry both on
the result.
@@ -13768,7 +13781,7 @@ def build_browser_tools(
await _require_single_target(page, selector)
pre: dict[str, Any] | None = None
try:
- pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))
+ pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _snapshot_arg(page, selector))
if isinstance(pre_raw, dict):
pre = pre_raw
except Exception:
@@ -15302,7 +15315,7 @@ def build_browser_tools(
# of this attempt, and ownCommittedSurface reads nothing while it is open.
pre_own = await _own_surface_text(page, selector)
try:
- await page.evaluate(_PRESNAPSHOT_JS)
+ await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
except Exception:
presnapshot_ok = False
LOG.info("taskv3 typeahead pre-snapshot failed; skipping suggestion probe", selector=selector)
@@ -15317,7 +15330,7 @@ def build_browser_tools(
# Focus may reveal help text or a validation note as well as a menu; only rows of a list
# are a reaction the finder may pick from, so everything else focus revealed is marked too.
try:
- await page.evaluate(_FOCUS_SNAPSHOT_JS, await _probe_arg(page, selector))
+ await page.evaluate(_FOCUS_SNAPSHOT_JS, await _snapshot_arg(page, selector))
except Exception:
pass
# `query` is what the caller chose to search with; the row committed must still be `value` exactly.
@@ -15391,7 +15404,7 @@ def build_browser_tools(
# The rows showing now are the widget's unfiltered list, not an answer to the query: only rows
# the search renders (or rewrites) may count.
try:
- await page.evaluate(_PRESNAPSHOT_JS)
+ await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
await page.evaluate(_WATCH_REWRITTEN_ROWS_JS)
except Exception:
return _settled(
@@ -16947,7 +16960,7 @@ def build_browser_tools(
async def _open_and_enumerate() -> tuple[dict[str, Any] | None, ToolResult | None]:
try:
- await page.evaluate(_PRESNAPSHOT_JS)
+ await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
except Exception:
return None, ToolResult.error(
f"could not snapshot the page to open {selector}'s option list — the field is NOT "
@@ -18007,7 +18020,7 @@ def build_browser_tools(
# reaction": the snapshot from before the first attempt would mark the rows the
# PREVIOUS query left on screen as pre-existing, and a widget that keeps its row
# nodes across a re-search would then have no reaction to show at all.
- await page.evaluate(_PRESNAPSHOT_JS)
+ await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
await input_dispatch.type_keys(page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung))
except Exception:
return []
@@ -18065,7 +18078,7 @@ def build_browser_tools(
return
await asyncio.sleep(0.4)
try:
- await page.evaluate(_FOCUS_SNAPSHOT_JS, await _probe_arg(page, selector))
+ await page.evaluate(_FOCUS_SNAPSHOT_JS, await _snapshot_arg(page, selector))
except Exception:
LOG.debug("taskv3 empty-query offer snapshot failed", selector=selector)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index c9d55b2b8..0ddca7218 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -80,6 +80,7 @@ from skyvern.forge.taskv3.loop import (
)
from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
+ POINTER_PARITY_FLAG,
run_arm_enabled,
)
from skyvern.forge.taskv3.tools import PageProvider, _record_frame_work
@@ -327,6 +328,35 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa
)
+async def test_execute_task_v3_buckets_the_pointer_parity_arm_per_run_with_the_workflow(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The arm is targeted at named workflows, so the resolve must carry the workflow id, or no group matches.
+ monkeypatch.setattr(settings, "TASK_V3_POINTER_PARITY", False)
+ provider = AsyncMock(return_value="treatment")
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)
+
+ _step, task, loop_mock, _post = await _run_execute_task_v3(
+ monkeypatch,
+ LoopOutcome(status="completed", reason="done", billable_actions=[]),
+ workflow_run_id="wr_pointer_parity",
+ context_overrides={"workflow_permanent_id": "wpid_pointer_parity"},
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ assert loop_mock.context.run_arms[POINTER_PARITY_FLAG] == (task.workflow_run_id, "treatment")
+ provider.assert_any_await(
+ POINTER_PARITY_FLAG,
+ task.workflow_run_id,
+ properties={
+ "organization_id": task.organization_id,
+ "workflow_permanent_id": task.workflow_permanent_id or "wpid_pointer_parity",
+ },
+ )
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("workflow_system_prompt", ["Always use formal salutations.", None])
async def test_execute_task_v3_passes_the_workflow_system_prompt_unlabelled(
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index 6b5e5cf51..cda563780 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -82,6 +82,7 @@ _DISCOVERED_BROWSER_API_CALLS = {
"hover": 1,
"press": 3,
"press_sequentially": 1,
+ "scroll_into_view_if_needed": 1,
"select_option": 2,
"set_files": 1,
"set_input_files": 1,
@@ -259,14 +260,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 206
+ assert sum(sum(methods.values()) for methods in observed.values()) == 207
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 7
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 199
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 200
def test_every_raw_evaluate_call_is_classified() -> None:
diff --git a/tests/unit/test_taskv3_input_dispatch.py b/tests/unit/test_taskv3_input_dispatch.py
index 8eb848f6b..739335fd0 100644
--- a/tests/unit/test_taskv3_input_dispatch.py
+++ b/tests/unit/test_taskv3_input_dispatch.py
@@ -93,3 +93,47 @@ async def test_an_element_handle_is_clicked_with_only_the_arguments_given() -> N
await input_dispatch.click_handle(_FakePage(), handle)
handle.click.assert_awaited_once_with()
+
+
[email protected]("treatment", [False, True])
+async def test_pointer_parity_moves_the_pointer_onto_an_input_or_click_target_first(
+ monkeypatch: pytest.MonkeyPatch, treatment: bool
+) -> None:
+ # v1 moves the pointer onto every input before typing and onto every click target before pressing; a v3 run
+ # on the TASK_V3_POINTER_PARITY treatment must too. Control keeps only Playwright's own move into the click.
+ async_playwright = pytest.importorskip("playwright.async_api").async_playwright
+ monkeypatch.setattr(input_dispatch.settings, "TASK_V3_POINTER_PARITY", treatment)
+ async with async_playwright() as pw:
+ try:
+ browser = await pw.chromium.launch(headless=True)
+ except Exception:
+ pytest.skip("Requires Playwright browsers installed")
+ try:
+ page = await browser.new_page(viewport={"width": 800, "height": 600})
+ await page.set_content(
+ '<input id="user" style="position:absolute;top:100px;left:100px;width:200px;height:30px">'
+ '<input id="pass" type="password" style="position:absolute;top:900px;left:100px;width:200px;height:30px">'
+ '<button id="go" style="position:absolute;top:960px;left:100px;width:200px;height:40px">Go</button>'
+ "<script>window.ev = []; for (const t of ['mousemove', 'input', 'pointerdown'])"
+ " document.addEventListener(t, (e) => window.ev.push([t, e.target.id || '', e.isTrusted]), true);"
+ "</script>"
+ )
+ await input_dispatch.fill(page, "#user", "alice", timeout=2000)
+ await input_dispatch.fill(page, page.locator("#pass"), "secret", timeout=2000)
+ # Off-centre on purpose: the press lands somewhere other than where the approach move ends.
+ await input_dispatch.click(page, "#go", timeout=2000, position={"x": 3, "y": 3})
+ events = await page.evaluate("() => window.ev")
+ finally:
+ await browser.close()
+ press = events.index(["pointerdown", "go", True])
+ moves_onto_button = events[:press].count(["mousemove", "go", True])
+ if treatment:
+ # The move lands on each field, below the fold too, before its first input event.
+ for field in ("user", "pass"):
+ first_input = events.index(["input", field, True])
+ assert ["mousemove", field, True] in events[:first_input], events
+ assert moves_onto_button >= 2, events
+ else:
+ assert all(kind != "mousemove" for kind, target, _ in events[:press] if target != "go"), events
+ assert moves_onto_button == 1, events
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index a0fa9c3e9..ebd330a41 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -17996,6 +17996,112 @@ async def test_pre_snapshot_survives_a_container_the_page_rebuilds_by_cloning()
assert await page.evaluate(_FIND_MENU_JS, {"sel": "#sort", "el": None}) is None
+@_skip_no_browser
+async def test_pointer_parity_run_writes_no_snapshot_attribute_and_still_reads_the_click_reaction(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The treatment of TASK_V3_POINTER_PARITY must not stamp every visible element just before a click, and
+ # must keep what that stamp is for: a menu the click opens reads as a reaction, a static list does not.
+ import skyvern.forge.taskv3.tools as tools_module # noqa: PLC0415
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ monkeypatch.setattr(tools_module.settings, "TASK_V3_POINTER_PARITY", True)
+ async with _live_page(
+ """<input id="user" type="text" style="position:absolute;top:10px;left:10px;width:200px;height:24px">
+ <button id="sort" style="position:absolute;top:10px;left:240px;width:120px;height:24px">Sort</button>
+ <div id="results" style="position:absolute;top:60px;left:10px;width:220px">
+ <a href="/a" role="option" style="display:block;height:20px">Result Alpha</a>
+ <a href="/b" role="option" style="display:block;height:20px">Result Beta</a>
+ <a href="/c" role="option" style="display:block;height:20px">Result Gamma</a>
+ </div>
+ <div id="menu" style="display:none;position:absolute;top:40px;left:240px;width:200px">
+ <div role="option" style="height:20px">Newest</div>
+ <div role="option" style="height:20px">Oldest</div>
+ <div role="option" style="height:20px">Relevance</div>
+ </div>
+ <script>
+ window.preWrites = 0;
+ new MutationObserver((recs) => {
+ for (const r of recs) if (r.attributeName === 'data-tv3-pre') window.preWrites++;
+ }).observe(document, {subtree: true, attributes: true});
+ </script>"""
+ ) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ assert (await _tool(tools, "type").handler({"selector": "#user", "text": "alice"})).status == "ok"
+ assert (await _tool(tools, "click").handler({"selector": "#sort"})).status == "ok"
+ assert await page.evaluate("() => window.preWrites") == 0
+
+ await page.evaluate(_CLICK_PRECHECK_JS, {"sel": "#sort", "el": None, "weak": True})
+ await page.evaluate("() => { document.getElementById('menu').style.display = 'block'; }")
+ found = await page.evaluate(_FIND_MENU_JS, {"sel": "#sort", "el": None})
+ assert isinstance(found, dict), found
+ assert [o["text"] for o in found["options"]] == ["Newest", "Oldest", "Relevance"]
+ assert await page.evaluate("() => window.preWrites") == 0
+
+
+@_skip_no_browser
+async def test_pointer_parity_click_still_reports_a_menu_its_hover_opens(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The treatment's pointer move must land after the click's baseline: a menu that opens on mouseenter is
+ # the click's reaction in control, and must stay one, or the model needs another look to get its rows.
+ import skyvern.forge.taskv3.tools as tools_module # noqa: PLC0415
+
+ monkeypatch.setattr(tools_module.settings, "TASK_V3_POINTER_PARITY", True)
+ async with _live_page(
+ """<button id="acct" style="position:absolute;top:10px;left:300px;width:120px;height:24px">Account</button>
+ <div id="m" role="menu" style="display:none;position:absolute;top:36px;left:300px;width:160px">
+ <div role="menuitem" style="height:20px">Profile</div>
+ <div role="menuitem" style="height:20px">Billing</div>
+ <div role="menuitem" style="height:20px">Sign out</div>
+ </div>
+ <script>
+ document.getElementById('acct').addEventListener('mouseenter', () => {
+ document.getElementById('m').style.display = 'block';
+ });
+ </script>"""
+ ) as page:
+ await page.mouse.move(5, 500)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#acct"})
+ assert r.status == "ok", r.content
+ assert "This click opened a menu" in r.content, r.content
+
+
+@_skip_no_browser
+async def test_write_free_snapshot_reads_existed_before_by_node_identity() -> None:
+ # The write-free carrier is node identity, the WeakSet the shadow-DOM half already used: a node that was
+ # visible reads as old, and any node created since -- a clone included, the disclosed price -- or a row
+ # the rewrite watcher saw change, down to its descendants, reads as a reaction.
+ from skyvern.forge.taskv3.tools import ( # noqa: PLC0415
+ _PIERCED_QUERY_JS,
+ _PRESNAPSHOT_JS,
+ _WATCH_REWRITTEN_ROWS_JS,
+ )
+
+ async with _live_page(
+ """<ul id="k" style="position:absolute;top:10px;left:10px;width:200px;margin:0">
+ <li style="height:20px">Kept</li></ul>
+ <ul id="a" style="position:absolute;top:40px;left:10px;width:200px;margin:0">
+ <li style="height:20px">Engineering</li></ul>
+ <ul id="c" style="position:absolute;top:80px;left:10px;width:200px;margin:0">
+ <li style="height:20px"><span>Paris</span></li></ul>"""
+ ) as page:
+ await page.evaluate(_PRESNAPSHOT_JS, True)
+ await page.evaluate(_WATCH_REWRITTEN_ROWS_JS)
+ await page.evaluate(
+ "() => { const a = document.getElementById('a'); a.parentNode.replaceChild(a.cloneNode(true), a);"
+ " document.querySelector('#c span').firstChild.nodeValue = 'Paris'; }"
+ )
+ await page.wait_for_timeout(50)
+ pre = await page.evaluate(
+ "() => {" + _PIERCED_QUERY_JS + " const q = (s) => preHas(document.querySelector(s));"
+ " return {kept: q('#k li'), clone: q('#a li'), rewritten: q('#c span'),"
+ " attrs: document.querySelectorAll('[data-tv3-pre]').length}; }"
+ )
+ assert pre == {"kept": True, "clone": False, "rewritten": False, "attrs": 0}, pre
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_reaction_gate_rejects_a_page_supplied_snapshot_impostor() -> None: