Skip to content

Automate browser based workflows with AI

Hold the Task V3 credential submit until 45 s into the block, behind a per-run flag, and delete the pointer treatment (#8812)

pedrohsdbf928fcb080c5
diff --git a/skyvern/config.py b/skyvern/config.py
index 269f11c9a..ee6453ceb 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -556,10 +556,9 @@ class Settings(BaseSettings):
# year-only segment groups to the segment path (SKY-17013). Force-on term only: runs are randomized per
# run by the flag of the same name, read through run_arm_enabled(DATE_SEGMENT_AIM_FLAG, ...).
TASK_V3_DATE_SEGMENT_AIM: bool = False
- # Move the pointer onto an input or click target before acting, as v1 does, and take the click and typing
- # pre-snapshots without writing an attribute to every visible element. Force-on term only: runs are randomized per run by the
- # flag of the same name, read through run_arm_enabled(POINTER_PARITY_FLAG, ...).
- TASK_V3_POINTER_PARITY: bool = False
+ # Hold the first click or Enter after a password fill until 45 s after Task V3 started the block. Force-on term
+ # only: runs are randomized per run by the flag of the same name, read through run_arm_enabled(LOGIN_PACE_FLAG, ...).
+ TASK_V3_LOGIN_PACE: bool = False
# Render the previous block's outcome (status / finish reason / final URL) and whether this is the
# last block into a v3 block's goal. Costs prompt tokens on every turn of the block, so it is
# measured via taskv3_block_context_tokens before it earns default-on. The outcome itself is
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 5226895ac..a5a2f2477 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -152,7 +152,6 @@ from skyvern.forge.sdk.core.skyvern_context import (
from skyvern.forge.sdk.db.datetime_utils import naive_utc_now
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.exceptions import NotFoundError
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.slim_llm_output import get_slim_output_template_value
@@ -187,6 +186,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import CredentialRelea
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSES
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.goal_check import (
BLOCK_COMPLETION_CHECK_PROMPT_NAME,
GOAL_CHECK_PROMPT_NAME,
@@ -201,7 +201,7 @@ from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
GOAL_CHECK_ENFORCE_FLAG,
GOAL_CHECK_FLAG,
- POINTER_PARITY_FLAG,
+ LOGIN_PACE_FLAG,
resolve_run_arm,
run_arm_enabled,
)
@@ -2347,18 +2347,12 @@ class ForgeAgent:
)
await resolve_run_arm(
context,
- POINTER_PARITY_FLAG,
+ LOGIN_PACE_FLAG,
distinct_id=task.workflow_run_id or task.task_id,
organization_id=task.organization_id,
- forced=settings.TASK_V3_POINTER_PARITY,
+ forced=settings.TASK_V3_LOGIN_PACE,
properties={"workflow_permanent_id": task.workflow_permanent_id or context.workflow_permanent_id or ""},
)
- if run_arm_enabled(POINTER_PARITY_FLAG, settings.TASK_V3_POINTER_PARITY):
- # The dose: the run's registered cursor strategy draws the moves, or a plain move does without one.
- LOG.info(
- "Task V3 pointer parity cursor",
- cursor_strategy=type(EventStrategyFactory.get_cursor_strategy()).__name__,
- )
# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.
goal_judge_shot: list[bytes] = []
page_free_validation = bool(
@@ -2532,6 +2526,8 @@ class ForgeAgent:
return True
return False
+ input_dispatch.start_login_pace(_should_cancel)
+
download_id = resolve_run_download_id(context, fallback_run_id=task.task_id)
attempt_started_at = await get_download_retry_started_at(
organization.organization_id, resolve_run_download_id(context, fallback_run_id=task.workflow_run_id)
@@ -3243,6 +3239,7 @@ class ForgeAgent:
code_typed_values=recovery_code_progress.typed_values if recovery_code_progress else (),
)
finally:
+ input_dispatch.end_login_pace()
if context and credential_parameter_key is not None:
context.active_credential_parameter_key = prev_active_credential_parameter_key
# Frames are already in memory, so a loop that raised or ran out of budget still
diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.py
index e0cac3e13..05fbefdf0 100644
--- a/skyvern/forge/taskv3/input_dispatch.py
+++ b/skyvern/forge/taskv3/input_dispatch.py
@@ -1,19 +1,24 @@
"""The one place Task V3 sends pointer and keyboard input to a page.
-Control runs the Playwright call each tool made before this module existed, argument for argument. The
-TASK_V3_POINTER_PARITY treatment first moves the pointer onto the input or click target, as v1's action setup does.
+Each gesture runs the Playwright call each tool made before this module existed, argument for argument. The
+TASK_V3_LOGIN_PACE treatment holds the first click or Enter after a password fill until LOGIN_PACE_FLOOR_S after Task V3
+started the block.
"""
from __future__ import annotations
+import asyncio
+import time
+from collections.abc import Awaitable, Callable
+from contextvars import ContextVar
+from dataclasses import dataclass
from typing import Any, TypeAlias, cast
import structlog
from playwright.async_api import ElementHandle, FileChooser, Frame, Locator, Page
from skyvern.config import settings
-from skyvern.forge.sdk.event.factory import EventStrategyFactory
-from skyvern.forge.taskv3.run_arms import POINTER_PARITY_FLAG, run_arm_enabled
+from skyvern.forge.taskv3.run_arms import LOGIN_PACE_FLAG, run_arm_enabled
from skyvern.webeye.browser_object_predicates import is_page_like
LOG = structlog.get_logger()
@@ -21,30 +26,168 @@ LOG = structlog.get_logger()
Realm: TypeAlias = Page | Frame
Target: TypeAlias = str | Locator | ElementHandle
+LOGIN_PACE_FLOOR_S = 45.0
+_CANCEL_POLL_S = 2.0
+
+
+class LoginPaceRefused(RuntimeError):
+ """The task was canceled, or the page moved, during the wait, so the held gesture is not sent."""
+
+
+@dataclass(frozen=True)
+class SecretFillAnchor:
+ top_url: str | None
+ frame: Frame | None
+ frame_url: str | None
+
+
+@dataclass
+class _LoginPace:
+ origin: float
+ should_cancel: Callable[[], Awaitable[bool]] | None = None
+ anchor: SecretFillAnchor | None = None
+ navigated: bool = False
+ unsubscribe: Callable[[], None] | None = None
+ done: bool = False
+ canceled: bool = False
+
+
+_login_pace: ContextVar[_LoginPace | None] = ContextVar("taskv3_login_pace", default=None)
-def _opts(**options: Any) -> dict[str, Any]:
- # Only what the caller passed, so each call carries exactly the keywords the tool always sent.
- return {key: value for key, value in options.items() if value is not None}
+def top_url(realm: Realm) -> str | None:
+ try:
+ return cast(Page, realm if is_page_like(realm) else cast(Frame, realm).page).url
+ except Exception:
+ return None
+
+
+def start_login_pace(should_cancel: Callable[[], Awaitable[bool]] | None = None) -> None:
+ """Start the block's clock at Task V3 start, after the block's navigation has landed (about 10 s after the block
+ is created). Later pages of the same login run on the same clock."""
+ on = run_arm_enabled(LOGIN_PACE_FLAG, settings.TASK_V3_LOGIN_PACE)
+ _login_pace.set(_LoginPace(origin=time.monotonic(), should_cancel=should_cancel) if on else None)
+
+
+def secret_fill_anchor(realm: Realm) -> SecretFillAnchor:
+ """Read before the fill, so a fill that moves the page or the frame does not move the anchor."""
+ frame = None if is_page_like(realm) else cast(Frame, realm)
+ return SecretFillAnchor(top_url(realm), frame, frame.url if frame is not None else None)
-def parity() -> bool:
- """Whether this run is on the TASK_V3_POINTER_PARITY treatment."""
- return run_arm_enabled(POINTER_PARITY_FLAG, settings.TASK_V3_POINTER_PARITY)
+def _unsubscribe(pace: _LoginPace) -> None:
+ if pace.unsubscribe is not None:
+ pace.unsubscribe()
+ pace.unsubscribe = None
-async def _pointer_onto(realm: Realm, target: Target, timeout: float | None) -> None:
- """Treatment: move the pointer onto ``target`` through the run's cursor strategy, as v1 does before it
- types or clicks. The strategy's own position then ends on the target the native gesture lands on."""
- if not parity():
+
+def note_secret_fill(realm: Realm, anchor: SecretFillAnchor) -> None:
+ pace = _login_pace.get()
+ if pace is None or pace.done:
return
+ _unsubscribe(pace)
+ pace.anchor, pace.navigated = anchor, False
+ try:
+ page = cast(Page, realm if is_page_like(realm) else cast(Frame, realm).page)
+
+ # A same-URL reload keeps both URLs, so only the event shows it; a same-URL history write also fires it.
+ def _on_navigated(frame: Frame) -> None:
+ if frame is anchor.frame or frame.parent_frame is None:
+ pace.navigated = True
+
+ page.on("framenavigated", _on_navigated)
+ pace.unsubscribe = lambda: page.remove_listener("framenavigated", _on_navigated)
+ except Exception:
+ LOG.warning("Task V3 login pace could not watch for a reload", exc_info=True)
+
+
+def _moved_by(pace: _LoginPace, anchor: SecretFillAnchor, realm: Realm) -> str | None:
+ """Which check saw the password's document move, or None. An unreadable URL counts as moved."""
+ url = top_url(realm)
+ if url is None or url != anchor.top_url:
+ return "top_url"
+ if anchor.frame is not None:
+ try:
+ if anchor.frame.is_detached():
+ return "frame_detached"
+ if anchor.frame.url != anchor.frame_url:
+ return "frame_url"
+ except Exception:
+ return "frame_unreadable"
+ return "navigated_event" if pace.navigated else None
+
+
+def _not_held(pace: _LoginPace, reason: str, moved_by: str | None = None) -> None:
+ pace.done = True
+ _unsubscribe(pace)
+ LOG.info(
+ "Task V3 login pace submit not held",
+ reason=reason,
+ nav_s=round(time.monotonic() - pace.origin, 1),
+ **({"moved_by": moved_by} if moved_by else {}),
+ )
+
+
+async def _cancel_seen(pace: _LoginPace) -> bool:
try:
- page = cast(Page, realm) if is_page_like(realm) else cast(Frame, realm).page
- locator = realm.locator(target).first if isinstance(target, str) else target
- await locator.scroll_into_view_if_needed(timeout=min(2000, timeout or 2000))
- # An ElementHandle answers bounding_box() as a Locator does, which is all the strategies read.
- await EventStrategyFactory.move_to_element(page, cast(Locator, locator))
+ return pace.should_cancel is not None and await pace.should_cancel()
except Exception:
- LOG.info("taskv3 pointer move skipped", exc_info=True)
+ # A failed status read must not refuse a healthy submit; the loop's own poll still catches the cancel.
+ return False
+
+
+async def _hold_submit(realm: Realm) -> None:
+ pace = _login_pace.get()
+ if pace is not None and pace.canceled:
+ # Sticky: a caller that swallows the refusal and sends another gesture must not get the credentials out.
+ raise LoginPaceRefused("the task was canceled during the login wait; the gesture was not sent")
+ if pace is None or pace.done or (anchor := pace.anchor) is None:
+ return
+ if moved_by := _moved_by(pace, anchor, realm):
+ # The page left the password page before any click or Enter reached here: something else submitted.
+ _not_held(pace, "page_moved_before_submit", moved_by)
+ return
+ waited = max(0.0, LOGIN_PACE_FLOOR_S - (time.monotonic() - pace.origin))
+ deadline = time.monotonic() + waited
+ try:
+ # Sliced so a run canceled, or a page that moved, during the wait never gets the held gesture.
+ while (left := deadline - time.monotonic()) > 0:
+ await asyncio.sleep(min(left, _CANCEL_POLL_S))
+ if await _cancel_seen(pace):
+ pace.canceled = True
+ _not_held(pace, "canceled_during_wait")
+ raise LoginPaceRefused("the task was canceled during the login wait; the gesture was not sent")
+ if moved_by := _moved_by(pace, anchor, realm):
+ _not_held(pace, "page_moved_during_wait", moved_by)
+ raise LoginPaceRefused(
+ "the page changed during the login wait, so the held click or key was not sent; "
+ "observe the page again"
+ )
+ except asyncio.CancelledError:
+ # A caller's timeout (the captcha solver's checkbox click) must not use up the hold the submit needs.
+ LOG.info("Task V3 login pace dwell interrupted", nav_s=round(time.monotonic() - pace.origin, 1))
+ raise
+ pace.done = True
+ _unsubscribe(pace)
+ LOG.info(
+ "Task V3 login pace dwell",
+ waited_s=round(waited, 1),
+ nav_to_submit_s=round(time.monotonic() - pace.origin, 1),
+ )
+
+
+def end_login_pace() -> None:
+ pace = _login_pace.get()
+ if pace is not None and pace.anchor is not None and not pace.done:
+ _not_held(pace, "no_click_or_enter_after_password")
+ if pace is not None:
+ _unsubscribe(pace)
+ _login_pace.set(None)
+
+
+def _opts(**options: Any) -> dict[str, Any]:
+ # Only what the caller passed, so each call carries exactly the keywords the tool always sent.
+ return {key: value for key, value in options.items() if value is not None}
async def click(
@@ -56,7 +199,7 @@ async def click(
position: dict[str, float] | None = None,
) -> None:
options = _opts(timeout=timeout, force=force, position=position)
- await _pointer_onto(realm, target, timeout)
+ await _hold_submit(realm)
if isinstance(target, str):
await realm.click(target, **options)
else:
@@ -71,26 +214,22 @@ async def click_handle(
force: bool | None = None,
position: dict[str, float] | None = None,
) -> None:
- await _pointer_onto(realm, handle, timeout)
+ await _hold_submit(realm)
await handle.click(**_opts(timeout=timeout, force=force, position=position))
async def click_at(page: Page, x: float, y: float) -> None:
- if parity():
- try:
- await EventStrategyFactory.move_cursor(page, x, y)
- except Exception:
- LOG.info("taskv3 pointer move skipped", exc_info=True)
+ await _hold_submit(page)
await page.mouse.click(x, y)
async def hover(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
- await _pointer_onto(realm, selector, timeout)
await realm.hover(selector, **_opts(timeout=timeout))
async def js_click(realm: Realm, root_query_js: str, arg: Any) -> Any:
"""Click a hidden native control from inside the page. No pointer events, as in v1."""
+ await _hold_submit(realm)
return await realm.evaluate(
"(arg) => { const _q = "
+ root_query_js
@@ -108,7 +247,6 @@ async def focus(realm: Realm, target: str | Locator, *, timeout: float | None =
async def fill(realm: Realm, target: Target, value: str, *, timeout: float | None = None) -> None:
- await _pointer_onto(realm, target, timeout)
if isinstance(target, str):
await realm.fill(target, value, **_opts(timeout=timeout))
else:
@@ -116,7 +254,6 @@ async def fill(realm: Realm, target: Target, value: str, *, timeout: float | Non
async def clear(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
- await _pointer_onto(realm, selector, timeout)
await realm.fill(selector, "", **_opts(timeout=timeout))
@@ -131,8 +268,6 @@ async def type_keys(
"""Key events for ``text``. ``target`` None types into whatever holds focus, with no focus of its own;
``realm`` is then the page."""
options = _opts(delay=delay, timeout=timeout)
- if target is not None:
- await _pointer_onto(realm, target, timeout)
if target is None:
await cast(Page, realm).keyboard.type(text, **options)
elif isinstance(target, str):
@@ -146,6 +281,8 @@ async def press(
) -> None:
"""A key press, raw as v1's keypress is. ``target`` None presses on ``realm``, the page's, keyboard."""
options = _opts(timeout=timeout)
+ if key.endswith("Enter"):
+ await _hold_submit(realm)
if target is None:
await cast(Page, realm).keyboard.press(key, **options)
elif isinstance(target, str):
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index b94f91780..612ee8ee3 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -19,11 +19,11 @@ LOG = structlog.get_logger()
DATE_SEGMENT_AIM_FLAG = "TASK_V3_DATE_SEGMENT_AIM"
GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"
GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"
-POINTER_PARITY_FLAG = "TASK_V3_POINTER_PARITY"
+LOGIN_PACE_FLAG = "TASK_V3_LOGIN_PACE"
# Person properties a flag is evaluated with beyond organization_id; resolve_run_arm drops any other key. The
# PostHog preflight (scripts/check_run_arm_flags.py) reads this mapping to accept release conditions on them.
-RUN_ARM_EXTRA_PROPERTIES: dict[str, tuple[str, ...]] = {POINTER_PARITY_FLAG: ("workflow_permanent_id",)}
+RUN_ARM_EXTRA_PROPERTIES: dict[str, tuple[str, ...]] = {LOGIN_PACE_FLAG: ("workflow_permanent_id",)}
def _pinned_arm(context: skyvern_context.SkyvernContext, flag: str, distinct_id: str) -> RunArm | None:
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index a50c9508e..4ab4ad436 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -1866,11 +1866,8 @@ _PIERCED_QUERY_JS = (
// a static list as a reaction. The WeakSet is the best carrier that costs no mutation, at one
// disclosed price -- a component that re-creates its own content by cloning reads as all-new.
// Absent (a navigation cleared window) means "no snapshot", never "everything is new".
- // A run that must write nothing (weak) keeps the light DOM in the WeakSet too, at the same disclosed
- // price: a container the page re-creates by cloning reads as all-new.
- const preMark = (el, inShadow, weak) => {
+ const preMark = (el, inShadow) => {
if (inShadow) window.__tv3_pre.add(el);
- else if (weak) window.__tv3_pre.add(el);
else el.setAttribute('data-tv3-pre', '1');
};
// instanceof, not truthiness: a page that pre-defines __tv3_pre as an accessor keeps its own
@@ -1936,23 +1933,17 @@ _INSTRUCTION_CLAUSE_RE = re.compile(
)
_PRESNAPSHOT_JS = (
- r"""(weak) => {"""
+ r"""() => {"""
+ _PIERCED_QUERY_JS
+ r"""
preReset();
pScopeEach((el, inShadow) => {
const r = el.getBoundingClientRect();
- if (r.width > 0 && r.height > 0) preMark(el, inShadow, !!weak);
+ if (r.width > 0 && r.height > 0) preMark(el, inShadow);
});
}"""
)
-
-def _pre_weak() -> bool:
- """Whether this run's pre-snapshots use only the WeakSet carrier, writing no attribute to the page."""
- return input_dispatch.parity()
-
-
# Behavioral, site-agnostic suggestion finder. After the caller types a value (with a pre-snapshot taken
# first), this looks for the suggestion list the typeahead rendered IN REACTION: a small, visible,
# leaf-ish row that did NOT exist/show before typing (not `data-tv3-pre`), sits in the dropdown region
@@ -2602,7 +2593,7 @@ _FOCUS_SNAPSHOT_JS = (
if (preHas(el) && !inOpenOwn) return;
const r = el.getBoundingClientRect();
if (r.width <= 0 || r.height <= 0) return;
- if (inOptionList(el)) focusMark(el, inShadow); else preMark(el, inShadow, !!arg.weak);
+ if (inOptionList(el)) focusMark(el, inShadow); else preMark(el, inShadow);
});
// Record what the list offered NOW: a widget that filters by re-rendering unmounts the rows the
// typed value does not match, so a later read would find nothing to name on an honest no-match.
@@ -3606,8 +3597,7 @@ _WATCH_REWRITTEN_ROWS_JS = r"""() => {
const row = t.closest(ROW);
if (row) {
unmark(row);
- // Every descendant, not only stamped ones: a run that writes nothing carries its marks in the WeakSet.
- row.querySelectorAll('*').forEach(unmark);
+ row.querySelectorAll('[data-tv3-pre]').forEach(unmark);
} else if (rec.type === 'childList' && t.matches(LIST)) {
t.querySelectorAll(ROW).forEach(unmark);
}
@@ -6754,7 +6744,7 @@ _CLICK_PRECHECK_JS = (
}
}
preReset();
- pScopeEach((el, inShadow) => { if (vis(el)) preMark(el, inShadow, !!arg.weak); });
+ pScopeEach((el, inShadow) => { if (vis(el)) preMark(el, inShadow); });
return { menuOpen: openRows.length > 0, isOption, containsMenu, optText, optState, optSel, optKids, optH, optVis };
}"""
)
@@ -13364,9 +13354,6 @@ def build_browser_tools(
element = None
return {"sel": selector, "el": element}
- async def _snapshot_arg(page: Any, selector: str) -> dict[str, Any]:
- return {**await _probe_arg(page, selector), "weak": _pre_weak()}
-
def _with_target_label(handler: ToolHandler) -> ToolHandler:
"""Capture the target's page-visible name and kind BEFORE the handler runs and carry both on
the result.
@@ -13828,7 +13815,7 @@ def build_browser_tools(
await _require_single_target(page, selector)
pre: dict[str, Any] | None = None
try:
- pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _snapshot_arg(page, selector))
+ pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))
if isinstance(pre_raw, dict):
pre = pre_raw
except Exception:
@@ -15362,7 +15349,7 @@ def build_browser_tools(
# of this attempt, and ownCommittedSurface reads nothing while it is open.
pre_own = await _own_surface_text(page, selector)
try:
- await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
+ await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
presnapshot_ok = False
LOG.info("taskv3 typeahead pre-snapshot failed; skipping suggestion probe", selector=selector)
@@ -15377,7 +15364,7 @@ def build_browser_tools(
# Focus may reveal help text or a validation note as well as a menu; only rows of a list
# are a reaction the finder may pick from, so everything else focus revealed is marked too.
try:
- await page.evaluate(_FOCUS_SNAPSHOT_JS, await _snapshot_arg(page, selector))
+ await page.evaluate(_FOCUS_SNAPSHOT_JS, await _probe_arg(page, selector))
except Exception:
pass
# `query` is what the caller chose to search with; the row committed must still be `value` exactly.
@@ -15451,7 +15438,7 @@ def build_browser_tools(
# The rows showing now are the widget's unfiltered list, not an answer to the query: only rows
# the search renders (or rewrites) may count.
try:
- await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
+ await page.evaluate(_PRESNAPSHOT_JS)
await page.evaluate(_WATCH_REWRITTEN_ROWS_JS)
except Exception:
return _settled(
@@ -16631,6 +16618,7 @@ def build_browser_tools(
maxlength: int | None = None
if text:
field_type, maxlength = await _field_shape(page, selector)
+ password_anchor = input_dispatch.secret_fill_anchor(page) if field_type == "password" else None
# One box of a field of one-character boxes cannot hold more than one character, so a longer text is
# either written one character per box and read back from every box, or not written at all.
box_text = maxlength == 1 and len(text) > 1
@@ -16759,6 +16747,8 @@ def build_browser_tools(
await field.dispose()
if not_held is not None:
return not_held
+ if password_anchor is not None:
+ input_dispatch.note_secret_fill(page, password_anchor)
if press_enter:
await input_dispatch.press(page, selector, "Enter")
return ToolResult.ok(f"typed into {selector}", data={"url_before": url_before})
@@ -17171,7 +17161,7 @@ def build_browser_tools(
async def _open_and_enumerate() -> tuple[dict[str, Any] | None, ToolResult | None]:
try:
- await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
+ await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
return None, ToolResult.error(
f"could not snapshot the page to open {selector}'s option list — the field is NOT "
@@ -18244,7 +18234,7 @@ def build_browser_tools(
# reaction": the snapshot from before the first attempt would mark the rows the
# PREVIOUS query left on screen as pre-existing, and a widget that keeps its row
# nodes across a re-search would then have no reaction to show at all.
- await page.evaluate(_PRESNAPSHOT_JS, _pre_weak())
+ await page.evaluate(_PRESNAPSHOT_JS)
await input_dispatch.type_keys(page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung))
except Exception:
return []
@@ -18302,7 +18292,7 @@ def build_browser_tools(
return
await asyncio.sleep(0.4)
try:
- await page.evaluate(_FOCUS_SNAPSHOT_JS, await _snapshot_arg(page, selector))
+ await page.evaluate(_FOCUS_SNAPSHOT_JS, await _probe_arg(page, selector))
except Exception:
LOG.debug("taskv3 empty-query offer snapshot failed", selector=selector)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 9af3c36ff..d865528c9 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -64,6 +64,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import CLOSE as PAGE_DERIVED_CLOSE
from skyvern.forge.sdk.workflow.page_derived_templates import OPEN as PAGE_DERIVED_OPEN
from skyvern.forge.taskv3 import engine as taskv3_engine
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3 import tools as taskv3_tools
from skyvern.forge.taskv3.auth_tools import VerificationFailure, VerificationState
from skyvern.forge.taskv3.engine import DEFAULT_MAX_SETTLE_DEFERRALS, MIN_ACTION_STEPS, run_task_v3_agent_loop
@@ -81,7 +82,7 @@ from skyvern.forge.taskv3.loop import (
)
from skyvern.forge.taskv3.run_arms import (
DATE_SEGMENT_AIM_FLAG,
- POINTER_PARITY_FLAG,
+ LOGIN_PACE_FLAG,
run_arm_enabled,
)
from skyvern.forge.taskv3.tools import PageProvider, _record_frame_work
@@ -189,6 +190,9 @@ async def _run_execute_task_v3(
loop_mock.context = context
loop_mock.active_credential_parameter_key_during_loop = context.active_credential_parameter_key
loop_mock.date_segment_aim_enabled_during_loop = run_arm_enabled(DATE_SEGMENT_AIM_FLAG, forced=False)
+ pace = input_dispatch._login_pace.get()
+ loop_mock.login_pace_during_loop = pace is not None
+ loop_mock.login_pace_polls_cancel = pace is not None and pace.should_cancel is not None
cb = kwargs.get("on_action_round")
if cb is not None and action_rounds:
for i, round_actions in enumerate(action_rounds):
@@ -331,30 +335,35 @@ async def test_execute_task_v3_buckets_the_date_segment_aim_arm_per_run(monkeypa
@pytest.mark.asyncio
-async def test_execute_task_v3_buckets_the_pointer_parity_arm_per_run_with_the_workflow(
- monkeypatch: pytest.MonkeyPatch,
[email protected]("variant, arm", [(None, "unrandomized"), ("control", "control"), ("treatment", "treatment")])
+async def test_execute_task_v3_holds_logins_only_on_the_login_pace_treatment(
+ monkeypatch: pytest.MonkeyPatch, variant: str | None, arm: str
) -> None:
- # The arm is targeted at named workflows, so the resolve must carry the workflow id, or no group matches.
- monkeypatch.setattr(settings, "TASK_V3_POINTER_PARITY", False)
- provider = AsyncMock(return_value="treatment")
+ # An absent or unset flag evaluates to no variant, which must leave the run unpaced. The arm is targeted at
+ # named workflows, so the resolve carries the workflow id or no release condition can match.
+ monkeypatch.setattr(settings, "TASK_V3_LOGIN_PACE", False)
+ provider = AsyncMock(side_effect=lambda flag, *_a, **_k: variant if flag == LOGIN_PACE_FLAG else None)
monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", provider)
_step, task, loop_mock, _post = await _run_execute_task_v3(
monkeypatch,
LoopOutcome(status="completed", reason="done", billable_actions=[]),
- workflow_run_id="wr_pointer_parity",
- context_overrides={"workflow_permanent_id": "wpid_pointer_parity"},
+ workflow_run_id="wr_login_pace",
+ context_overrides={"workflow_permanent_id": "wpid_login_pace"},
data_extraction_goal=None,
extracted_information_schema=None,
)
- assert loop_mock.context.run_arms[POINTER_PARITY_FLAG] == (task.workflow_run_id, "treatment")
+ assert loop_mock.context.run_arms[LOGIN_PACE_FLAG] == (task.workflow_run_id, arm)
+ assert loop_mock.login_pace_during_loop is (arm == "treatment")
+ assert loop_mock.login_pace_polls_cancel is (arm == "treatment")
+ assert input_dispatch._login_pace.get() is None
provider.assert_any_await(
- POINTER_PARITY_FLAG,
+ LOGIN_PACE_FLAG,
task.workflow_run_id,
properties={
"organization_id": task.organization_id,
- "workflow_permanent_id": task.workflow_permanent_id or "wpid_pointer_parity",
+ "workflow_permanent_id": task.workflow_permanent_id or "wpid_login_pace",
},
)
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index cda563780..6b5e5cf51 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -82,7 +82,6 @@ _DISCOVERED_BROWSER_API_CALLS = {
"hover": 1,
"press": 3,
"press_sequentially": 1,
- "scroll_into_view_if_needed": 1,
"select_option": 2,
"set_files": 1,
"set_input_files": 1,
@@ -260,14 +259,14 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 207
+ assert sum(sum(methods.values()) for methods in observed.values()) == 206
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 7
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 200
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 199
def test_every_raw_evaluate_call_is_classified() -> None:
diff --git a/tests/unit/test_taskv3_input_dispatch.py b/tests/unit/test_taskv3_input_dispatch.py
index 739335fd0..298a91b31 100644
--- a/tests/unit/test_taskv3_input_dispatch.py
+++ b/tests/unit/test_taskv3_input_dispatch.py
@@ -1,13 +1,21 @@
from __future__ import annotations
-from collections.abc import Awaitable, Callable
+import asyncio
+from collections.abc import Awaitable, Callable, Iterator
+from dataclasses import replace
+from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from playwright.async_api import ElementHandle
+from structlog.testing import capture_logs
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.taskv3 import input_dispatch
+from skyvern.forge.taskv3.run_arms import LOGIN_PACE_FLAG
+from tests.unit.scoped_asyncio import ScopedAsyncio
Call = tuple[str, tuple[Any, ...], dict[str, Any]]
@@ -39,6 +47,18 @@ class _FakePage(_Device):
def locator(self, selector: str) -> _FakeLocator:
return _FakeLocator(self, selector)
+ # As a frame realm: the main frame of its own page, never detached, with no navigation events.
+ parent_frame = None
+
+ def is_detached(self) -> bool:
+ return False
+
+ def on(self, event: str, handler: Callable[..., None]) -> None:
+ pass
+
+ def remove_listener(self, event: str, handler: Callable[..., None]) -> None:
+ pass
+
@pytest.fixture
def page() -> _FakePage:
@@ -95,45 +115,230 @@ async def test_an_element_handle_is_clicked_with_only_the_arguments_given() -> N
handle.click.assert_awaited_once_with()
+class _Clock:
+ def __init__(self) -> None:
+ self.now = 1000.0
+ self.sleeps: list[float] = []
+
+ async def sleep(self, seconds: float) -> None:
+ self.sleeps.append(seconds)
+ self.now += seconds
+
+
+def clock(monkeypatch: pytest.MonkeyPatch) -> _Clock:
+ clock = _Clock()
+ monkeypatch.setattr(input_dispatch, "time", SimpleNamespace(monotonic=lambda: clock.now))
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=clock.sleep))
+ return clock
+
+
+def login_page() -> _FakePage:
+ page = _FakePage()
+ page.url = "https://login.test/password"
+ page.page = page
+ return page
+
+
+def arm(request: pytest.FixtureRequest) -> Iterator[str]:
+ arm = getattr(request, "param", "treatment")
+ skyvern_context.set(SkyvernContext(run_arms={LOGIN_PACE_FLAG: ("wr_login", arm)}))
+ yield arm
+ input_dispatch.end_login_pace()
+ skyvern_context.reset()
+
+
[email protected]("arm", ["treatment", "control", "unrandomized"], indirect=True)
+async def test_only_the_treatment_holds_the_submit_after_a_password_fill(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ input_dispatch.start_login_pace()
+ clock.now += 5
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ await input_dispatch.click(login_page, "#sign-in")
+
+ assert sum(clock.sleeps) == (40.0 if arm == "treatment" else 0)
+ assert login_page.calls[-1][0] == "page.click"
+
+
[email protected]("elapsed, waited", [(30.0, 15.0), (60.0, 0.0)])
+async def test_the_dwell_is_measured_from_the_block_start_across_both_login_pages(
+ arm: str, clock: _Clock, login_page: _FakePage, elapsed: float, waited: float
+) -> None:
+ input_dispatch.start_login_pace()
+ login_page.url = "https://login.test/username"
+ clock.now += elapsed / 2
+ await input_dispatch.click(login_page, "#continue")
+ login_page.url = "https://login.test/password"
+ clock.now += elapsed / 2
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ with capture_logs() as logs:
+ await input_dispatch.press(login_page, "#pw", "Enter")
+
+ # Continue came before the password fill, so only the submit waits, and only for what the floor still needs.
+ assert sum(clock.sleeps) == waited
+ dwell = [log for log in logs if log["event"] == "Task V3 login pace dwell"]
+ assert [(log["waited_s"], log["nav_to_submit_s"]) for log in dwell] == [(waited, max(elapsed, 45.0))]
+
+
+ "gesture",
+ [
+ lambda page: input_dispatch.click(page, "#sign-in"),
+ lambda page: input_dispatch.click(page, page.locator("#sign-in")),
+ lambda page: input_dispatch.click_at(page, 10.0, 20.0),
+ lambda page: input_dispatch.press(page, "#pw", "Enter"),
+ lambda page: input_dispatch.press(page, None, "Enter"),
+ lambda page: input_dispatch.js_click(page, "document", {"sel": "#sign-in"}),
+ ],
+)
+async def test_every_click_and_enter_after_a_password_fill_is_held_once(
+ arm: str, clock: _Clock, login_page: _FakePage, gesture: Callable[[_FakePage], Awaitable[Any]]
+) -> None:
+ input_dispatch.start_login_pace()
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ await input_dispatch.press(login_page, "#pw", "Tab")
+ assert clock.sleeps == []
+
+ await gesture(login_page)
+ # A retyped password and a second submit on the same login wait no more.
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ await gesture(login_page)
+
+ assert sum(clock.sleeps) == 45.0
+
+
+async def test_no_dwell_and_no_mechanism_log_without_a_password_fill(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ input_dispatch.start_login_pace()
+ with capture_logs() as logs:
+ await input_dispatch.click(login_page, "#search")
+ await input_dispatch.press(login_page, "#q", "Enter")
+ input_dispatch.end_login_pace()
+
+ assert clock.sleeps == []
+ assert logs == []
+
+
+async def test_a_submit_the_hook_never_saw_is_logged_as_not_held(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ input_dispatch.start_login_pace()
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ clock.now += 8
+ # The page submitted on its own: the next click is already on another page and must not take the dwell.
+ login_page.url = "https://login.test/challenge"
+ with capture_logs() as logs:
+ await input_dispatch.click(login_page, "#verify")
+ input_dispatch.start_login_pace()
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ input_dispatch.end_login_pace()
+
+ assert clock.sleeps == []
+ assert [(log["event"], log["reason"], log["nav_s"]) for log in logs] == [
+ ("Task V3 login pace submit not held", "page_moved_before_submit", 8.0),
+ ("Task V3 login pace submit not held", "no_click_or_enter_after_password", 0.0),
+ ]
+
+
@pytest.mark.asyncio
[email protected]("treatment", [False, True])
-async def test_pointer_parity_moves_the_pointer_onto_an_input_or_click_target_first(
- monkeypatch: pytest.MonkeyPatch, treatment: bool
+async def test_a_hold_cancelled_by_the_callers_timeout_still_holds_the_submit(
+ arm: str, clock: _Clock, login_page: _FakePage, monkeypatch: pytest.MonkeyPatch
) -> None:
- # v1 moves the pointer onto every input before typing and onto every click target before pressing; a v3 run
- # on the TASK_V3_POINTER_PARITY treatment must too. Control keeps only Playwright's own move into the click.
- async_playwright = pytest.importorskip("playwright.async_api").async_playwright
- monkeypatch.setattr(input_dispatch.settings, "TASK_V3_POINTER_PARITY", treatment)
- async with async_playwright() as pw:
- try:
- browser = await pw.chromium.launch(headless=True)
- except Exception:
- pytest.skip("Requires Playwright browsers installed")
- try:
- page = await browser.new_page(viewport={"width": 800, "height": 600})
- await page.set_content(
- '<input id="user" style="position:absolute;top:100px;left:100px;width:200px;height:30px">'
- '<input id="pass" type="password" style="position:absolute;top:900px;left:100px;width:200px;height:30px">'
- '<button id="go" style="position:absolute;top:960px;left:100px;width:200px;height:40px">Go</button>'
- "<script>window.ev = []; for (const t of ['mousemove', 'input', 'pointerdown'])"
- " document.addEventListener(t, (e) => window.ev.push([t, e.target.id || '', e.isTrusted]), true);"
- "</script>"
- )
- await input_dispatch.fill(page, "#user", "alice", timeout=2000)
- await input_dispatch.fill(page, page.locator("#pass"), "secret", timeout=2000)
- # Off-centre on purpose: the press lands somewhere other than where the approach move ends.
- await input_dispatch.click(page, "#go", timeout=2000, position={"x": 3, "y": 3})
- events = await page.evaluate("() => window.ev")
- finally:
- await browser.close()
- press = events.index(["pointerdown", "go", True])
- moves_onto_button = events[:press].count(["mousemove", "go", True])
- if treatment:
- # The move lands on each field, below the fold too, before its first input event.
- for field in ("user", "pass"):
- first_input = events.index(["input", field, True])
- assert ["mousemove", field, True] in events[:first_input], events
- assert moves_onto_button >= 2, events
- else:
- assert all(kind != "mousemove" for kind, target, _ in events[:press] if target != "go"), events
- assert moves_onto_button == 1, events
+ input_dispatch.start_login_pace()
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ real_sleep = clock.sleep
+
+ async def _cancelled_once(seconds: float) -> None:
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=real_sleep))
+ clock.now += 5
+ raise asyncio.CancelledError
+
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=_cancelled_once))
+ with pytest.raises(asyncio.CancelledError):
+ await input_dispatch.click(login_page, "#captcha-checkbox")
+ await input_dispatch.click(login_page, "#sign-in")
+
+ assert sum(clock.sleeps) == 40.0
+ assert login_page.calls == [("page.click", ("#sign-in",), {})]
+
+
+async def test_a_task_canceled_during_the_wait_never_sends_any_held_submit(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ polls: list[float] = []
+
+ async def _should_cancel() -> bool:
+ polls.append(clock.now)
+ return len(polls) == 3
+
+ input_dispatch.start_login_pace(_should_cancel)
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ with capture_logs() as logs:
+ with pytest.raises(input_dispatch.LoginPaceRefused):
+ await input_dispatch.click(login_page, "#sign-in")
+ # A caller that swallows the refusal and tries another submit is refused too.
+ with pytest.raises(input_dispatch.LoginPaceRefused):
+ await input_dispatch.press(login_page, "#pw", "Enter")
+ input_dispatch.end_login_pace()
+
+ assert login_page.calls == []
+ assert sum(clock.sleeps) == 6.0
+ assert [log["reason"] for log in logs] == ["canceled_during_wait"]
+
+
+async def test_a_page_that_moves_during_the_wait_does_not_get_the_held_click(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ async def _page_moves_on_second_poll() -> bool:
+ if clock.now >= 1004:
+ login_page.url = "https://login.test/verify"
+ return False
+
+ input_dispatch.start_login_pace(_page_moves_on_second_poll)
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ with capture_logs() as logs:
+ with pytest.raises(input_dispatch.LoginPaceRefused):
+ await input_dispatch.click(login_page, "#sign-in")
+ await input_dispatch.click(login_page, "#continue-on-new-page")
+
+ assert login_page.calls == [("page.click", ("#continue-on-new-page",), {})]
+ assert [log["reason"] for log in logs] == ["page_moved_during_wait"]
+
+
+async def test_a_failed_cancel_read_still_sends_the_held_submit(arm: str, clock: _Clock, login_page: _FakePage) -> None:
+ async def _status_read_fails() -> bool:
+ raise ConnectionError("db blip")
+
+ input_dispatch.start_login_pace(_status_read_fails)
+ input_dispatch.note_secret_fill(login_page, input_dispatch.secret_fill_anchor(login_page))
+ await input_dispatch.click(login_page, "#sign-in")
+
+ assert sum(clock.sleeps) == 45.0
+ assert login_page.calls == [("page.click", ("#sign-in",), {})]
+
+
+async def test_a_frame_that_moved_during_the_password_fill_does_not_hold_the_next_click(
+ arm: str, clock: _Clock, login_page: _FakePage
+) -> None:
+ input_dispatch.start_login_pace()
+ anchor = input_dispatch.secret_fill_anchor(login_page)
+ # The frame's own URL moved while the password went in; the top URL did not.
+ input_dispatch.note_secret_fill(login_page, replace(anchor, frame_url="https://login.test/frame-before"))
+ with capture_logs() as logs:
+ await input_dispatch.click(login_page, "#next")
+
+ assert clock.sleeps == []
+ assert [(log["reason"], log["moved_by"]) for log in logs] == [("page_moved_before_submit", "frame_url")]
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 91c1539d2..3c5da16eb 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -53,6 +53,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import (
CredentialReleaseGuard,
)
from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.auth_tools import VerificationState
from skyvern.forge.taskv3.code_surface import (
CodeToolSurface,
@@ -67,6 +68,7 @@ from skyvern.forge.taskv3.loop import (
ToolSpec,
_navigate_record_fields,
)
+from skyvern.forge.taskv3.run_arms import LOGIN_PACE_FLAG
from skyvern.forge.taskv3.tools import (
_MENU_OPTION_TEXTS_JS,
_OPAQUE_ID_RUN_RE,
@@ -15589,6 +15591,160 @@ async def test_a_throwing_getattribute_on_the_occluder_does_not_defeat_occlusion
assert r.status == "error", r.content
+_LOGIN_FORM_HTML = """
+<form onsubmit="event.preventDefault(); document.body.dataset.submitted = '1'">
+ <label>Email <input id="user" type="email"></label>
+ <label>Password <input id="pw" type="password"></label>
+ <button id="sign-in" type="submit">Sign in</button>
+</form>
+"""
+
+
+@_skip_no_browser
[email protected]("submit", ["press_enter", "click"])
+async def test_login_pace_treatment_holds_the_submit_after_typing_a_password(
+ monkeypatch: pytest.MonkeyPatch, submit: str
+) -> None:
+ clock = SimpleNamespace(now=1000.0, sleeps=[])
+
+ async def _sleep(seconds: float) -> None:
+ clock.sleeps.append(seconds)
+ clock.now += seconds
+
+ monkeypatch.setattr(input_dispatch, "time", SimpleNamespace(monotonic=lambda: clock.now))
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=_sleep))
+ with skyvern_context.scoped(SkyvernContext(run_arms={LOGIN_PACE_FLAG: ("wr_login", "treatment")})):
+ async with _content_page(_LOGIN_FORM_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ type_tool = _tool(tools, "type").handler
+ input_dispatch.start_login_pace()
+ clock.now += 12
+ try:
+ # Enter after an email is not a credential submit and goes straight through.
+ assert (await type_tool({"selector": "#user", "text": "[email protected]", "press_enter": True})).status == "ok"
+ assert clock.sleeps == []
+ if submit == "press_enter":
+ r = await type_tool({"selector": "#pw", "text": "hunter2", "press_enter": True})
+ else:
+ assert (await type_tool({"selector": "#pw", "text": "hunter2"})).status == "ok"
+ r = await _tool(tools, "click").handler({"selector": "#sign-in"})
+ finally:
+ input_dispatch.end_login_pace()
+ assert r.status == "ok", r.content
+ assert sum(clock.sleeps) == 33.0
+ assert await page.evaluate("document.body.dataset.submitted") == "1"
+
+
+@_skip_no_browser
+async def test_login_pace_a_password_fill_that_moves_the_page_does_not_hold_the_next_click(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A form that submits itself as the password lands: the next click is on another page, not the submit.
+ clock = SimpleNamespace(now=1000.0, sleeps=[])
+
+ async def _sleep(seconds: float) -> None:
+ clock.sleeps.append(seconds)
+ clock.now += seconds
+
+ monkeypatch.setattr(input_dispatch, "time", SimpleNamespace(monotonic=lambda: clock.now))
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=_sleep))
+ html = (
+ '<input id="pw" type="password" oninput="history.pushState({}, \'\', \'#/verify\')">'
+ '<button id="next" onclick="this.dataset.clicked = 1">Next</button>'
+ )
+ with skyvern_context.scoped(SkyvernContext(run_arms={LOGIN_PACE_FLAG: ("wr_login", "treatment")})):
+ async with _content_page(html) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ input_dispatch.start_login_pace()
+ try:
+ with capture_logs() as logs:
+ # Appending reads the field back after a settle wait, so the page has moved by the time it arms.
+ typed = await _tool(tools, "type").handler({"selector": "#pw", "text": "hunter2", "clear": False})
+ assert typed.status == "ok", typed.content
+ r = await _tool(tools, "click").handler({"selector": "#next"})
+ finally:
+ input_dispatch.end_login_pace()
+ assert r.status == "ok", r.content
+ assert clock.sleeps == []
+ assert [log["reason"] for log in logs if log["event"] == "Task V3 login pace submit not held"] == [
+ "page_moved_before_submit"
+ ]
+
+
+_FRAMED_LOGIN_HTML = (
+ '<iframe id="login" srcdoc="<input id=pw type=password><button id=sign-in onclick=parent.signedIn=1>Sign in</button>"></iframe>'
+ '<iframe id="challenge" srcdoc="<p>challenge</p>"></iframe>'
+)
+
+
+async def _change_login_frame(page: Page, change: str) -> None:
+ login, challenge = page.frames[1], page.frames[2]
+ if change == "navigate":
+ async with page.expect_event("framenavigated", predicate=lambda frame: frame is login):
+ await login.evaluate("location.href = 'about:blank'")
+ elif change == "reload":
+ async with page.expect_event("framenavigated", predicate=lambda frame: frame is login):
+ await login.evaluate("setTimeout(() => location.reload())")
+ elif change == "detach":
+ async with page.expect_event("framedetached"):
+ await page.evaluate("document.getElementById('login').remove()")
+ else:
+ # Another frame on the page, a challenge widget, navigating is not the password page moving.
+ async with page.expect_event("framenavigated", predicate=lambda frame: frame is challenge):
+ await challenge.evaluate("location.href = 'about:blank'")
+
+
+@_skip_no_browser
[email protected]("change", ["navigate", "reload", "detach", "other_frame"])
+async def test_login_pace_a_password_frame_that_changes_during_the_wait_does_not_get_the_held_click(
+ monkeypatch: pytest.MonkeyPatch, change: str
+) -> None:
+ clock = SimpleNamespace(now=1000.0, sleeps=[])
+
+ async def _sleep(seconds: float) -> None:
+ clock.sleeps.append(seconds)
+ clock.now += seconds
+
+ monkeypatch.setattr(input_dispatch, "time", SimpleNamespace(monotonic=lambda: clock.now))
+ monkeypatch.setattr(input_dispatch, "asyncio", ScopedAsyncio(sleep=_sleep))
+ with skyvern_context.scoped(SkyvernContext(run_arms={LOGIN_PACE_FLAG: ("wr_login", "treatment")})):
+ async with _content_page(_FRAMED_LOGIN_HTML) as page:
+ await page.wait_for_function("document.querySelectorAll('iframe').length === 2")
+ login = page.frames[1]
+ await login.wait_for_selector("#sign-in")
+ changed: list[str] = []
+
+ async def _change_on_first_poll() -> bool:
+ if not changed:
+ changed.append(change)
+ await _change_login_frame(page, change)
+ return False
+
+ input_dispatch.start_login_pace(_change_on_first_poll)
+ try:
+ anchor = input_dispatch.secret_fill_anchor(login)
+ await input_dispatch.fill(login, "#pw", "hunter2")
+ input_dispatch.note_secret_fill(login, anchor)
+ with capture_logs() as logs:
+ if change == "other_frame":
+ await input_dispatch.click(login, "#sign-in")
+ else:
+ with pytest.raises(input_dispatch.LoginPaceRefused):
+ await input_dispatch.click(login, "#sign-in")
+ finally:
+ input_dispatch.end_login_pace()
+ if change == "other_frame":
+ assert sum(clock.sleeps) == 45.0
+ assert await page.evaluate("window.signedIn") == 1
+ else:
+ moved_by = {"navigate": "frame_url", "reload": "navigated_event", "detach": "frame_detached"}[change]
+ assert [(log["reason"], log["moved_by"]) for log in logs] == [("page_moved_during_wait", moved_by)]
+ assert await page.evaluate("window.signedIn") is None
+
+
# Design systems routinely put an inner wrapper between the input and the element the skin is
# positioned against, so "the field's immediate parent" is narrower than "the field's own control".
_SKIN_ONE_WRAPPER_DEEPER_HTML = """
@@ -18611,112 +18767,6 @@ async def test_pre_snapshot_survives_a_container_the_page_rebuilds_by_cloning()
assert await page.evaluate(_FIND_MENU_JS, {"sel": "#sort", "el": None}) is None
-@_skip_no_browser
-async def test_pointer_parity_run_writes_no_snapshot_attribute_and_still_reads_the_click_reaction(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # The treatment of TASK_V3_POINTER_PARITY must not stamp every visible element just before a click, and
- # must keep what that stamp is for: a menu the click opens reads as a reaction, a static list does not.
- import skyvern.forge.taskv3.tools as tools_module # noqa: PLC0415
- from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
-
- monkeypatch.setattr(tools_module.settings, "TASK_V3_POINTER_PARITY", True)
- async with _live_page(
- """<input id="user" type="text" style="position:absolute;top:10px;left:10px;width:200px;height:24px">
- <button id="sort" style="position:absolute;top:10px;left:240px;width:120px;height:24px">Sort</button>
- <div id="results" style="position:absolute;top:60px;left:10px;width:220px">
- <a href="/a" role="option" style="display:block;height:20px">Result Alpha</a>
- <a href="/b" role="option" style="display:block;height:20px">Result Beta</a>
- <a href="/c" role="option" style="display:block;height:20px">Result Gamma</a>
- </div>
- <div id="menu" style="display:none;position:absolute;top:40px;left:240px;width:200px">
- <div role="option" style="height:20px">Newest</div>
- <div role="option" style="height:20px">Oldest</div>
- <div role="option" style="height:20px">Relevance</div>
- </div>
- <script>
- window.preWrites = 0;
- new MutationObserver((recs) => {
- for (const r of recs) if (r.attributeName === 'data-tv3-pre') window.preWrites++;
- }).observe(document, {subtree: true, attributes: true});
- </script>"""
- ) as page:
- tools = build_browser_tools(_fixed_page_provider(page))
- assert (await _tool(tools, "type").handler({"selector": "#user", "text": "alice"})).status == "ok"
- assert (await _tool(tools, "click").handler({"selector": "#sort"})).status == "ok"
- assert await page.evaluate("() => window.preWrites") == 0
-
- await page.evaluate(_CLICK_PRECHECK_JS, {"sel": "#sort", "el": None, "weak": True})
- await page.evaluate("() => { document.getElementById('menu').style.display = 'block'; }")
- found = await page.evaluate(_FIND_MENU_JS, {"sel": "#sort", "el": None})
- assert isinstance(found, dict), found
- assert [o["text"] for o in found["options"]] == ["Newest", "Oldest", "Relevance"]
- assert await page.evaluate("() => window.preWrites") == 0
-
-
-@_skip_no_browser
-async def test_pointer_parity_click_still_reports_a_menu_its_hover_opens(monkeypatch: pytest.MonkeyPatch) -> None:
- # The treatment's pointer move must land after the click's baseline: a menu that opens on mouseenter is
- # the click's reaction in control, and must stay one, or the model needs another look to get its rows.
- import skyvern.forge.taskv3.tools as tools_module # noqa: PLC0415
-
- monkeypatch.setattr(tools_module.settings, "TASK_V3_POINTER_PARITY", True)
- async with _live_page(
- """<button id="acct" style="position:absolute;top:10px;left:300px;width:120px;height:24px">Account</button>
- <div id="m" role="menu" style="display:none;position:absolute;top:36px;left:300px;width:160px">
- <div role="menuitem" style="height:20px">Profile</div>
- <div role="menuitem" style="height:20px">Billing</div>
- <div role="menuitem" style="height:20px">Sign out</div>
- </div>
- <script>
- document.getElementById('acct').addEventListener('mouseenter', () => {
- document.getElementById('m').style.display = 'block';
- });
- </script>"""
- ) as page:
- await page.mouse.move(5, 500)
- r = await _tool(build_browser_tools(_fixed_page_provider(page)), "click").handler({"selector": "#acct"})
- assert r.status == "ok", r.content
- assert "This click opened a menu" in r.content, r.content
-
-
-@_skip_no_browser
-async def test_write_free_snapshot_reads_existed_before_by_node_identity() -> None:
- # The write-free carrier is node identity, the WeakSet the shadow-DOM half already used: a node that was
- # visible reads as old, and any node created since -- a clone included, the disclosed price -- or a row
- # the rewrite watcher saw change, down to its descendants, reads as a reaction.
- from skyvern.forge.taskv3.tools import ( # noqa: PLC0415
- _PIERCED_QUERY_JS,
- _PRESNAPSHOT_JS,
- _WATCH_REWRITTEN_ROWS_JS,
- )
-
- async with _live_page(
- """<ul id="k" style="position:absolute;top:10px;left:10px;width:200px;margin:0">
- <li style="height:20px">Kept</li></ul>
- <ul id="a" style="position:absolute;top:40px;left:10px;width:200px;margin:0">
- <li style="height:20px">Engineering</li></ul>
- <ul id="c" style="position:absolute;top:80px;left:10px;width:200px;margin:0">
- <li style="height:20px"><span>Paris</span></li></ul>"""
- ) as page:
- await page.evaluate(_PRESNAPSHOT_JS, True)
- await page.evaluate(_WATCH_REWRITTEN_ROWS_JS)
- await page.evaluate(
- "() => { const a = document.getElementById('a'); a.parentNode.replaceChild(a.cloneNode(true), a);"
- " document.querySelector('#c span').firstChild.nodeValue = 'Paris'; }"
- )
- await page.wait_for_timeout(50)
- pre = await page.evaluate(
- "() => {" + _PIERCED_QUERY_JS + " const q = (s) => preHas(document.querySelector(s));"
- " return {kept: q('#k li'), clone: q('#a li'), rewritten: q('#c span'),"
- " attrs: document.querySelectorAll('[data-tv3-pre]').length}; }"
- )
- assert pre == {"kept": True, "clone": False, "rewritten": False, "attrs": 0}, pre
-
-
@_skip_no_browser
@pytest.mark.asyncio
async def test_reaction_gate_rejects_a_page_supplied_snapshot_impostor() -> None: