Skip to content

Automate browser based workflows with AI

Stop telling Task V3 to dismiss a covering layer that contains the field itself (#8722)

pedrohsdb20dba4cc188e
diff --git a/skyvern/config.py b/skyvern/config.py
index 704ecac40..da975bd3e 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -547,6 +547,10 @@ class Settings(BaseSettings):
# year-only segment groups to the segment path (SKY-17013). Force-on term only: runs are randomized per
# run by the flag of the same name, read through run_arm_enabled(DATE_SEGMENT_AIM_FLAG, ...).
TASK_V3_DATE_SEGMENT_AIM: bool = False
+ # Send Task V3's clicks, typing and scrolls through EventStrategyFactory, as v1 does, so a run follows
+ # whatever USE_EVENT_STRATEGIES registered. Force-on term only: runs are randomized per run by
+ # the flag of the same name, read through run_arm_enabled(HUMANIZED_INPUT_FLAG, ...). Off: plain Playwright.
+ TASK_V3_HUMANIZED_INPUT: bool = False
# Render the previous block's outcome (status / finish reason / final URL) and whether this is the
# last block into a v3 block's goal. Costs prompt tokens on every turn of the block, so it is
# measured via taskv3_block_context_tokens before it earns default-on. The outcome itself is
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 31861c2e2..edce7d5fc 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -185,6 +185,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import CredentialRelea
from skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterType
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
from skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSES
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.goal_check import (
GOAL_CHECK_PROMPT_NAME,
PRE_JUDGE_SKIP_REASONS,
@@ -199,6 +200,7 @@ from skyvern.forge.taskv3.run_arms import (
EXTRACTION_REPORTS_FLAG,
GOAL_CHECK_ENFORCE_FLAG,
GOAL_CHECK_FLAG,
+ HUMANIZED_INPUT_FLAG,
REQUIRED_FIELD_ANSWERS_FLAG,
TYPE_COORDINATE_CLICK_FLAG,
resolve_run_arm,
@@ -2372,6 +2374,15 @@ class ForgeAgent:
organization_id=task.organization_id,
forced=settings.TASK_V3_EXTRACTION_REPORTS,
)
+ await resolve_run_arm(
+ context,
+ HUMANIZED_INPUT_FLAG,
+ distinct_id=task.workflow_run_id or task.task_id,
+ organization_id=task.organization_id,
+ forced=settings.TASK_V3_HUMANIZED_INPUT,
+ )
+ # Here as well as on the loop's finish line, so a run whose loop raises still records its profile.
+ LOG.info("Task V3 humanized input resolved", **input_dispatch.arm_fields())
# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.
goal_judge_shot: list[bytes] = []
page_free_validation = bool(
diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.py
index 96e8f9045..5c794d1e2 100644
--- a/skyvern/forge/sdk/event/factory.py
+++ b/skyvern/forge/sdk/event/factory.py
@@ -39,6 +39,7 @@ class EventStrategyFactory:
__cursor_vis_cache: CursorEventStrategy | None = None
__input: InputEventStrategy | None = None
__scroll: ScrollEventStrategy | None = None
+ __profile: str | None = None
__metrics: _EventMetrics = _EventMetrics()
# -- setters ----------------------------------------------------------------
@@ -62,8 +63,22 @@ class EventStrategyFactory:
EventStrategyFactory.__cursor_vis_cache = None
EventStrategyFactory.__input = None
EventStrategyFactory.__scroll = None
+ EventStrategyFactory.__profile = None
EventStrategyFactory.__metrics = _EventMetrics()
+ @staticmethod
+ def record_profile(profile: str) -> None:
+ """Name the profile whoever registered the current strategies registered them under."""
+ EventStrategyFactory.__profile = profile
+
+ @staticmethod
+ def registered_profile() -> str:
+ """The recorded profile; "none" when nothing is registered, "unrecorded" when strategies were set unnamed."""
+ if EventStrategyFactory.__profile is not None:
+ return EventStrategyFactory.__profile
+ strategies = (EventStrategyFactory.__cursor, EventStrategyFactory.__input, EventStrategyFactory.__scroll)
+ return "none" if all(strategy is None for strategy in strategies) else "unrecorded"
+
# -- getters (always return a non-None strategy) ----------------------------
@staticmethod
diff --git a/skyvern/forge/taskv3/captcha_tools.py b/skyvern/forge/taskv3/captcha_tools.py
index 0a7ebdfc3..8f3218488 100644
--- a/skyvern/forge/taskv3/captcha_tools.py
+++ b/skyvern/forge/taskv3/captcha_tools.py
@@ -20,6 +20,7 @@ on the tool-call record; ``tool_status`` alone cannot tell a working detector fr
from __future__ import annotations
import asyncio
+import functools
import re
from typing import Any
from urllib.parse import urlsplit
@@ -29,6 +30,7 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.schemas.tasks import Task
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.loop import REF_SELECTOR_RE, ToolResult, ToolSpec
from skyvern.forge.taskv3.tools import PageProvider, _invalid_selector_result, _normalize_selector
from skyvern.webeye.utils.captcha_solver import (
@@ -301,6 +303,8 @@ def build_captcha_tools(
workflow_run_id=task.workflow_run_id,
browser_session_id=task.browser_session_id,
probe_child_frames=True,
+ click=functools.partial(input_dispatch.click, page),
+ click_handle=functools.partial(input_dispatch.click_handle, page),
)
except CaptchaChallengeUnsolvedError:
failed_attempts += 1
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index efb2353f8..efdedc2bd 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -35,6 +35,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import VISION_FALLBACK_PROMPT
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTask
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.workflow.models.credential_release import CredentialReleaseGuard
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.code_surface import apply_surface, configured_surface
from skyvern.forge.taskv3.goal_check import (
GOAL_CHECK_TIMEOUT_SECONDS,
@@ -715,6 +716,7 @@ async def run_task_v3_agent_loop(
# The run's model, so exposure rates on this line split per model like the re-ask line's.
llm_key=llm_caller.llm_key,
unlisted_reask=outcome.unlisted_reask,
+ **input_dispatch.arm_fields(),
# The loop's progress signals ride here rather than on records of their own: this line
# already fires exactly once per run and already carries block_type, so collapsing removes a
# per-run indexed event and makes the join to block_type free instead of a second lookup.
diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.py
new file mode 100644
index 000000000..663ddf3ec
--- /dev/null
+++ b/skyvern/forge/taskv3/input_dispatch.py
@@ -0,0 +1,281 @@
+"""The one place Task V3 sends pointer and keyboard input to a page.
+
+Control runs the Playwright call each tool made before this module existed, argument for argument. Treatment
+sends a gesture through EventStrategyFactory wherever v1 does and calls Playwright directly wherever v1 does, so
+the run follows whatever USE_EVENT_STRATEGIES registered for the process, like a v1 block of the same run.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, TypeAlias, cast
+
+import structlog
+from playwright.async_api import ElementHandle, FileChooser, Frame, Locator, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+
+from skyvern.config import settings
+from skyvern.constants import TEXT_PRESS_MAX_LENGTH
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
+from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, current_run_arm, run_arm_enabled
+from skyvern.webeye.browser_object_predicates import is_page_like
+
+LOG = structlog.get_logger()
+
+Realm: TypeAlias = Page | Frame
+Target: TypeAlias = str | Locator | ElementHandle
+
+# After the cursor reaches a target, wait until the page has painted twice and then gone this long without a DOM
+# mutation, so a hover-intent menu crossed on the way has finished opening or closing before a caller takes its
+# pre-action baseline. Capped: a page that never stops mutating (a ticker, a spinner) costs the cap, not a stall.
+_SETTLE_QUIET_MS = 150
+_SETTLE_CAP_MS = 1000
+_SETTLE_JS = r"""(arg) => new Promise((resolve) => {
+ const started = performance.now();
+ let last = started;
+ const obs = new MutationObserver(() => { last = performance.now(); });
+ const tick = () => {
+ const now = performance.now();
+ if (now - last >= arg.quietMs || now - started >= arg.capMs) { obs.disconnect(); resolve(true); return; }
+ setTimeout(tick, 25);
+ };
+ requestAnimationFrame(() => requestAnimationFrame(() => {
+ obs.observe(document.documentElement, { subtree: true, childList: true, attributes: true, characterData: true });
+ last = performance.now();
+ setTimeout(tick, 25);
+ }));
+ setTimeout(() => { obs.disconnect(); resolve(false); }, arg.capMs);
+})"""
+
+
+def arm_fields() -> dict[str, Any]:
+ return {
+ "humanized_input_arm": current_run_arm(HUMANIZED_INPUT_FLAG),
+ "humanized_input_in_effect": humanized(),
+ "global_event_strategy_profile": EventStrategyFactory.registered_profile(),
+ }
+
+
+def humanized() -> bool:
+ return run_arm_enabled(HUMANIZED_INPUT_FLAG, settings.TASK_V3_HUMANIZED_INPUT)
+
+
+def _opts(**options: Any) -> dict[str, Any]:
+ # Only what the caller passed, so a control call carries exactly the keywords the tool always sent.
+ return {key: value for key, value in options.items() if value is not None}
+
+
+def _page_of(realm: Realm) -> Page:
+ # A Frame has no mouse or keyboard; the factory always drives the tab that owns it.
+ return cast(Page, realm) if is_page_like(realm) else cast(Frame, realm).page
+
+
+def _locator(realm: Realm, target: str | Locator) -> Locator:
+ # A page-level selector method acts on the first match; a locator is strict unless told otherwise.
+ return realm.locator(target).first if isinstance(target, str) else target
+
+
+async def _move_to_element(page: Page, element: ElementHandle) -> None:
+ try:
+ box = await element.bounding_box()
+ if box is None:
+ return
+ await EventStrategyFactory.move_cursor(page, box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)
+ except Exception:
+ LOG.debug("taskv3 cursor move to a handed-in element failed, clicking anyway", exc_info=True)
+
+
+async def approach(realm: Realm, target: str | Locator) -> None:
+ """Treatment: bring the cursor to ``target`` and let the page settle, so travel effects predate a baseline."""
+ if not humanized():
+ return
+ locator = _locator(realm, target)
+ try:
+ # The cursor is clamped to the viewport, so an off-screen target would otherwise be scrolled to, and
+ # travelled to again, only after the baseline.
+ await locator.scroll_into_view_if_needed(timeout=2000)
+ except Exception:
+ LOG.debug("taskv3 scroll before cursor approach failed", exc_info=True)
+ await EventStrategyFactory.move_to_element(locator.page, locator)
+ try:
+ async with asyncio.timeout((_SETTLE_CAP_MS + 500) / 1000):
+ await realm.evaluate(_SETTLE_JS, {"quietMs": _SETTLE_QUIET_MS, "capMs": _SETTLE_CAP_MS})
+ except Exception:
+ LOG.debug("taskv3 settle after cursor approach did not complete", exc_info=True)
+
+
+async def click(
+ realm: Realm,
+ target: str | Locator,
+ *,
+ timeout: float | None = None,
+ force: bool | None = None,
+ position: dict[str, float] | None = None,
+) -> None:
+ options = _opts(timeout=timeout, force=force, position=position)
+ if not humanized():
+ if isinstance(target, str):
+ await realm.click(target, **options)
+ else:
+ await target.click(**options)
+ return
+ locator = _locator(realm, target)
+ if force is not None or position is not None:
+ await EventStrategyFactory.move_to_element(locator.page, locator)
+ if isinstance(target, str):
+ await realm.click(target, **options)
+ else:
+ await target.click(**options)
+ return
+ await EventStrategyFactory.click_element(locator.page, locator, timeout=timeout)
+
+
+async def click_handle(
+ realm: Realm,
+ handle: ElementHandle,
+ *,
+ timeout: float | None = None,
+ force: bool | None = None,
+ position: dict[str, float] | None = None,
+) -> None:
+ if humanized():
+ # v1's pattern for a click the cursor strategy cannot express: move there, then click directly.
+ await _move_to_element(_page_of(realm), handle)
+ await handle.click(**_opts(timeout=timeout, force=force, position=position))
+
+
+async def click_at(page: Page, x: float, y: float) -> None:
+ if humanized():
+ await EventStrategyFactory.move_cursor(page, x, y)
+ await page.mouse.click(x, y)
+
+
+async def hover(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
+ if humanized():
+ locator = _locator(realm, selector)
+ await EventStrategyFactory.move_to_element(locator.page, locator)
+ await realm.hover(selector, **_opts(timeout=timeout))
+
+
+async def js_click(realm: Realm, root_query_js: str, arg: Any) -> Any:
+ """Click a hidden native control from inside the page. No pointer events, in either arm, as in v1."""
+ return await realm.evaluate(
+ "(arg) => { const _q = "
+ + root_query_js
+ + "; const el = _q.find(arg.sel) || arg.el;"
+ + " if (!el || !el.isConnected) return false; el.click(); return true; }",
+ arg,
+ )
+
+
+async def focus(realm: Realm, target: str | Locator, *, timeout: float | None = None) -> None:
+ if isinstance(target, str):
+ await realm.focus(target, **_opts(timeout=timeout))
+ else:
+ await target.focus(**_opts(timeout=timeout))
+
+
+async def fill(realm: Realm, target: Target, value: str, *, timeout: float | None = None) -> None:
+ """An atomic write, in both arms: v1 fills plain fields and secrets in one call and never humanizes them."""
+ if isinstance(target, str):
+ await realm.fill(target, value, **_opts(timeout=timeout))
+ else:
+ await target.fill(value, **_opts(timeout=timeout))
+
+
+async def clear(realm: Realm, selector: str, *, timeout: float | None = None) -> None:
+ if not humanized():
+ await realm.fill(selector, "", **_opts(timeout=timeout))
+ return
+ locator = _locator(realm, selector)
+ await EventStrategyFactory.clear_field(locator.page, locator, 0, timeout=timeout)
+
+
+async def type_keys(
+ realm: Realm,
+ target: str | Locator | None,
+ text: str,
+ *,
+ delay: float | None = None,
+ timeout: float | None = None,
+ secret: bool = False,
+ replace: bool = False,
+) -> None:
+ """Key events for ``text``. ``target`` None types into whatever holds focus, with no focus of its own;
+ ``realm`` is then the page.
+
+ A secret is typed key by key as before in both arms: a humanized keyboard mistypes and backspaces, and a
+ fill would not advance split code boxes. ``replace`` says the caller has just emptied the field, so
+ treatment may write the untyped part with fill."""
+ options = _opts(delay=delay, timeout=timeout)
+ if secret or not humanized():
+ if target is None:
+ await cast(Page, realm).keyboard.type(text, **options)
+ elif isinstance(target, str):
+ await realm.type(target, text, **options)
+ else:
+ await target.press_sequentially(text, **options)
+ return
+ # As v1 types: all but the last few characters in one write, the tail through the registered strategy,
+ # which owns the cadence. An emptied field takes the head as a fill; one being appended to takes it as
+ # inserted text at the caret, since a fill would replace what the field already holds.
+ cut = max(len(text) - TEXT_PRESS_MAX_LENGTH, 0)
+ head, tail = text[:cut], text[cut:]
+ page = cast(Page, realm) if target is None else _page_of(realm)
+ seconds = None if not timeout else timeout / 1000
+ try:
+ async with asyncio.timeout(seconds):
+ if head and replace and target is not None:
+ await fill(realm, target, head, timeout=timeout)
+ elif target is not None:
+ # Focused once: a humanized keyboard given a locator re-focuses the field before every key,
+ # which puts the caret back at the start of a select-on-focus field.
+ await focus(realm, target, timeout=timeout)
+ if head and not (replace and target is not None):
+ await page.keyboard.insert_text(head)
+ await EventStrategyFactory.type_text(page, None, tail, timeout=timeout)
+ except TimeoutError as exc:
+ raise PlaywrightTimeoutError(f"Timeout {timeout}ms exceeded.") from exc
+
+
+async def press(
+ realm: Realm, target: str | Locator | ElementHandle | None, key: str, *, timeout: float | None = None
+) -> None:
+ """A key press, raw in both arms as v1's keypress is. ``target`` None presses on ``realm``, the page's, keyboard."""
+ options = _opts(timeout=timeout)
+ if target is None:
+ await cast(Page, realm).keyboard.press(key, **options)
+ elif isinstance(target, str):
+ await realm.press(target, key, **options)
+ else:
+ await target.press(key, **options)
+
+
+async def select_option(
+ realm: Realm,
+ selector: str,
+ *,
+ label: str | list[str] | None = None,
+ value: str | list[str] | None = None,
+ timeout: float | None = None,
+ force: bool | None = None,
+) -> None:
+ if label is not None:
+ await realm.select_option(selector, label=label, **_opts(timeout=timeout, force=force))
+ else:
+ await realm.select_option(selector, value=value, **_opts(timeout=timeout, force=force))
+
+
+async def set_files(chooser: FileChooser, paths: list[str]) -> None:
+ await chooser.set_files(paths)
+
+
+async def set_input_files(target: Locator | ElementHandle, paths: list[str]) -> None:
+ await target.set_input_files(paths)
+
+
+async def wheel(page: Page, delta_x: float, delta_y: float) -> None:
+ if humanized():
+ await EventStrategyFactory.scroll_by(page, delta_x, delta_y)
+ else:
+ await page.mouse.wheel(delta_x, delta_y)
diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.py
index f2f0200ae..261745d25 100644
--- a/skyvern/forge/taskv3/loop.py
+++ b/skyvern/forge/taskv3/loop.py
@@ -164,6 +164,9 @@ CoveredBranch = Literal[
"unnamed",
# The layer intercepts the pointer but paints nothing, so it is absent from the screenshot.
"invisible",
+ # The layer is the field's own container and a non-interactive element in it takes the pointer;
+ # the message names that stamped element to click, so no controls are listed.
+ "own_container",
]
# WHICH ELEMENT the probe named as the layer, which the branch cannot recover. `named` with zero
diff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.py
index 206e7984d..cbefa1196 100644
--- a/skyvern/forge/taskv3/run_arms.py
+++ b/skyvern/forge/taskv3/run_arms.py
@@ -23,6 +23,7 @@ CUSTOMER_PRECEDENCE_FLAG = "TASK_V3_CUSTOMER_PRECEDENCE"
EXTRACTION_REPORTS_FLAG = "TASK_V3_EXTRACTION_REPORTS"
GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"
GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"
+HUMANIZED_INPUT_FLAG = "TASK_V3_HUMANIZED_INPUT"
# Person properties a flag is evaluated with beyond organization_id; resolve_run_arm drops any other key. The
# PostHog preflight (scripts/check_run_arm_flags.py) reads this mapping to accept release conditions on them.
@@ -101,3 +102,12 @@ def run_arm_enabled(flag: str, forced: bool) -> bool:
return False
pin = context.run_arms.get(flag)
return pin is not None and pin[1] == "treatment"
+
+
+def current_run_arm(flag: str) -> RunArm | None:
+ """The arm pinned for ``flag`` on the current run, or None when it was never resolved."""
+ context = skyvern_context.current()
+ if context is None:
+ return None
+ pin = context.run_arms.get(flag)
+ return pin[1] if pin is not None else None
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index c80a13327..7acdb8cab 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import (
CredentialReleaseGuard,
release_target_url,
)
+from skyvern.forge.taskv3 import input_dispatch
from skyvern.forge.taskv3.loop import (
ACTION_OUTCOME_DATA_KEY,
FILL_TOOLS,
@@ -543,8 +544,8 @@ SELECTION_REPORT_OPTION_WIDTH = 80
def _selection_report(options: list[str]) -> str:
shown = [
- o[:SELECTION_REPORT_OPTION_WIDTH] + "…" if len(o) > SELECTION_REPORT_OPTION_WIDTH else o
- for o in options[:SELECTION_REPORT_MAX_OPTIONS]
+ m[:SELECTION_REPORT_OPTION_WIDTH] + "…" if len(m) > SELECTION_REPORT_OPTION_WIDTH else m
+ for m in (_model_text(o, SELECTION_REPORT_OPTION_WIDTH + 1) for o in options[:SELECTION_REPORT_MAX_OPTIONS])
]
if len(shown) < len(options):
return f"{shown!r} (showing {len(shown)} of {len(options)})"
@@ -924,7 +925,7 @@ def _commit_gate(
def _unproven_row_error(value: str, selector: str, seen: str, row: dict[str, Any]) -> ToolResult:
- label = str(row.get("text") or "")[:60]
+ label = _model_text(row.get("text") or "", 60)
return ToolResult.error(
f"{value!r} matched no option in {selector}'s list — {seen}, so select_combobox cannot prove {label!r} is "
f'the only match; the field is NOT filled — if the visible row [data-tv3-menu="{row.get("n")}"] '
@@ -4188,6 +4189,7 @@ _TYPE_TARGET_PROBE_JS = (
+ _NATIVE_LABEL_JS
+ r"""
try { _q.all('[data-tv3-cover]').forEach((n) => n.removeAttribute('data-tv3-cover')); } catch (e) { /* best-effort */ }
+ try { _q.all('[data-tv3-catcher]').forEach((n) => n.removeAttribute('data-tv3-catcher')); } catch (e) { /* best-effort */ }
// A host-anchored selector's two halves straddle a shadow boundary, so no single root can match it
// and a per-root lookup finds nothing -- which would read as "no field here" and skip the check on
@@ -5175,6 +5177,15 @@ _TYPE_TARGET_PROBE_JS = (
try { layer.setAttribute('data-tv3-cover', '1'); } catch (e) { /* best-effort */ }
out.occluder = { selector: layerSelector, name: layerName, controls, truncated };
out.occluder.layerKind = layerKind;
+ // The walk can stop on a pinned ancestor of the field itself: a sticky header whose collapsed
+ // search form lays a transparent click-catcher over its own inputs. There is nothing to dismiss.
+ // An interactive hit (the form's own submit button) is not a catcher: clicking it could submit.
+ const activates = """
+ + json.dumps(_INTERACTIVE_HIT_SEL)
+ + r""";
+ if (related(layer, el) && ![hit, hitResult.top].some((n) => n && n.closest && n.closest(activates))) {
+ try { hit.setAttribute('data-tv3-catcher', '1'); out.occluder.ownContainer = true; } catch (e) { /* best-effort */ }
+ }
// Whether a PERSON would see this layer at all. A leftover consent backdrop still intercepts the
// pointer (elementFromPoint returned it) but can paint nothing -- fully transparent, no visible
// control, heading or text -- so the field looks clear on screen and "dismiss the overlay you
@@ -10285,7 +10296,7 @@ async def _click_stamped_row(page: Any, stamp: str, want: str, timeout: int) ->
try:
if not await handle.evaluate(_STAMPED_ROW_GUARD_JS, want):
return False
- await handle.click(timeout=timeout)
+ await input_dispatch.click_handle(page, handle, timeout=timeout)
finally:
try:
await handle.dispose()
@@ -12193,6 +12204,10 @@ def build_browser_tools(
# backdrop, and telling the model to press Escape on it abandons the verification.
if str(occluder.get("challengeFrame") or "").strip():
return "challenge"
+ # Ahead of `invisible`: a transparent pinned ancestor paints nothing but the field it holds, yet its
+ # catcher is still the thing to click.
+ if occluder.get("ownContainer"):
+ return "own_container"
if occluder.get("invisible"):
return "invisible"
return "named"
@@ -12222,7 +12237,7 @@ def build_browser_tools(
# recorded -- it is page text, and these names carry personal data.
parts = _named_controls(occluder)
branch = _covered_branch(occluder)
- _record_covered(occluder, branch, controls=parts)
+ _record_covered(occluder, branch, controls=[] if branch == "own_container" else parts)
if branch == "clipped":
# A dismissal is the one instruction that cannot be carried out here: the dialog, overlay
# and banner it would name are what this branch is entered BECAUSE the probe ruled out.
@@ -12285,6 +12300,14 @@ def build_browser_tools(
f"({challenge_frame}), so it cannot be {verb}{also}. Its controls: {controls_desc}.",
error_class="covered",
)
+ if branch == "own_container":
+ return ToolResult.error(
+ f"{selector} cannot be {verb}: an element inside {layer_desc}, the container this field "
+ "itself sits in, takes the pointer at the field's position. Do not try to dismiss it; it "
+ 'holds the field. Click [data-tv3-catcher="1"] to activate the field, then retry '
+ f"{selector}.",
+ error_class="covered",
+ )
return ToolResult.error(
f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. "
# The layer may be a general modal, not just a consent wall -- these are every control
@@ -12750,6 +12773,7 @@ def build_browser_tools(
)
else:
await _require_single_target(page, selector)
+ await input_dispatch.approach(page, selector)
pre: dict[str, Any] | None = None
try:
pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))
@@ -12762,7 +12786,7 @@ def build_browser_tools(
# commit baseline must be the POST-hover fingerprint, or a mere highlight would read as
# "its state changed" commit evidence on a no-op click.
try:
- await page.hover(selector, timeout=2000)
+ await input_dispatch.hover(page, selector, timeout=2000)
hovered = await page.evaluate(_MENU_AFTER_JS, await _probe_arg(page, selector))
if isinstance(hovered, dict) and hovered.get("optState"):
pre["optState"] = hovered["optState"]
@@ -12940,7 +12964,7 @@ def build_browser_tools(
"different control, or use the native control directly if the page exposes one"
)
try:
- await _current_page().mouse.click(float(label_click["x"]), float(label_click["y"]))
+ await input_dispatch.click_at(_current_page(), float(label_click["x"]), float(label_click["y"]))
except Exception as e:
return ToolResult.error(
f"click on {selector} via its label failed ({type(e).__name__}) — the page may have "
@@ -12951,13 +12975,7 @@ def build_browser_tools(
# Resolved again here rather than reused: this evaluate is the action, not a probe, and a
# selector naming a control through its host resolves ONLY through the handle -- a node the
# page replaced while the probes ran would be clicked off-document, silently.
- fired = await page.evaluate(
- "(arg) => { const _q = "
- + _ROOT_QUERY_JS
- + "; const el = _q.find(arg.sel) || arg.el;"
- + " if (!el || !el.isConnected) return false; el.click(); return true; }",
- await _probe_arg(page, selector),
- )
+ fired = await input_dispatch.js_click(page, _ROOT_QUERY_JS, await _probe_arg(page, selector))
if not fired:
return ToolResult.error(
f"{selector} left the page before the click could land — it was replaced by a "
@@ -12976,9 +12994,9 @@ def build_browser_tools(
# The probe already answered: the only thing "over" this control is its own label
# (slotted, or a sibling for=id), which the driver's containment check would wait
# 15s to reject.
- await page.click(selector, timeout=_ACTION_TIMEOUT_MS, force=True)
+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS, force=True)
else:
- await page.click(selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS)
except Exception as e:
gone = False
try:
@@ -13051,7 +13069,7 @@ def build_browser_tools(
return ToolResult.ok(f"{selector} is already selected — no change needed")
try:
await page.locator(selector).first.wait_for(state="visible", timeout=3000)
- await page.click(selector, timeout=5000, force=True)
+ await input_dispatch.click(page, selector, timeout=5000, force=True)
except Exception:
raise e from None
else:
@@ -13194,7 +13212,7 @@ def build_browser_tools(
return error
selector = args["selector"]
await _require_single_target(page, selector)
- await page.hover(selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.hover(page, selector, timeout=_ACTION_TIMEOUT_MS)
return ToolResult.ok(f"hovered {selector}")
async def _annotate_challenge_frame(realm: Any, top_page: Any, occluder: dict[str, Any] | None) -> None:
@@ -13307,7 +13325,7 @@ def build_browser_tools(
return False
realm = realm.parent_frame
await page.evaluate("() => { window.__tv3_doc = 1; }")
- await top.mouse.click(x, y)
+ await input_dispatch.click_at(top, x, y)
except Exception:
return False
return True
@@ -13341,7 +13359,7 @@ def build_browser_tools(
try:
# Still focused explicitly: the press need not move the caret, and the display it landed on
# may have no handler that forwards focus to the input.
- await page.focus(selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.focus(page, selector, timeout=_ACTION_TIMEOUT_MS)
held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))
except Exception:
held = None
@@ -13371,7 +13389,7 @@ def build_browser_tools(
# "is this still the same document" exactly -- the same technique the pre-snapshot uses.
await page.evaluate("() => { window.__tv3_doc = 1; }")
try:
- await page.click(selector, timeout=_ACTION_TIMEOUT_MS, force=True)
+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS, force=True)
except Exception as exc:
# Force skips the hit-target check but not the viewport one, which rejects any box of
# at most one square pixel: a segment input kept sub-pixel under its own display layer.
@@ -13398,7 +13416,7 @@ def build_browser_tools(
return False, occluder, "click"
else:
try:
- await page.click(selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS)
except Exception as exc:
# A segmented control can keep its real input off-viewport with tabindex=-1 under an
# aria-hidden display layer: the probe finds nothing on top of it, but the click's
@@ -13413,7 +13431,7 @@ def build_browser_tools(
if focused is False:
# focus() needs no hit target, so it repairs a skin that swallowed the click without
# forwarding it. Typing then goes to the field rather than wherever the caret was.
- await page.focus(selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.focus(page, selector, timeout=_ACTION_TIMEOUT_MS)
return True, None, reach
def _occluder_labels_hold(occluder: dict[str, Any] | None, value: str) -> bool:
@@ -14096,7 +14114,9 @@ def build_browser_tools(
# Short wait on purpose: this field was writable moments ago and visibly moved, so a
# page that has since hidden or disabled it is not going to become actionable -- the
# full timeout would just buy that refusal once per field.
- await page.fill(f'[data-tv3-collateral="{tag}"]', was, timeout=_COLLATERAL_FILL_TIMEOUT_MS)
+ await input_dispatch.fill(
+ page, f'[data-tv3-collateral="{tag}"]', was, timeout=_COLLATERAL_FILL_TIMEOUT_MS
+ )
except Exception:
LOG.debug("taskv3 collateral restore failed", tag=tag)
# Count what actually went back, over the fields that MOVED. Counting over the whole scan would
@@ -14125,7 +14145,7 @@ def build_browser_tools(
if current.strip() and not any(current.strip().lower() == q.strip().lower() for q in typed):
return
try:
- await page.fill(selector, pre_value, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.fill(page, selector, pre_value, timeout=_ACTION_TIMEOUT_MS)
except Exception:
LOG.debug("taskv3 pre-type value restore failed", selector=selector)
@@ -14223,7 +14243,7 @@ def build_browser_tools(
# A page can replace the field during cleanup; writes and Escape must stay on the original node.
# released_input is set only once a dispatch returned, so a fill that raised first stays uncharged.
if current:
- await offer.element.fill("", timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.fill(_current_page(), offer.element, "", timeout=_ACTION_TIMEOUT_MS)
released_input = True
if await offer.element.evaluate(read_value) != "":
continue
@@ -14265,6 +14285,7 @@ def build_browser_tools(
collateral: list[list[str]] | None = None,
query: str | None = None,
press_enter: bool = False,
+ secret: bool = False,
) -> tuple[_TypeaheadPick, str | None, str | None, _Reach]:
# Keystroke-type (so a widget's async suggestion fetch fires on real key events). Snapshot the
# visible DOM BEFORE the focus click, not just before typing: a widget that opens its full list on
@@ -14279,6 +14300,7 @@ def build_browser_tools(
# Same reason: fill()/type() below open the widget's own list (aria-expanded=true) for the rest
# of this attempt, and ownCommittedSurface reads nothing while it is open.
pre_own = await _own_surface_text(page, selector)
+ await input_dispatch.approach(page, selector)
try:
await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
@@ -14306,8 +14328,10 @@ def build_browser_tools(
# Only a field the click could not reach can misroute its keys, and only that path restores.
# A checked click leaves the page untagged and pays nothing.
collateral.extend(await _capture_collateral(page, selector))
- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)
- await page.type(selector, typed, delay=15, timeout=_typing_timeout_ms(typed))
+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.type_keys(
+ page, selector, typed, delay=15, timeout=_typing_timeout_ms(typed), secret=secret, replace=True
+ )
if collateral:
collateral[:] = await _collateral_moved_while_typing(page, collateral)
if not presnapshot_ok or reach != "click":
@@ -14407,7 +14431,7 @@ def build_browser_tools(
try:
try:
- await page.press(selector, "Enter", timeout=5000)
+ await input_dispatch.press(page, selector, "Enter", timeout=5000)
except Exception:
return _settled(
ToolResult.error(
@@ -14500,9 +14524,9 @@ def build_browser_tools(
return None
try:
if element is None:
- await page.press(selector, "Escape", timeout=5000)
+ await input_dispatch.press(page, selector, "Escape", timeout=5000)
else:
- await element.press("Escape", timeout=5000)
+ await input_dispatch.press(page, element, "Escape", timeout=5000)
except Exception:
LOG.debug("taskv3 lingering typeahead list close failed", selector=selector)
return None
@@ -14776,7 +14800,7 @@ def build_browser_tools(
for _ in range(4):
if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):
return
- await locator.press("Backspace")
+ await input_dispatch.press(page, locator, "Backspace")
return
held = _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector))
if held:
@@ -14784,15 +14808,15 @@ def build_browser_tools(
caret = await locator.evaluate(_CARET_TO_END_JS, timeout=2000)
# End is only a caret key in a text box; an ARIA spinbutton takes it as "set to maximum".
if caret == "end_key" and await locator.evaluate(_IS_TEXT_BOX_JS, timeout=2000):
- await locator.press("End")
+ await input_dispatch.press(page, locator, "End")
elif caret == "end_key":
# No caret key here, so Delete clears the digits after the press point.
for _ in range(len(held.group(0))):
- await locator.press("Delete")
+ await input_dispatch.press(page, locator, "Delete")
if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):
return
for _ in range(len(held.group(0)) if held else 0):
- await locator.press("Backspace")
+ await input_dispatch.press(page, locator, "Backspace")
if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):
return
except Exception:
@@ -14888,7 +14912,7 @@ def build_browser_tools(
except Exception:
return "focus"
try:
- await locator.click(timeout=2000)
+ await input_dispatch.click(page, locator, timeout=2000)
except Exception:
return "focus"
return "click"
@@ -14920,7 +14944,9 @@ def build_browser_tools(
# Pressed through Playwright as the layer ELEMENT, never forced: its actionability and hit-target checks
# refuse the press when the layer has been hidden or covered since the aim measured it.
try:
- await layer.click(position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500)
+ await input_dispatch.click_handle(
+ page, layer, position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500
+ )
except Exception:
return "focus"
return "mirror"
@@ -14937,14 +14963,14 @@ def build_browser_tools(
pass
aim = await _aim_date_segment(page, selector, locator)
try:
- await locator.focus(timeout=2000)
+ await input_dispatch.focus(page, locator, timeout=2000)
except Exception:
return False, aim
await _clear_date_segment(page, selector, locator)
# locator.fill() does not commit digits into a date spinbutton segment; only real keystrokes
# advance it, so this and the element-focused fallback below both TYPE rather than fill.
try:
- await _current_page().keyboard.type(digits, delay=40)
+ await input_dispatch.type_keys(_current_page(), None, digits, delay=40)
except Exception:
pass
if await _date_segment_holds(page, selector, digits):
@@ -14952,9 +14978,9 @@ def build_browser_tools(
try:
if _date_segment_aim_on():
aim = await _aim_date_segment(page, selector, locator)
- await locator.focus(timeout=2000)
+ await input_dispatch.focus(page, locator, timeout=2000)
await _clear_date_segment(page, selector, locator)
- await locator.press_sequentially(digits, delay=40)
+ await input_dispatch.type_keys(page, locator, digits, delay=40)
except Exception:
return False, aim
return await _date_segment_holds(page, selector, digits), aim
@@ -15190,7 +15216,6 @@ def build_browser_tools(
url_before: str | None,
) -> ToolResult:
n = len(text)
- # Registered as a secret only where it came from one: the one-time-code resolver registers the whole code.
url_before_fill = await _url(_current_page())
field = f"the {n}-box code field at {selector}"
@@ -15232,12 +15257,12 @@ def build_browser_tools(
held: str | None = None
try:
# fill() waits for the box to be enabled, so its timeout bounds the wait for the box before it.
- await box.fill(char, timeout=_CODE_BOX_ENABLE_WAIT_MS)
+ await input_dispatch.fill(page, box, char, timeout=_CODE_BOX_ENABLE_WAIT_MS)
typed = i + 1
held = await box.input_value(timeout=1000)
if not held:
# v1's fallback: a box that drops a programmatic value may still take the key.
- await box.press_sequentially(char, timeout=1000)
+ await input_dispatch.type_keys(page, box, char, timeout=1000, secret=secret)
held = await box.input_value(timeout=1000)
except Exception:
held = None
@@ -15296,7 +15321,7 @@ def build_browser_tools(
)
if not press_enter:
return ToolResult.ok(filled)
- await page.press(f'[data-tv3-codebox="{n - 1}"]', "Enter")
+ await input_dispatch.press(page, f'[data-tv3-codebox="{n - 1}"]', "Enter")
return ToolResult.ok(f"{filled}, then pressed Enter", data={"url_before": url_before})
kept = sum(bool(box_state.get("value")) for box_state in state)
if remounted:
@@ -15405,8 +15430,8 @@ def build_browser_tools(
)
if not matched:
return ToolResult.error(
- f"the field at {selector} has {run} one-character boxes from that box on, but the text has {len(code)} "
- "characters once spaces and dashes are dropped. The length does not match the box count, "
+ f"the field at {selector} has {run} one-character boxes from that box on, but the text has "
+ f"{len(code)} characters once spaces and dashes are dropped. The length does not match the box count, "
"so nothing was typed. Re-observe the field and type one character for each box.",
error_class="text_not_held",
)
@@ -15456,7 +15481,7 @@ def build_browser_tools(
if not reachable:
return _covered_error(selector, occluder)
if clear:
- await page.fill(selector, text, timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.fill(page, selector, text, timeout=_ACTION_TIMEOUT_MS)
if box_text and await _read_field_value(page, selector) != text:
return ToolResult.error(
f"typed into {selector}, but it takes one character and does not hold the typed text "
@@ -15491,7 +15516,9 @@ def build_browser_tools(
at_end = False
if at_end:
# page.type() would focus the field again, and a fresh focus puts the caret back at the start.
- sent = await _type_keys_before_deadline(text, press_end=at_end == "end_key")
+ sent = await _type_keys_before_deadline(
+ text, press_end=at_end == "end_key", secret=text != args.get("text", "")
+ )
if sent < len(text):
progress = (
"partway" if text != args.get("text", "") else f"after {sent} of {len(text)} characters"
@@ -15503,7 +15530,9 @@ def build_browser_tools(
error_class="text_not_held",
)
else:
- await page.type(selector, text, timeout=_typing_timeout_ms(text))
+ await input_dispatch.type_keys(
+ page, selector, text, timeout=_typing_timeout_ms(text), secret=text != args.get("text", "")
+ )
not_held = await _appended_text_not_held(
field, selector, text, before, text_is_secret=text != args.get("text", "")
)
@@ -15513,7 +15542,7 @@ def build_browser_tools(
if not_held is not None:
return not_held
if press_enter:
- await page.press(selector, "Enter")
+ await input_dispatch.press(page, selector, "Enter")
return ToolResult.ok(f"typed into {selector}", data={"url_before": url_before})
return ToolResult.ok(f"typed into {selector}")
@@ -15543,7 +15572,7 @@ def build_browser_tools(
# The editable element focus landed on after focusing the target, or None when it landed on none.
# Fail-open: a probe error must not refuse a field type() could have filled.
try:
- await page.focus(selector, timeout=2000)
+ await input_dispatch.focus(page, selector, timeout=2000)
handle = await page.locator(selector).first.evaluate_handle(_FOCUS_IN_EDITABLE_JS, timeout=2000)
except Exception:
return _FOCUS_PROBE_FAILED
@@ -15552,14 +15581,14 @@ def build_browser_tools(
await handle.dispose()
return element
- async def _type_keys_before_deadline(text: str, *, press_end: bool) -> int:
+ async def _type_keys_before_deadline(text: str, *, press_end: bool, secret: bool) -> int:
# One key at a time: a single keyboard.type(text) keeps typing in the browser after its caller stops
# waiting, into whatever field the next action focuses. The keyboard is the page's; a frame has none.
- keyboard = _current_page().keyboard
+ top = _current_page()
deadline = time.monotonic() + _typing_timeout_ms(text) / 1000
if press_end:
try:
- await asyncio.wait_for(keyboard.press("End"), deadline - time.monotonic())
+ await asyncio.wait_for(input_dispatch.press(top, None, "End"), deadline - time.monotonic())
except asyncio.TimeoutError:
return 0
for sent, key in enumerate(text):
@@ -15567,7 +15596,7 @@ def build_browser_tools(
if remaining <= 0:
return sent
try:
- await asyncio.wait_for(keyboard.type(key), remaining)
+ await asyncio.wait_for(input_dispatch.type_keys(top, None, key, secret=secret), remaining)
except asyncio.TimeoutError:
return sent
return len(text)
@@ -15662,7 +15691,7 @@ def build_browser_tools(
collateral: list[list[str]] = []
try:
pick, pre_value, pre_own, reach = await _type_and_commit(
- page, selector, text, rounds=3, focus_fallback=True, collateral=collateral
+ page, selector, text, rounds=3, focus_fallback=True, collateral=collateral, secret=text_is_secret
)
except _FieldCovered as exc:
return _covered_error(exc.selector, exc.occluder)
@@ -15683,7 +15712,7 @@ def build_browser_tools(
reacted = await _find_suggestion_rows(page, selector, text, any_region=True) is not None
if not reacted:
try:
- await _current_page().keyboard.press("Tab")
+ await input_dispatch.press(_current_page(), None, "Tab")
except Exception:
pass
held = await _read_field_value(page, selector)
@@ -15872,12 +15901,13 @@ def build_browser_tools(
menu_open = False
if menu_open:
try:
- await _current_page().keyboard.press("Escape")
+ await input_dispatch.press(_current_page(), None, "Escape")
await asyncio.sleep(0.1)
except Exception:
pass
async def _open_and_enumerate() -> tuple[dict[str, Any] | None, ToolResult | None]:
+ await input_dispatch.approach(page, selector)
try:
await page.evaluate(_PRESNAPSHOT_JS)
except Exception:
@@ -15893,7 +15923,7 @@ def build_browser_tools(
# 5s, not the 15s a routine click waits: the control is already present (we just typed
# into it, or it is a visible button), so it opens at once — a long wait here only delays
# the error on a field that unmounted itself, which must fail loudly, not slowly.
- await page.click(selector, timeout=5000)
+ await input_dispatch.click(page, selector, timeout=5000)
except Exception:
return None, ToolResult.error(
f"could not click {selector} to open its option list — the field is NOT filled; "
@@ -16289,7 +16319,7 @@ def build_browser_tools(
contenders = _value_contenders(value, row_texts)
def _ambiguous_error() -> ToolResult:
- named = "; ".join(repr(t[:60]) for t in contenders[:15])
+ named = "; ".join(repr(_model_text(t, 60)) for t in contenders[:15])
return ToolResult.error(
f"{value!r} is ambiguous in {selector}'s list — it names more than one option "
f"({named}); pass the one option's full text"
@@ -16355,7 +16385,7 @@ def build_browser_tools(
async def _take_back() -> ToolResult | None:
try:
- await page.fill(box_sel, str(box.get("prior") or ""), timeout=2000)
+ await input_dispatch.fill(page, box_sel, str(box.get("prior") or ""), timeout=2000)
except Exception:
LOG.debug("taskv3 popup filter take-back failed", selector=selector)
# A box that is not a filter (a key sink that commits the focused row) shows up here whatever
@@ -16422,9 +16452,11 @@ def build_browser_tools(
base_rows, _, _ = await _read_rows(mark=True)
base_texts = [str(o.get("text") or "") for o in base_rows]
try:
- await page.fill(box_sel, "", timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.clear(page, box_sel, timeout=_ACTION_TIMEOUT_MS)
cleared = await _box_value()
- await page.type(box_sel, query[:1], delay=15, timeout=_typing_timeout_ms(query[:1]))
+ await input_dispatch.type_keys(
+ page, box_sel, query[:1], delay=15, timeout=_typing_timeout_ms(query[:1])
+ )
except Exception:
return await _take_back()
# A key sink swallows the keys it forwards to the list, so a box whose value does not newly show the
@@ -16454,7 +16486,9 @@ def build_browser_tools(
)
try:
if query[1:]:
- await page.type(box_sel, query[1:], delay=15, timeout=_typing_timeout_ms(query[1:]))
+ await input_dispatch.type_keys(
+ page, box_sel, query[1:], delay=15, timeout=_typing_timeout_ms(query[1:])
+ )
except Exception:
return await _take_back()
last: list[str] | None = None
@@ -16534,7 +16568,9 @@ def build_browser_tools(
several = list(dict.fromkeys(contenders + several))
if len(several) < 2 or verdict == "incomplete":
several = []
- listed = "; ".join(repr(t[:60]) for t in (several or [str(o.get("text") or "") for o in filtered])[:15])
+ listed = "; ".join(
+ repr(_model_text(t, 60)) for t in (several or [str(o.get("text") or "") for o in filtered])[:15]
+ )
more = f"; +{len(filtered) - 15} more" if len(filtered) > 15 and not several else ""
if visible is not None:
return _unproven_row_error(
@@ -16684,7 +16720,7 @@ def build_browser_tools(
# itself.
try:
if await page.evaluate(_MENU_OPEN_JS, probe):
- await _current_page().keyboard.press("Escape")
+ await input_dispatch.press(_current_page(), None, "Escape")
except Exception:
pass
return ToolResult.ok(f"{matched!r} was already selected for {selector}; left it as is")
@@ -16906,14 +16942,16 @@ def build_browser_tools(
# anything (closed over from _commit_custom_combobox_attempt) — re-reading it here
# would see the list THIS rung's own typing just opened and always read as changed.
try:
- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)
await asyncio.sleep(0.2)
# A rung is a fresh question, so it needs a fresh answer to "what appeared in
# reaction": the snapshot from before the first attempt would mark the rows the
# PREVIOUS query left on screen as pre-existing, and a widget that keeps its row
# nodes across a re-search would then have no reaction to show at all.
await page.evaluate(_PRESNAPSHOT_JS)
- await page.type(selector, rung, delay=15, timeout=_typing_timeout_ms(rung))
+ await input_dispatch.type_keys(
+ page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung), replace=True
+ )
except Exception:
return []
rung_pick = await _commit_typeahead(
@@ -16962,7 +17000,7 @@ def build_browser_tools(
# the focus pass so those rows are recorded as offered, exactly as if the field had opened
# showing them — _FOCUS_OFFERED_LABELS_JS then reads them under the same own-list attribution.
try:
- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)
+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)
except Exception:
return
await asyncio.sleep(0.4)
@@ -17254,16 +17292,20 @@ def build_browser_tools(
# holds ['Alpha']` and told the model to re-pass a set it had never asked for.
if label_list is not None:
by_label, asked = True, label_list
- await page.select_option(selector, label=label_list, timeout=_ACTION_TIMEOUT_MS, force=force)
+ await input_dispatch.select_option(
+ page, selector, label=label_list, timeout=_ACTION_TIMEOUT_MS, force=force
+ )
elif value_list is not None:
by_label, asked = False, value_list
- await page.select_option(selector, value=value_list, timeout=_ACTION_TIMEOUT_MS, force=force)
+ await input_dispatch.select_option(
+ page, selector, value=value_list, timeout=_ACTION_TIMEOUT_MS, force=force
+ )
elif label is not None:
by_label, asked = True, [label]
- await page.select_option(selector, label=label, timeout=_ACTION_TIMEOUT_MS, force=force)
+ await input_dispatch.select_option(page, selector, label=label, timeout=_ACTION_TIMEOUT_MS, force=force)
else:
by_label, asked = False, ([value] if isinstance(value, str) else [])
- await page.select_option(selector, value=value, timeout=_ACTION_TIMEOUT_MS, force=force)
+ await input_dispatch.select_option(page, selector, value=value, timeout=_ACTION_TIMEOUT_MS, force=force)
# A set-valued control is read back whether or not it was forced: without it a call that
# discarded every prior selection reports the same bare success as one that added to them.
if not force and not is_multi:
@@ -17334,13 +17376,13 @@ def build_browser_tools(
url_before = await _url(_current_page())
if selector:
await _require_single_target(page, selector)
- await page.press(selector, key)
+ await input_dispatch.press(page, selector, key)
else:
# Page-level, not realm-level: a Frame has no `keyboard`, and an unaddressed keypress goes
# to whatever holds focus regardless of which document that is. Reached only with no
# selector, so the realm is the page today -- written through _current_page() so it stays
# correct if that ever stops being true.
- await _current_page().keyboard.press(key)
+ await input_dispatch.press(_current_page(), None, key)
return ToolResult.ok(f"pressed {key}", data={"url_before": url_before})
async def scroll(args: dict[str, Any]) -> ToolResult:
@@ -17360,7 +17402,7 @@ def build_browser_tools(
# Reachable WITH a selector -- a ref that resolved to a selector now matching nothing falls
# through to here -- so `page` can be a child frame, and a Frame has no `mouse`. The wheel is a
# viewport gesture either way, so it belongs to the page.
- await _current_page().mouse.wheel(0, amount)
+ await input_dispatch.wheel(_current_page(), 0, amount)
return ToolResult.ok(f"scrolled {amount}px")
async def wait(args: dict[str, Any]) -> ToolResult:
@@ -17627,7 +17669,7 @@ def build_browser_tools(
# picker it opens, as v1 does. The picker belongs to the page, whichever frame opened it.
try:
async with _current_page().expect_file_chooser(timeout=_FILE_CHOOSER_TIMEOUT_MS) as chooser_info:
- await el.click(timeout=_FILE_CHOOSER_TIMEOUT_MS)
+ await input_dispatch.click_handle(page, el, timeout=_FILE_CHOOSER_TIMEOUT_MS)
chooser = await chooser_info.value
except Exception as exc:
if not is_driver_timeout_error(exc):
@@ -17639,10 +17681,10 @@ def build_browser_tools(
{**staged, "page_state_changed": True},
error_class="no_file_input",
)
- await chooser.set_files([local_path])
+ await input_dispatch.set_files(chooser, [local_path])
file_input = chooser.element
else:
- await file_input.set_input_files([local_path])
+ await input_dispatch.set_input_files(file_input, [local_path])
populated = await _input_holds_file(file_input)
# Settle + a small randomized delay so the upload and a following submit are not dispatched
# in the same instant, matching v1's upload cadence (the engine that clears this step reliably).
@@ -17674,7 +17716,7 @@ def build_browser_tools(
new_lines = _newly_rendered_lines(text_before, text_after)
new_text = "\n".join(new_lines)
said = " | ".join(line for line in new_lines if _mentions_filename(line, local_path))
- said = said[:_FILENAME_MENTION_CHARS]
+ said = _model_text(said, _FILENAME_MENTION_CHARS)
if probe.saw_upload() and not _UPLOAD_REJECTION_WORDS.search(new_text):
LOG.info("taskv3 file_upload input cleared after attach but the page shows the uploaded file")
return ToolResult.ok(
diff --git a/skyvern/webeye/utils/captcha_solver.py b/skyvern/webeye/utils/captcha_solver.py
index 66ff4f06b..2742a7bf8 100644
--- a/skyvern/webeye/utils/captcha_solver.py
+++ b/skyvern/webeye/utils/captcha_solver.py
@@ -17,14 +17,16 @@ from __future__ import annotations
import asyncio
import contextlib
import time
+from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING, Any, Literal
from urllib.parse import urlparse
import structlog
+from playwright.async_api import ElementHandle
from playwright.async_api import Error as PlaywrightError
-from playwright.async_api import Frame, Page
+from playwright.async_api import Frame, Locator, Page
from skyvern.config import settings
from skyvern.forge import app
@@ -48,6 +50,9 @@ class ChallengeStatus(StrEnum):
ChallengeArm = Literal["vendor_handler", "dom_checkbox", "recaptcha_anchor_frame", "extension", "token"]
+# How the ladder clicks a widget; a caller that routes its input elsewhere supplies its own.
+ChallengeClick = Callable[[Locator], Awaitable[None]]
+ChallengeHandleClick = Callable[[ElementHandle], Awaitable[None]]
ChallengePageState = Literal["clear", "challenged", "not_rechecked"]
@@ -313,6 +318,8 @@ async def solve_challenge_ladder(
workflow_run_id: str | None = None,
browser_session_id: str | None = None,
probe_child_frames: bool = False,
+ click: ChallengeClick | None = None,
+ click_handle: ChallengeHandleClick | None = None,
) -> bool:
"""``solve_challenge`` as a bool: True when solved, False when no challenge is on the page, and
CaptchaChallengeUnsolvedError when one is present but unsolved or unsupported."""
@@ -322,6 +329,8 @@ async def solve_challenge_ladder(
workflow_run_id=workflow_run_id,
browser_session_id=browser_session_id,
probe_child_frames=probe_child_frames,
+ click=click,
+ click_handle=click_handle,
)
if outcome.status in (ChallengeStatus.UNSOLVED, ChallengeStatus.UNSUPPORTED):
raise CaptchaChallengeUnsolvedError("CAPTCHA could not be solved.")
@@ -335,6 +344,8 @@ async def solve_challenge(
workflow_run_id: str | None = None,
browser_session_id: str | None = None,
probe_child_frames: bool = False,
+ click: ChallengeClick | None = None,
+ click_handle: ChallengeHandleClick | None = None,
) -> ChallengeOutcome:
"""Solve a detected challenge inside one captcha-solver lifecycle scope and report what happened. A vendor page
goes only to its handler and counts as cleared only when a fresh probe finds neither the vendor nor a widget."""
@@ -378,6 +389,8 @@ async def solve_challenge(
workflow_run_id=workflow_run_id,
browser_session_id=browser_session_id,
probe_child_frames=probe_child_frames,
+ click=click or _click_directly,
+ click_handle=click_handle or _click_directly,
)
except TimeoutError:
LOG.info("CAPTCHA ladder stopped at its budget", arm="ladder")
@@ -443,6 +456,10 @@ async def _rendered_frame_vendor(page: Page | RecordingPage) -> str | None:
return await rendered_challenge_vendor([frame for frame in page.frames if frame.parent_frame is not None])
+async def _click_directly(target: Locator | ElementHandle) -> None:
+ await target.click()
+
+
async def _solve_challenge_ladder_impl(
page: Page | RecordingPage,
*,
@@ -451,6 +468,8 @@ async def _solve_challenge_ladder_impl(
workflow_run_id: str | None = None,
browser_session_id: str | None = None,
probe_child_frames: bool = False,
+ click: ChallengeClick = _click_directly,
+ click_handle: ChallengeHandleClick = _click_directly,
) -> ChallengeArm | None:
"""The arm that solved a detected widget, or None when the cheap probes find none and no solver is called.
Raises CaptchaChallengeUnsolvedError when no arm, each clamped to the budget left since ``start``, clears it."""
@@ -474,14 +493,14 @@ async def _solve_challenge_ladder_impl(
candidate = checkbox.first
try:
if await candidate.is_visible() and await candidate.is_enabled():
- await candidate.click()
+ await click(candidate)
await page.wait_for_timeout(100)
if await candidate.is_checked() or await _bounded_locator_count(checkbox) == 0:
continuation = page.locator(_CAPTCHA_CONTINUE_SELECTOR)
if await _bounded_locator_count(continuation) == 1:
continuation_candidate = continuation.first
if await continuation_candidate.is_visible() and await continuation_candidate.is_enabled():
- await continuation_candidate.click()
+ await click(continuation_candidate)
await page.wait_for_timeout(100)
if await _bounded_locator_count(checkbox) == 0:
return "dom_checkbox"
@@ -515,7 +534,7 @@ async def _solve_challenge_ladder_impl(
if await candidate.get_attribute("aria-checked") == "true":
break
page_url_before_click = urlparse(page.url)._replace(fragment="").geturl()
- await candidate.click()
+ await click_handle(candidate)
anchor_clicked = True
LOG.info("CAPTCHA anchor frame clicked", arm="recaptcha_anchor_frame")
await page.wait_for_timeout(_RECAPTCHA_ANCHOR_SETTLE_MS)
diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.py
index 5740f91af..d090234df 100644
--- a/tests/unit/test_browser_effect_sink_enumeration.py
+++ b/tests/unit/test_browser_effect_sink_enumeration.py
@@ -1,4 +1,5 @@
import ast
+import re
from collections import Counter
from pathlib import Path
@@ -72,6 +73,24 @@ _DISCOVERED_BROWSER_API_CALLS = {
{"clear": 1, "click": 2, "move": 2, "scroll_into_view_if_needed": 1, "type": 3, "wheel": 1}
),
"skyvern/forge/sdk/event/factory.py": Counter({"click": 1, "wheel": 1}),
+ "skyvern/forge/taskv3/input_dispatch.py": Counter(
+ {
+ "click": 6,
+ "evaluate": 2,
+ "fill": 3,
+ "focus": 2,
+ "hover": 1,
+ "insert_text": 1,
+ "press": 3,
+ "press_sequentially": 1,
+ "scroll_into_view_if_needed": 1,
+ "select_option": 2,
+ "set_files": 1,
+ "set_input_files": 1,
+ "type": 2,
+ "wheel": 1,
+ }
+ ),
"skyvern/webeye/real_browser_state.py": Counter({"close": 6, "evaluate": 1, "goto": 1, "new_page": 3, "reload": 2}),
}
@@ -125,6 +144,8 @@ _EVALUATE_CALLERS = {
}
),
"skyvern/webeye/real_browser_state.py": Counter({"stop_page_loading": 1}),
+ # The settle wait after a treatment cursor approach, and the hidden-native-control click V3 fires in-page.
+ "skyvern/forge/taskv3/input_dispatch.py": Counter({"approach": 1, "js_click": 1}),
# OTP box/scope/document marker writes, visual masks and read-only hit-test geometry.
"skyvern/webeye/utils/dom.py": Counter(
{"apply_secret_visual_mask": 1, "mark_totp_box": 2, "blur": 1, "_pointer_interceptor_matches_label": 1}
@@ -165,6 +186,7 @@ def _owned_source_paths() -> tuple[str, ...]:
Path("skyvern/webeye/utils/dom.py"),
*Path("skyvern/webeye/actions").glob("*.py"),
*Path("skyvern/forge/sdk/event").glob("*.py"),
+ Path("skyvern/forge/taskv3/input_dispatch.py"),
}
return tuple(sorted(path.as_posix() for path in paths))
@@ -239,24 +261,142 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:
}
assert observed == _DISCOVERED_BROWSER_API_CALLS
- assert sum(sum(methods.values()) for methods in observed.values()) == 184
+ assert sum(sum(methods.values()) for methods in observed.values()) == 211
handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)
classified_non_browser = Counter(
{signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES}
)
assert classified_non_browser == _NON_BROWSER_CANDIDATES
assert sum(_NON_BROWSER_CANDIDATES.values()) == 7
- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 177
+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 204
def test_every_raw_evaluate_call_is_classified() -> None:
observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}
assert observed == _EVALUATE_CALLERS
- assert sum(sum(callers.values()) for callers in observed.values()) == 37
+ assert sum(sum(callers.values()) for callers in observed.values()) == 39
def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:
observed = {path: callers for path in _owned_source_paths() if (callers := _cdp_send_callers(path))}
assert observed == _CDP_SENDS
+
+
+# Task V3 sends input only through input_dispatch, which picks the plain or the humanized transport per run.
+_INPUT_DISPATCH = Path("skyvern/forge/taskv3/input_dispatch.py")
+_TASKV3_BANNED_INPUT_METHODS = frozenset(
+ """check click dblclick dispatch_event drag_and_drop drag_to fill focus hover insert_text press press_sequentially select_option
+ select_text set_checked set_files set_input_files tap type uncheck""".split()
+)
+_TASKV3_BANNED_DEVICES = frozenset({"mouse", "keyboard"})
+_IN_PAGE_INPUT_JS = re.compile(r"\.click\(\)|dispatchEvent\(|new (Mouse|Keyboard|Pointer)Event|Input\.dispatch")
+# `clear` is also a list and dict method, so it is banned only on a receiver shaped like an element.
+_ELEMENT_SOURCES = frozenset({"locator", "query_selector", "element_handle", "nth", "first", "last"})
+
+# Where each input_dispatch gesture reaches the event strategy factory: a gesture that loses one sends treatment
+# runs down the plain path without anything else going red.
+_INPUT_DISPATCH_FACTORY_CALLS = Counter(
+ {
+ ("_move_to_element", "move_cursor"): 1,
+ ("approach", "move_to_element"): 1,
+ ("click", "click_element"): 1,
+ ("click", "move_to_element"): 1,
+ ("click_at", "move_cursor"): 1,
+ ("hover", "move_to_element"): 1,
+ ("clear", "clear_field"): 1,
+ ("type_keys", "type_text"): 1,
+ ("wheel", "scroll_by"): 1,
+ ("arm_fields", "registered_profile"): 1,
+ }
+)
+
+
+def _enclosing_function(parents: dict[ast.AST, ast.AST], node: ast.AST) -> str:
+ current = node
+ while current in parents:
+ current = parents[current]
+ if isinstance(current, (ast.AsyncFunctionDef, ast.FunctionDef)):
+ return current.name
+ return "<module>"
+
+
+def _is_element(node: ast.expr, element_names: set[str]) -> bool:
+ if isinstance(node, ast.Await):
+ node = node.value
+ if isinstance(node, ast.Call):
+ node = node.func
+ if isinstance(node, ast.Name):
+ return node.id in element_names
+ return isinstance(node, ast.Attribute) and (node.attr in _ELEMENT_SOURCES or node.attr.startswith("get_by_"))
+
+
+def _direct_input_sites(path: Path, only_function: str | None = None) -> list[str]:
+ tree = ast.parse(path.read_text())
+ parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)}
+ element_names = {
+ target.id
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Assign) and _is_element(node.value, set())
+ for target in node.targets
+ if isinstance(target, ast.Name)
+ }
+ sites: list[str] = []
+ for node in ast.walk(tree):
+ found: str | None = None
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ receiver = node.func.value
+ dispatched = isinstance(receiver, ast.Name) and receiver.id == "input_dispatch"
+ if node.func.attr in _TASKV3_BANNED_INPUT_METHODS and not dispatched:
+ found = f"{node.func.attr} on {ast.unparse(node.func.value)}"
+ elif node.func.attr == "clear" and _is_element(receiver, element_names):
+ found = f"clear on {ast.unparse(receiver)}"
+ elif isinstance(node, ast.Attribute) and node.attr in _TASKV3_BANNED_DEVICES:
+ found = f".{node.attr} on {ast.unparse(node.value)}"
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "getattr"
+ and len(node.args) >= 2
+ and isinstance(node.args[1], ast.Constant)
+ and node.args[1].value in _TASKV3_BANNED_INPUT_METHODS | _TASKV3_BANNED_DEVICES
+ ):
+ found = f"getattr {node.args[1].value!r}"
+ if isinstance(node, ast.Constant) and isinstance(node.value, str) and _IN_PAGE_INPUT_JS.search(node.value):
+ found = "in-page input script"
+ if found is None:
+ continue
+ function = _enclosing_function(parents, node)
+ if only_function is not None and function != only_function:
+ continue
+ sites.append(f"{path}:{getattr(node, 'lineno', '?')} {function} {found}")
+ return sites
+
+
+def test_taskv3_input_flows_only_through_input_dispatch() -> None:
+ sites = [
+ site
+ for path in sorted(Path("skyvern/forge/taskv3").rglob("*.py"))
+ if path != _INPUT_DISPATCH
+ for site in _direct_input_sites(path)
+ ]
+ # The captcha ladder lives outside taskv3 and V3 hands it the click to use.
+ sites += _direct_input_sites(Path("skyvern/webeye/utils/captcha_solver.py"), "_solve_challenge_ladder_impl")
+
+ assert sites == []
+
+
+def test_input_dispatch_reaches_the_event_strategy_factory_from_every_humanized_gesture() -> None:
+ tree = ast.parse(_INPUT_DISPATCH.read_text())
+ parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)}
+ observed = Counter(
+ (_enclosing_function(parents, node), node.func.attr)
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "EventStrategyFactory"
+ )
+
+ assert observed == _INPUT_DISPATCH_FACTORY_CALLS
diff --git a/tests/unit/test_taskv3_captcha_tools.py b/tests/unit/test_taskv3_captcha_tools.py
index f12006616..e412c6b72 100644
--- a/tests/unit/test_taskv3_captcha_tools.py
+++ b/tests/unit/test_taskv3_captcha_tools.py
@@ -11,7 +11,7 @@ import pytest
from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
-from skyvern.forge.taskv3 import captcha_tools
+from skyvern.forge.taskv3 import captcha_tools, input_dispatch
from skyvern.webeye.utils import captcha_solver as captcha_solver_module
from skyvern.webeye.utils.captcha_solver import CaptchaChallengeUnsolvedError
from tests.unit.conftest import OcrRecordingAgentFunction, ScopeRecordingAgentFunction
@@ -206,9 +206,20 @@ async def test_solve_captcha_threads_ids(monkeypatch: pytest.MonkeyPatch) -> Non
_task(workflow_run_id="wr_9", browser_session_id="bs_9"), _provider(page), organization_id="o_9"
)
await tools[0].handler({})
- ladder.assert_awaited_once_with(
- page, organization_id="o_9", workflow_run_id="wr_9", browser_session_id="bs_9", probe_child_frames=True
- )
+ ladder.assert_awaited_once()
+ kwargs = dict(ladder.await_args.kwargs)
+ # The ladder's widget clicks go through V3's input dispatch, bound to the page being solved.
+ click = kwargs.pop("click")
+ click_handle = kwargs.pop("click_handle")
+ assert (click.func, click.args) == (input_dispatch.click, (page,))
+ assert (click_handle.func, click_handle.args) == (input_dispatch.click_handle, (page,))
+ assert ladder.await_args.args == (page,)
+ assert kwargs == {
+ "organization_id": "o_9",
+ "workflow_run_id": "wr_9",
+ "browser_session_id": "bs_9",
+ "probe_child_frames": True,
+ }
@pytest.mark.asyncio
diff --git a/tests/unit/test_taskv3_input_dispatch.py b/tests/unit/test_taskv3_input_dispatch.py
new file mode 100644
index 000000000..77833f749
--- /dev/null
+++ b/tests/unit/test_taskv3_input_dispatch.py
@@ -0,0 +1,384 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable, Iterator
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from playwright.async_api import ElementHandle, Locator, Page
+
+from skyvern.forge import app
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import RunArm, SkyvernContext
+from skyvern.forge.sdk.event.base import CursorEventStrategy, InputEventStrategy, ScrollEventStrategy
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
+from skyvern.forge.taskv3 import input_dispatch
+from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, resolve_run_arm
+
+Call = tuple[str, tuple[Any, ...], dict[str, Any]]
+
+
+class _Device:
+ def __init__(self, log: list[Call], name: str) -> None:
+ self._log = log
+ self._name = name
+
+ def __getattr__(self, attr: str) -> Callable[..., Awaitable[None]]:
+ async def record(*args: Any, **kwargs: Any) -> None:
+ self._log.append((f"{self._name}.{attr}", args, kwargs))
+
+ return record
+
+
+class _FakeLocator(_Device):
+ def __init__(self, page: _FakePage, selector: str) -> None:
+ super().__init__(page.calls, f"locator({selector})")
+ self.page = page
+ self.first = self
+
+
+class _FakeHandle(_Device):
+ def __init__(self, page: _FakePage) -> None:
+ super().__init__(page.calls, "handle")
+
+ async def bounding_box(self) -> dict[str, float]:
+ return {"x": 10.0, "y": 20.0, "width": 4.0, "height": 6.0}
+
+
+class _FakePage(_Device):
+ def __init__(self) -> None:
+ self.calls: list[Call] = []
+ super().__init__(self.calls, "page")
+ self.keyboard = _Device(self.calls, "keyboard")
+ self.mouse = _Device(self.calls, "mouse")
+
+ def __getattr__(self, attr: str) -> Callable[..., Awaitable[None]]:
+ # A top-level page has neither, which is how the dispatcher tells it from a frame.
+ if attr in ("page", "parent_frame"):
+ raise AttributeError(attr)
+ return super().__getattr__(attr)
+
+ def locator(self, selector: str) -> _FakeLocator:
+ return _FakeLocator(self, selector)
+
+
+class _RecordingCursor(CursorEventStrategy):
+ def __init__(self, log: list[Call]) -> None:
+ self._log = log
+
+ async def move_to(self, page: Page, x: float, y: float) -> None:
+ self._log.append(("cursor.move_to", (x, y), {}))
+
+ async def move_to_element(self, page: Page, locator: Locator) -> tuple[float, float]:
+ self._log.append(("cursor.move_to_element", (locator,), {}))
+ return 0.0, 0.0
+
+ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:
+ self._log.append(("cursor.click", (locator,), {"timeout": timeout}))
+
+
+class _RecordingInput(InputEventStrategy):
+ def __init__(self, log: list[Call]) -> None:
+ self._log = log
+
+ async def type_text(
+ self,
+ page: Page,
+ locator: Locator | None,
+ text: str,
+ *,
+ timeout: float | None,
+ delay: float | None = None,
+ no_wait_after: bool | None = None,
+ allow_batched_playwright: bool = False,
+ ) -> None:
+ self._log.append(("input.type_text", (locator, text), {"delay": delay}))
+
+ async def clear_field(
+ self,
+ page: Page,
+ locator: Locator,
+ char_count: int,
+ *,
+ timeout: float | None,
+ force: bool | None = None,
+ no_wait_after: bool | None = None,
+ ) -> None:
+ self._log.append(("input.clear_field", (locator,), {"timeout": timeout}))
+
+
+class _RecordingScroll(ScrollEventStrategy):
+ def __init__(self, log: list[Call]) -> None:
+ self._log = log
+
+ async def scroll_to_element(self, page: Page, locator: Locator) -> None:
+ self._log.append(("scroll.scroll_to_element", (locator,), {}))
+
+ async def scroll_by(self, page: Page, delta_y: float) -> None:
+ self._log.append(("scroll.scroll_by", (delta_y,), {}))
+
+
+def page() -> _FakePage:
+ return _FakePage()
+
+
+def humanized_strategies(page: _FakePage) -> Iterator[list[Call]]:
+ """A humanized registration, as USE_EVENT_STRATEGIES makes it, logging into the page's own call list."""
+ EventStrategyFactory.set_cursor_strategy(_RecordingCursor(page.calls))
+ EventStrategyFactory.set_input_strategy(_RecordingInput(page.calls))
+ EventStrategyFactory.set_scroll_strategy(_RecordingScroll(page.calls))
+ EventStrategyFactory.record_profile("default")
+ try:
+ yield page.calls
+ finally:
+ EventStrategyFactory.reset()
+
+
+def _pin(arm: RunArm) -> Iterator[SkyvernContext]:
+ context = SkyvernContext()
+ context.run_arms = {HUMANIZED_INPUT_FLAG: ("wr_1", arm)}
+ skyvern_context.set(context)
+ try:
+ yield context
+ finally:
+ skyvern_context.reset()
+
+
+def control() -> Iterator[SkyvernContext]:
+ yield from _pin("control")
+
+
+def treatment() -> Iterator[SkyvernContext]:
+ yield from _pin("treatment")
+
+
+def _names(calls: list[Call]) -> list[str]:
+ return [name for name, _, _ in calls]
+
+
+ ("reader", "humanized"),
+ [
+ (AsyncMock(return_value="treatment"), True),
+ (AsyncMock(return_value="control"), False),
+ (AsyncMock(return_value=None), False),
+ (AsyncMock(return_value="true"), False),
+ (AsyncMock(side_effect=RuntimeError("flag service down")), False),
+ ],
+ ids=["treatment", "control", "no_variant", "unknown_variant", "flag_error"],
+)
+async def test_only_a_treatment_run_sends_its_clicks_through_the_registered_strategy(
+ monkeypatch: pytest.MonkeyPatch,
+ page: _FakePage,
+ humanized_strategies: list[Call],
+ reader: AsyncMock,
+ humanized: bool,
+) -> None:
+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)
+ context = SkyvernContext()
+ skyvern_context.set(context)
+ try:
+ await resolve_run_arm(context, HUMANIZED_INPUT_FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)
+ await input_dispatch.click(page, "#go", timeout=1234)
+ finally:
+ skyvern_context.reset()
+
+ assert _names(page.calls) == (["cursor.click"] if humanized else ["page.click"])
+
+
+async def test_input_outside_a_resolved_run_is_plain_playwright(
+ page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ skyvern_context.reset()
+
+ await input_dispatch.click(page, "#go", timeout=1234)
+ await input_dispatch.type_keys(page, "#q", "abc", delay=15, timeout=900)
+ await input_dispatch.wheel(page, 0, 800)
+
+ assert page.calls == [
+ ("page.click", ("#go",), {"timeout": 1234}),
+ ("page.type", ("#q", "abc"), {"delay": 15, "timeout": 900}),
+ ("mouse.wheel", (0, 800), {}),
+ ]
+
+
+async def test_control_sends_each_gesture_as_the_exact_playwright_call_it_replaced(
+ control: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ locator = page.locator("#seg")
+ await input_dispatch.approach(page, "#go")
+ await input_dispatch.click(page, "#go", timeout=5000, force=True)
+ await input_dispatch.click(page, locator, timeout=2000)
+ await input_dispatch.click_at(page, 10.0, 20.0)
+ await input_dispatch.hover(page, "#menu", timeout=2000)
+ await input_dispatch.focus(page, "#f", timeout=2000)
+ await input_dispatch.fill(page, "#f", "v", timeout=100)
+ await input_dispatch.clear(page, "#f", timeout=100)
+ await input_dispatch.type_keys(page, "#f", "ab", delay=15, timeout=900)
+ await input_dispatch.type_keys(page, "#f", "ab", timeout=900)
+ await input_dispatch.type_keys(page, None, "7", delay=40)
+ await input_dispatch.type_keys(page, locator, "7", delay=40)
+ await input_dispatch.press(page, "#f", "Enter")
+ await input_dispatch.press(page, None, "Escape")
+ await input_dispatch.press(page, locator, "Backspace")
+ await input_dispatch.select_option(page, "#s", value=["a"], timeout=100, force=False)
+ await input_dispatch.wheel(page, 0, -800)
+
+ assert page.calls == [
+ ("page.click", ("#go",), {"timeout": 5000, "force": True}),
+ ("locator(#seg).click", (), {"timeout": 2000}),
+ ("mouse.click", (10.0, 20.0), {}),
+ ("page.hover", ("#menu",), {"timeout": 2000}),
+ ("page.focus", ("#f",), {"timeout": 2000}),
+ ("page.fill", ("#f", "v"), {"timeout": 100}),
+ ("page.fill", ("#f", ""), {"timeout": 100}),
+ ("page.type", ("#f", "ab"), {"delay": 15, "timeout": 900}),
+ ("page.type", ("#f", "ab"), {"timeout": 900}),
+ ("keyboard.type", ("7",), {"delay": 40}),
+ ("locator(#seg).press_sequentially", ("7",), {"delay": 40}),
+ ("page.press", ("#f", "Enter"), {}),
+ ("keyboard.press", ("Escape",), {}),
+ ("locator(#seg).press", ("Backspace",), {}),
+ ("page.select_option", ("#s",), {"value": ["a"], "timeout": 100, "force": False}),
+ ("mouse.wheel", (0, -800), {}),
+ ]
+
+
+async def test_control_clicks_an_element_handle_with_only_the_arguments_given(control: SkyvernContext) -> None:
+ handle = MagicMock(spec=ElementHandle)
+ handle.click = AsyncMock()
+
+ await input_dispatch.click_handle(_FakePage(), handle)
+
+ handle.click.assert_awaited_once_with()
+
+
+async def test_treatment_routes_pointer_typing_and_scroll_through_the_registered_strategies(
+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ await input_dispatch.click(page, "#go", timeout=1234)
+ await input_dispatch.click(page, "#go", timeout=5000, force=True)
+ await input_dispatch.click_at(page, 10.0, 20.0)
+ await input_dispatch.hover(page, "#menu", timeout=2000)
+ await input_dispatch.clear(page, "#f", timeout=100)
+ await input_dispatch.wheel(page, 0, 800)
+
+ assert _names(page.calls) == [
+ "cursor.click",
+ # The strategy has no forced click: it moves there, then Playwright clicks exactly as before.
+ "cursor.move_to_element",
+ "page.click",
+ "cursor.move_to",
+ "mouse.click",
+ "cursor.move_to_element",
+ "page.hover",
+ "input.clear_field",
+ "scroll.scroll_by",
+ ]
+ assert page.calls[2] == ("page.click", ("#go",), {"timeout": 5000, "force": True})
+
+
+async def test_treatment_clicks_a_locator_through_the_strategy_and_a_handle_after_a_cursor_move(
+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ await input_dispatch.click(page, page.locator("#seg"), timeout=2000)
+ await input_dispatch.click_handle(page, _FakeHandle(page), position={"x": 1.0, "y": 2.0}, timeout=1500)
+
+ assert _names(page.calls) == ["cursor.click", "cursor.move_to", "handle.click"]
+ assert page.calls[1:] == [
+ ("cursor.move_to", (12.0, 23.0), {}),
+ ("handle.click", (), {"position": {"x": 1.0, "y": 2.0}, "timeout": 1500}),
+ ]
+
+
+async def test_treatment_types_through_the_keyboard_after_one_focus(
+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ await input_dispatch.type_keys(page, "#f", "abc", delay=15, timeout=900)
+
+ # The strategy gets no locator (given one, a humanized keyboard re-focuses the field before every key) and no
+ # delay: as in v1, the registered strategy owns the cadence.
+ assert page.calls == [
+ ("page.focus", ("#f",), {"timeout": 900}),
+ ("input.type_text", (None, "abc"), {"delay": None}),
+ ]
+
+
+ ("replace", "head_write"),
+ [
+ (True, ("page.fill", ("#f", "Hello, "), {"timeout": 900})),
+ (False, ("keyboard.insert_text", ("Hello, ",), {})),
+ ],
+ ids=["emptied_field_fills_the_head", "append_inserts_the_head_at_the_caret"],
+)
+async def test_treatment_humanizes_only_the_last_ten_characters_as_v1_does(
+ treatment: SkyvernContext,
+ page: _FakePage,
+ humanized_strategies: list[Call],
+ replace: bool,
+ head_write: Call,
+) -> None:
+ await input_dispatch.type_keys(page, "#f", "Hello, 0123456789", delay=15, timeout=900, replace=replace)
+
+ assert page.calls[-2:] == [head_write, ("input.type_text", (None, "0123456789"), {"delay": None})]
+
+
[email protected]("arm", ["control", "treatment"])
+async def test_a_secret_is_typed_key_by_key_exactly_as_before_in_both_arms(
+ page: _FakePage, humanized_strategies: list[Call], arm: RunArm
+) -> None:
+ # Split one-time-code boxes advance on each keystroke, so neither a fill nor a humanized keyboard may take over.
+ for _ in _pin(arm):
+ await input_dispatch.type_keys(page, "#code", "s3cret", delay=15, timeout=900, secret=True, replace=True)
+ await input_dispatch.type_keys(page, page.locator("#box"), "7", timeout=1000, secret=True)
+
+ assert page.calls == [
+ ("page.type", ("#code", "s3cret"), {"delay": 15, "timeout": 900}),
+ ("locator(#box).press_sequentially", ("7",), {"timeout": 1000}),
+ ]
+
+
+async def test_treatment_approach_moves_then_waits_for_the_page_to_settle(
+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]
+) -> None:
+ await input_dispatch.approach(page, "#go")
+
+ assert _names(page.calls) == ["locator(#go).scroll_into_view_if_needed", "cursor.move_to_element", "page.evaluate"]
+
+
+def test_arm_fields_name_the_arm_whether_it_is_in_effect_and_the_registered_profile(
+ treatment: SkyvernContext, humanized_strategies: list[Call]
+) -> None:
+ assert input_dispatch.arm_fields() == {
+ "humanized_input_arm": "treatment",
+ "humanized_input_in_effect": True,
+ "global_event_strategy_profile": "default",
+ }
+
+
+def test_registered_profile_tells_no_registration_from_an_unnamed_one() -> None:
+ EventStrategyFactory.reset()
+ try:
+ assert EventStrategyFactory.registered_profile() == "none"
+ EventStrategyFactory.set_cursor_strategy(_RecordingCursor([]))
+ assert EventStrategyFactory.registered_profile() == "unrecorded"
+ EventStrategyFactory.record_profile("kernel")
+ assert EventStrategyFactory.registered_profile() == "kernel"
+ finally:
+ EventStrategyFactory.reset()
+ assert EventStrategyFactory.registered_profile() == "none"
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index ac7f2995b..253569be1 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -6,9 +6,11 @@ operations (no task-ecosystem) with the right args, without a live browser.
from __future__ import annotations
+import ast
import asyncio
import contextlib
import html
+import inspect
import itertools
import json
import os
@@ -42,6 +44,8 @@ from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODE
from skyvern.forge import app
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.event.default import DefaultInputStrategy
+from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.services import credentials as credentials_module
from skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager, WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import _recorded_task_nav_error_codes
@@ -68,6 +72,7 @@ from skyvern.forge.taskv3.tools import (
_MENU_OPTION_TEXTS_JS,
_OPAQUE_ID_RUN_RE,
_SEMANTIC_COMMIT_STATE_JS,
+ _TYPE_TARGET_PROBE_JS,
NAVIGATION_DEAD_END_STATUSES,
OBSERVE_DISPLAY_WIDTHS,
OBSERVE_RETAIN_WIDTH_MIN,
@@ -20809,6 +20814,112 @@ async def test_covered_error_message_when_the_occluding_layer_has_no_controls_at
assert "no controls were found on it" in r.content, r.content
+class _CoveredProbeFakePage(_TypeaheadFakePage):
+ """Answers the type-target probe with a fixed covered reading, so the message the typing path
+ builds from it is checked without a browser."""
+
+ def __init__(self, occluder: dict[str, Any]) -> None:
+ super().__init__(field_type="email")
+ self._occluder = occluder
+
+ async def evaluate(self, js: str, arg: Any = None) -> Any:
+ if js == _TYPE_TARGET_PROBE_JS:
+ return {"exists": True, "occluded": True, "occluder": self._occluder}
+ return await super().evaluate(js, arg)
+
+
+# The probe sets `ownContainer` when the named layer is an ancestor of the field and the element taking
+# the pointer is not itself interactive: a sticky header holding the input, with a transparent
+# click-catcher span over it that the probe stamps `data-tv3-catcher`.
+_OWN_CONTAINER_OCCLUDER = {
+ "selector": "#header",
+ "name": "People Phone Search",
+ "controls": [{"selector": "#tab-people", "label": "People"}],
+ "truncated": False,
+ "layerKind": "qualified",
+ "ownContainer": True,
+}
+
+
+async def test_a_cover_inside_the_fields_own_container_names_the_stamped_catcher_to_click() -> None:
+ page = _CoveredProbeFakePage(_OWN_CONTAINER_OCCLUDER)
+ taskv3_loop._COVERED_LAYER.set(None)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(
+ {"selector": "#name", "text": "Jane Doe"}
+ )
+ assert r.error_class == "covered", r.content
+ assert 'Click [data-tv3-catcher="1"] to activate the field, then retry #name.' in r.content, r.content
+ assert "Do not try to dismiss it" in r.content, r.content
+ assert "closes or dismisses" not in r.content, r.content
+ assert not any(call[0] in ("fill", "type") for call in page.calls), page.calls
+ recorded = taskv3_loop._COVERED_LAYER.get() or {}
+ assert recorded == {"branch": "own_container", "controls": 0, "layer_kind": "qualified"}, recorded
+
+
+async def test_a_cover_without_the_own_container_flag_keeps_the_dismissal_message() -> None:
+ occluder = {**_OWN_CONTAINER_OCCLUDER, "selector": "#banner", "name": "Accept cookies"}
+ del occluder["ownContainer"]
+ page = _CoveredProbeFakePage(occluder)
+ taskv3_loop._COVERED_LAYER.set(None)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(
+ {"selector": "#name", "text": "Jane Doe"}
+ )
+ assert r.content == (
+ '#name is covered by "Accept cookies" (#banner), so it cannot be typed into — a person could not '
+ 'click it either. Its controls: #tab-people "People". Pick whichever one actually closes or '
+ "dismisses the layer, then retry #name."
+ ), r.content
+ recorded = taskv3_loop._COVERED_LAYER.get() or {}
+ assert recorded == {"branch": "named", "controls": 1, "layer_kind": "qualified"}, recorded
+
+
+async def test_a_transparent_own_container_still_names_the_catcher_rather_than_a_ghost_cover() -> None:
+ page = _CoveredProbeFakePage({**_OWN_CONTAINER_OCCLUDER, "controls": [], "invisible": True})
+ taskv3_loop._COVERED_LAYER.set(None)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(
+ {"selector": "#name", "text": "Jane Doe"}
+ )
+ assert 'Click [data-tv3-catcher="1"] to activate the field, then retry #name.' in r.content, r.content
+ assert (taskv3_loop._COVERED_LAYER.get() or {}).get("branch") == "own_container"
+
+
+_STICKY_FORM_HTML = """
+<div id="header" style="position:sticky;top:0;width:400px;height:140px;background:#eee">
+ <form><input id="name" type="text" style="display:block;margin:4px;width:200px">{over}</form>
+</div>
+"""
+_SIBLING_BANNER_HTML = """
+<input id="name" type="text" style="position:absolute;top:100px;left:100px;width:150px;height:30px">
+<div id="banner" style="position:fixed;top:90px;left:90px;width:300px;height:50px;z-index:10;background:#f88">
+ Accept cookies <button id="accept">Accept</button>
+</div>
+"""
+
+
+@_skip_no_browser
+async def test_the_real_probe_names_a_non_interactive_catcher_but_not_a_submit_button_or_a_banner() -> None:
+ catcher = '<span id="catcher" style="position:absolute;inset:0;z-index:1"></span>'
+ submit = '<button id="go" type="submit" style="position:absolute;left:0;top:0;width:220px;height:40px">Go</button>'
+ async with _content_page(_STICKY_FORM_HTML.format(over=catcher)) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(
+ {"selector": "#name", "text": "Jane Doe"}
+ )
+ assert 'Click [data-tv3-catcher="1"] to activate the field' in r.content, r.content
+ assert await page.locator('[data-tv3-catcher="1"]').evaluate("n => n.id") == "catcher"
+ for markup in (_STICKY_FORM_HTML.format(over=submit), _SIBLING_BANNER_HTML):
+ async with _content_page(markup) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(
+ {"selector": "#name", "text": "Jane Doe"}
+ )
+ assert r.error_class == "covered", r.content
+ assert "Pick whichever one actually closes or dismisses the layer" in r.content, r.content
+ assert await page.locator("[data-tv3-catcher]").count() == 0
+
+
# A cover that qualifies as NOTHING: not pinned, no layer role, no aria-modal, not view-sized, and
# not an ancestor of the field. The walk finds no layer and names the hit element itself, which is
# the production shape behind most zero-control refusals -- an option row or a value cell, which has
@@ -35362,9 +35473,35 @@ def _straddle_renders() -> list[tuple[str, int, Callable[[str], str]]]:
[{"n": 1, "text": "Jane Doe", "vals": [f"data-code={t}"]}, {"n": 2, "text": "Jane Doe"}]
),
),
+ (
+ "unproven-row",
+ 60,
+ lambda t: taskv3_tools._unproven_row_error(
+ "John", "#who", "scrolled to the end", {"n": 1, "text": t}
+ ).content,
+ ),
+ (
+ "selection-report",
+ taskv3_tools.SELECTION_REPORT_OPTION_WIDTH,
+ lambda t: taskv3_tools._selection_report([t, "B"]),
+ ),
]
+def test_every_row_taking_error_constructor_is_in_the_straddle_table() -> None:
+ # A new refusal that renders page rows must join the table above, or its mask is never checked.
+ tree = ast.parse(Path(taskv3_tools.__file__).read_text())
+ constructors = {
+ f.name
+ for f in tree.body
+ if isinstance(f, ast.FunctionDef)
+ and f.name.endswith("_error")
+ and {"row", "rows"} & {a.arg for a in f.args.args + f.args.kwonlyargs}
+ }
+ table = inspect.getsource(_straddle_renders)
+ assert constructors and all(name in table for name in constructors), sorted(constructors)
+
+
@pytest.mark.parametrize(("cap", "render"), [pytest.param(c, r, id=i) for i, c, r in _straddle_renders()])
def test_a_row_listing_masks_a_hidden_value_before_any_cut_so_no_fragment_of_it_survives(
cap: int, render: Callable[[str], str]
@@ -35461,6 +35598,38 @@ async def test_dom_a_typeahead_suggestion_masks_a_hidden_value_straddling_its_di
skyvern_context.reset()
+@_skip_no_browser
[email protected]("shape", ["twin_far_down_the_list", "list_search_box"])
+async def test_dom_a_select_combobox_refusal_masks_a_hidden_value_straddling_its_row_cap(shape: str) -> None:
+ # The refusals select_combobox builds inline name the rows that made the value ambiguous, each cut to 60.
+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"
+ fragments = {secret[:k] for k in range(4, len(secret) + 1)}
+ ctx = SkyvernContext(organization_id="o_1")
+ ctx.register_secret_value(secret, hide_from_model=True)
+ skyvern_context.set(ctx)
+ try:
+ for start in (60 - len(secret) + 4, 60 - len(secret) // 2, 60 - 4):
+ label = "North " + "." * (start - len("North ")) + secret
+ rows = [(f"Row {i:03d}", f"+{3000 + i}") for i in range(1, 71)]
+ if shape == "twin_far_down_the_list":
+ rows[5], rows[60] = (label, "+9001"), (label, "+9002")
+ html = _cc_widget_html(rows, 30)
+ else:
+ rows[10], rows[50] = (label, "+9010"), ("North Dakota", "+9050")
+ html = _cc_widget_html([*rows, ("", "")], 12, search_box=True)
+ async with _content_page(html) as page:
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(
+ {"selector": "#cc", "value": "North"}
+ )
+ shown = ctx.hide_from_model(r.content)
+ assert r.status == "error", (start, shown)
+ leaked = sorted(f for f in fragments if f in shown)
+ assert not leaked, (start, leaked, shown)
+ finally:
+ skyvern_context.reset()
+
+
@pytest.mark.asyncio
async def test_get_html_offset_normalization_survives_the_selector_guards_arg_copy() -> None:
# `_with_selector_guard` hands the handler a COPY of args whenever a selector is present, and the
@@ -37681,3 +37850,26 @@ async def test_the_failed_read_count_resets_on_a_success_and_on_navigation(monke
await page.goto("http://shop.test/review?again=1")
after = await act(1)
assert after[4] is None and after[1] == 0, after
+
+
+@_skip_no_browser
+async def test_an_appended_credential_never_reaches_the_humanized_keyboard(monkeypatch: pytest.MonkeyPatch) -> None:
+ class _RefusingInput(DefaultInputStrategy):
+ async def type_text(self, *args: Any, **kwargs: Any) -> None:
+ raise AssertionError("a resolved credential was sent through the registered input strategy")
+
+ monkeypatch.setattr(settings, "TASK_V3_HUMANIZED_INPUT", True)
+ EventStrategyFactory.set_input_strategy(_RefusingInput())
+ try:
+ async with _content_page('<!doctype html><html><body><input id="t" value="id-"></body></html>') as page:
+ tools = build_browser_tools(
+ _fixed_page_provider(page),
+ resolve_typed_text=lambda text: "real-secret" if text == "placeholder_abc" else text,
+ )
+ r = await _tool(tools, "type").handler({"selector": "#t", "text": "placeholder_abc", "clear": False})
+ held = await page.evaluate("() => document.getElementById('t').value")
+ finally:
+ EventStrategyFactory.reset()
+ assert r.status == "ok", r.content
+ assert held == "id-real-secret"