Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop telling Task V3 to dismiss a covering layer that contains the field itself (#8722)
pedrohsdb20dba4cc188e
diff --git a/skyvern/config.py b/skyvern/config.pyindex 704ecac40..da975bd3e 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -547,6 +547,10 @@ class Settings(BaseSettings):# year-only segment groups to the segment path (SKY-17013). Force-on term only: runs are randomized per# run by the flag of the same name, read through run_arm_enabled(DATE_SEGMENT_AIM_FLAG, ...).TASK_V3_DATE_SEGMENT_AIM: bool = False+ # Send Task V3's clicks, typing and scrolls through EventStrategyFactory, as v1 does, so a run follows+ # whatever USE_EVENT_STRATEGIES registered. Force-on term only: runs are randomized per run by+ # the flag of the same name, read through run_arm_enabled(HUMANIZED_INPUT_FLAG, ...). Off: plain Playwright.+ TASK_V3_HUMANIZED_INPUT: bool = False# Render the previous block's outcome (status / finish reason / final URL) and whether this is the# last block into a v3 block's goal. Costs prompt tokens on every turn of the block, so it is# measured via taskv3_block_context_tokens before it earns default-on. The outcome itself isdiff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex 31861c2e2..edce7d5fc 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -185,6 +185,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import CredentialReleafrom skyvern.forge.sdk.workflow.models.parameter import WorkflowParameter, WorkflowParameterTypefrom skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatusfrom skyvern.forge.sdk.workflow.page_derived_templates import NO_RENDER_RECORD, UNVERIFIED_ROOT_CLASSES+from skyvern.forge.taskv3 import input_dispatchfrom skyvern.forge.taskv3.goal_check import (GOAL_CHECK_PROMPT_NAME,PRE_JUDGE_SKIP_REASONS,@@ -199,6 +200,7 @@ from skyvern.forge.taskv3.run_arms import (EXTRACTION_REPORTS_FLAG,GOAL_CHECK_ENFORCE_FLAG,GOAL_CHECK_FLAG,+ HUMANIZED_INPUT_FLAG,REQUIRED_FIELD_ANSWERS_FLAG,TYPE_COORDINATE_CLICK_FLAG,resolve_run_arm,@@ -2372,6 +2374,15 @@ class ForgeAgent:organization_id=task.organization_id,forced=settings.TASK_V3_EXTRACTION_REPORTS,)+ await resolve_run_arm(+ context,+ HUMANIZED_INPUT_FLAG,+ distinct_id=task.workflow_run_id or task.task_id,+ organization_id=task.organization_id,+ forced=settings.TASK_V3_HUMANIZED_INPUT,+ )+ # Here as well as on the loop's finish line, so a run whose loop raises still records its profile.+ LOG.info("Task V3 humanized input resolved", **input_dispatch.arm_fields())# The judge's finish-time screenshot, reused as the decision screenshot of an accepted completion.goal_judge_shot: list[bytes] = []page_free_validation = bool(diff --git a/skyvern/forge/sdk/event/factory.py b/skyvern/forge/sdk/event/factory.pyindex 96e8f9045..5c794d1e2 100644--- a/skyvern/forge/sdk/event/factory.py+++ b/skyvern/forge/sdk/event/factory.py@@ -39,6 +39,7 @@ class EventStrategyFactory:__cursor_vis_cache: CursorEventStrategy | None = None__input: InputEventStrategy | None = None__scroll: ScrollEventStrategy | None = None+ __profile: str | None = None__metrics: _EventMetrics = _EventMetrics()# -- setters ----------------------------------------------------------------@@ -62,8 +63,22 @@ class EventStrategyFactory:EventStrategyFactory.__cursor_vis_cache = NoneEventStrategyFactory.__input = NoneEventStrategyFactory.__scroll = None+ EventStrategyFactory.__profile = NoneEventStrategyFactory.__metrics = _EventMetrics()+ @staticmethod+ def record_profile(profile: str) -> None:+ """Name the profile whoever registered the current strategies registered them under."""+ EventStrategyFactory.__profile = profile++ @staticmethod+ def registered_profile() -> str:+ """The recorded profile; "none" when nothing is registered, "unrecorded" when strategies were set unnamed."""+ if EventStrategyFactory.__profile is not None:+ return EventStrategyFactory.__profile+ strategies = (EventStrategyFactory.__cursor, EventStrategyFactory.__input, EventStrategyFactory.__scroll)+ return "none" if all(strategy is None for strategy in strategies) else "unrecorded"+# -- getters (always return a non-None strategy) ----------------------------@staticmethoddiff --git a/skyvern/forge/taskv3/captcha_tools.py b/skyvern/forge/taskv3/captcha_tools.pyindex 0a7ebdfc3..8f3218488 100644--- a/skyvern/forge/taskv3/captcha_tools.py+++ b/skyvern/forge/taskv3/captcha_tools.py@@ -20,6 +20,7 @@ on the tool-call record; ``tool_status`` alone cannot tell a working detector frfrom __future__ import annotationsimport asyncio+import functoolsimport refrom typing import Anyfrom urllib.parse import urlsplit@@ -29,6 +30,7 @@ import structlogfrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.tasks import Task+from skyvern.forge.taskv3 import input_dispatchfrom skyvern.forge.taskv3.loop import REF_SELECTOR_RE, ToolResult, ToolSpecfrom skyvern.forge.taskv3.tools import PageProvider, _invalid_selector_result, _normalize_selectorfrom skyvern.webeye.utils.captcha_solver import (@@ -301,6 +303,8 @@ def build_captcha_tools(workflow_run_id=task.workflow_run_id,browser_session_id=task.browser_session_id,probe_child_frames=True,+ click=functools.partial(input_dispatch.click, page),+ click_handle=functools.partial(input_dispatch.click_handle, page),)except CaptchaChallengeUnsolvedError:failed_attempts += 1diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.pyindex efb2353f8..efdedc2bd 100644--- a/skyvern/forge/taskv3/engine.py+++ b/skyvern/forge/taskv3/engine.py@@ -35,6 +35,7 @@ from skyvern.forge.sdk.api.llm.api_handler_factory import VISION_FALLBACK_PROMPTfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorRetryableTaskfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.workflow.models.credential_release import CredentialReleaseGuard+from skyvern.forge.taskv3 import input_dispatchfrom skyvern.forge.taskv3.code_surface import apply_surface, configured_surfacefrom skyvern.forge.taskv3.goal_check import (GOAL_CHECK_TIMEOUT_SECONDS,@@ -715,6 +716,7 @@ async def run_task_v3_agent_loop(# The run's model, so exposure rates on this line split per model like the re-ask line's.llm_key=llm_caller.llm_key,unlisted_reask=outcome.unlisted_reask,+ **input_dispatch.arm_fields(),# The loop's progress signals ride here rather than on records of their own: this line# already fires exactly once per run and already carries block_type, so collapsing removes a# per-run indexed event and makes the join to block_type free instead of a second lookup.diff --git a/skyvern/forge/taskv3/input_dispatch.py b/skyvern/forge/taskv3/input_dispatch.pynew file mode 100644index 000000000..663ddf3ec--- /dev/null+++ b/skyvern/forge/taskv3/input_dispatch.py@@ -0,0 +1,281 @@+"""The one place Task V3 sends pointer and keyboard input to a page.++Control runs the Playwright call each tool made before this module existed, argument for argument. Treatment+sends a gesture through EventStrategyFactory wherever v1 does and calls Playwright directly wherever v1 does, so+the run follows whatever USE_EVENT_STRATEGIES registered for the process, like a v1 block of the same run.+"""++from __future__ import annotations++import asyncio+from typing import Any, TypeAlias, cast++import structlog+from playwright.async_api import ElementHandle, FileChooser, Frame, Locator, Page+from playwright.async_api import TimeoutError as PlaywrightTimeoutError++from skyvern.config import settings+from skyvern.constants import TEXT_PRESS_MAX_LENGTH+from skyvern.forge.sdk.event.factory import EventStrategyFactory+from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, current_run_arm, run_arm_enabled+from skyvern.webeye.browser_object_predicates import is_page_like++LOG = structlog.get_logger()++Realm: TypeAlias = Page | Frame+Target: TypeAlias = str | Locator | ElementHandle++# After the cursor reaches a target, wait until the page has painted twice and then gone this long without a DOM+# mutation, so a hover-intent menu crossed on the way has finished opening or closing before a caller takes its+# pre-action baseline. Capped: a page that never stops mutating (a ticker, a spinner) costs the cap, not a stall.+_SETTLE_QUIET_MS = 150+_SETTLE_CAP_MS = 1000+_SETTLE_JS = r"""(arg) => new Promise((resolve) => {+ const started = performance.now();+ let last = started;+ const obs = new MutationObserver(() => { last = performance.now(); });+ const tick = () => {+ const now = performance.now();+ if (now - last >= arg.quietMs || now - started >= arg.capMs) { obs.disconnect(); resolve(true); return; }+ setTimeout(tick, 25);+ };+ requestAnimationFrame(() => requestAnimationFrame(() => {+ obs.observe(document.documentElement, { subtree: true, childList: true, attributes: true, characterData: true });+ last = performance.now();+ setTimeout(tick, 25);+ }));+ setTimeout(() => { obs.disconnect(); resolve(false); }, arg.capMs);+})"""+++def arm_fields() -> dict[str, Any]:+ return {+ "humanized_input_arm": current_run_arm(HUMANIZED_INPUT_FLAG),+ "humanized_input_in_effect": humanized(),+ "global_event_strategy_profile": EventStrategyFactory.registered_profile(),+ }+++def humanized() -> bool:+ return run_arm_enabled(HUMANIZED_INPUT_FLAG, settings.TASK_V3_HUMANIZED_INPUT)+++def _opts(**options: Any) -> dict[str, Any]:+ # Only what the caller passed, so a control call carries exactly the keywords the tool always sent.+ return {key: value for key, value in options.items() if value is not None}+++def _page_of(realm: Realm) -> Page:+ # A Frame has no mouse or keyboard; the factory always drives the tab that owns it.+ return cast(Page, realm) if is_page_like(realm) else cast(Frame, realm).page+++def _locator(realm: Realm, target: str | Locator) -> Locator:+ # A page-level selector method acts on the first match; a locator is strict unless told otherwise.+ return realm.locator(target).first if isinstance(target, str) else target+++async def _move_to_element(page: Page, element: ElementHandle) -> None:+ try:+ box = await element.bounding_box()+ if box is None:+ return+ await EventStrategyFactory.move_cursor(page, box["x"] + box["width"] / 2, box["y"] + box["height"] / 2)+ except Exception:+ LOG.debug("taskv3 cursor move to a handed-in element failed, clicking anyway", exc_info=True)+++async def approach(realm: Realm, target: str | Locator) -> None:+ """Treatment: bring the cursor to ``target`` and let the page settle, so travel effects predate a baseline."""+ if not humanized():+ return+ locator = _locator(realm, target)+ try:+ # The cursor is clamped to the viewport, so an off-screen target would otherwise be scrolled to, and+ # travelled to again, only after the baseline.+ await locator.scroll_into_view_if_needed(timeout=2000)+ except Exception:+ LOG.debug("taskv3 scroll before cursor approach failed", exc_info=True)+ await EventStrategyFactory.move_to_element(locator.page, locator)+ try:+ async with asyncio.timeout((_SETTLE_CAP_MS + 500) / 1000):+ await realm.evaluate(_SETTLE_JS, {"quietMs": _SETTLE_QUIET_MS, "capMs": _SETTLE_CAP_MS})+ except Exception:+ LOG.debug("taskv3 settle after cursor approach did not complete", exc_info=True)+++async def click(+ realm: Realm,+ target: str | Locator,+ *,+ timeout: float | None = None,+ force: bool | None = None,+ position: dict[str, float] | None = None,+) -> None:+ options = _opts(timeout=timeout, force=force, position=position)+ if not humanized():+ if isinstance(target, str):+ await realm.click(target, **options)+ else:+ await target.click(**options)+ return+ locator = _locator(realm, target)+ if force is not None or position is not None:+ await EventStrategyFactory.move_to_element(locator.page, locator)+ if isinstance(target, str):+ await realm.click(target, **options)+ else:+ await target.click(**options)+ return+ await EventStrategyFactory.click_element(locator.page, locator, timeout=timeout)+++async def click_handle(+ realm: Realm,+ handle: ElementHandle,+ *,+ timeout: float | None = None,+ force: bool | None = None,+ position: dict[str, float] | None = None,+) -> None:+ if humanized():+ # v1's pattern for a click the cursor strategy cannot express: move there, then click directly.+ await _move_to_element(_page_of(realm), handle)+ await handle.click(**_opts(timeout=timeout, force=force, position=position))+++async def click_at(page: Page, x: float, y: float) -> None:+ if humanized():+ await EventStrategyFactory.move_cursor(page, x, y)+ await page.mouse.click(x, y)+++async def hover(realm: Realm, selector: str, *, timeout: float | None = None) -> None:+ if humanized():+ locator = _locator(realm, selector)+ await EventStrategyFactory.move_to_element(locator.page, locator)+ await realm.hover(selector, **_opts(timeout=timeout))+++async def js_click(realm: Realm, root_query_js: str, arg: Any) -> Any:+ """Click a hidden native control from inside the page. No pointer events, in either arm, as in v1."""+ return await realm.evaluate(+ "(arg) => { const _q = "+ + root_query_js+ + "; const el = _q.find(arg.sel) || arg.el;"+ + " if (!el || !el.isConnected) return false; el.click(); return true; }",+ arg,+ )+++async def focus(realm: Realm, target: str | Locator, *, timeout: float | None = None) -> None:+ if isinstance(target, str):+ await realm.focus(target, **_opts(timeout=timeout))+ else:+ await target.focus(**_opts(timeout=timeout))+++async def fill(realm: Realm, target: Target, value: str, *, timeout: float | None = None) -> None:+ """An atomic write, in both arms: v1 fills plain fields and secrets in one call and never humanizes them."""+ if isinstance(target, str):+ await realm.fill(target, value, **_opts(timeout=timeout))+ else:+ await target.fill(value, **_opts(timeout=timeout))+++async def clear(realm: Realm, selector: str, *, timeout: float | None = None) -> None:+ if not humanized():+ await realm.fill(selector, "", **_opts(timeout=timeout))+ return+ locator = _locator(realm, selector)+ await EventStrategyFactory.clear_field(locator.page, locator, 0, timeout=timeout)+++async def type_keys(+ realm: Realm,+ target: str | Locator | None,+ text: str,+ *,+ delay: float | None = None,+ timeout: float | None = None,+ secret: bool = False,+ replace: bool = False,+) -> None:+ """Key events for ``text``. ``target`` None types into whatever holds focus, with no focus of its own;+ ``realm`` is then the page.++ A secret is typed key by key as before in both arms: a humanized keyboard mistypes and backspaces, and a+ fill would not advance split code boxes. ``replace`` says the caller has just emptied the field, so+ treatment may write the untyped part with fill."""+ options = _opts(delay=delay, timeout=timeout)+ if secret or not humanized():+ if target is None:+ await cast(Page, realm).keyboard.type(text, **options)+ elif isinstance(target, str):+ await realm.type(target, text, **options)+ else:+ await target.press_sequentially(text, **options)+ return+ # As v1 types: all but the last few characters in one write, the tail through the registered strategy,+ # which owns the cadence. An emptied field takes the head as a fill; one being appended to takes it as+ # inserted text at the caret, since a fill would replace what the field already holds.+ cut = max(len(text) - TEXT_PRESS_MAX_LENGTH, 0)+ head, tail = text[:cut], text[cut:]+ page = cast(Page, realm) if target is None else _page_of(realm)+ seconds = None if not timeout else timeout / 1000+ try:+ async with asyncio.timeout(seconds):+ if head and replace and target is not None:+ await fill(realm, target, head, timeout=timeout)+ elif target is not None:+ # Focused once: a humanized keyboard given a locator re-focuses the field before every key,+ # which puts the caret back at the start of a select-on-focus field.+ await focus(realm, target, timeout=timeout)+ if head and not (replace and target is not None):+ await page.keyboard.insert_text(head)+ await EventStrategyFactory.type_text(page, None, tail, timeout=timeout)+ except TimeoutError as exc:+ raise PlaywrightTimeoutError(f"Timeout {timeout}ms exceeded.") from exc+++async def press(+ realm: Realm, target: str | Locator | ElementHandle | None, key: str, *, timeout: float | None = None+) -> None:+ """A key press, raw in both arms as v1's keypress is. ``target`` None presses on ``realm``, the page's, keyboard."""+ options = _opts(timeout=timeout)+ if target is None:+ await cast(Page, realm).keyboard.press(key, **options)+ elif isinstance(target, str):+ await realm.press(target, key, **options)+ else:+ await target.press(key, **options)+++async def select_option(+ realm: Realm,+ selector: str,+ *,+ label: str | list[str] | None = None,+ value: str | list[str] | None = None,+ timeout: float | None = None,+ force: bool | None = None,+) -> None:+ if label is not None:+ await realm.select_option(selector, label=label, **_opts(timeout=timeout, force=force))+ else:+ await realm.select_option(selector, value=value, **_opts(timeout=timeout, force=force))+++async def set_files(chooser: FileChooser, paths: list[str]) -> None:+ await chooser.set_files(paths)+++async def set_input_files(target: Locator | ElementHandle, paths: list[str]) -> None:+ await target.set_input_files(paths)+++async def wheel(page: Page, delta_x: float, delta_y: float) -> None:+ if humanized():+ await EventStrategyFactory.scroll_by(page, delta_x, delta_y)+ else:+ await page.mouse.wheel(delta_x, delta_y)diff --git a/skyvern/forge/taskv3/loop.py b/skyvern/forge/taskv3/loop.pyindex f2f0200ae..261745d25 100644--- a/skyvern/forge/taskv3/loop.py+++ b/skyvern/forge/taskv3/loop.py@@ -164,6 +164,9 @@ CoveredBranch = Literal["unnamed",# The layer intercepts the pointer but paints nothing, so it is absent from the screenshot."invisible",+ # The layer is the field's own container and a non-interactive element in it takes the pointer;+ # the message names that stamped element to click, so no controls are listed.+ "own_container",]# WHICH ELEMENT the probe named as the layer, which the branch cannot recover. `named` with zerodiff --git a/skyvern/forge/taskv3/run_arms.py b/skyvern/forge/taskv3/run_arms.pyindex 206e7984d..cbefa1196 100644--- a/skyvern/forge/taskv3/run_arms.py+++ b/skyvern/forge/taskv3/run_arms.py@@ -23,6 +23,7 @@ CUSTOMER_PRECEDENCE_FLAG = "TASK_V3_CUSTOMER_PRECEDENCE"EXTRACTION_REPORTS_FLAG = "TASK_V3_EXTRACTION_REPORTS"GOAL_CHECK_FLAG = "TASK_V3_GOAL_CHECK"GOAL_CHECK_ENFORCE_FLAG = "TASK_V3_GOAL_CHECK_ENFORCE"+HUMANIZED_INPUT_FLAG = "TASK_V3_HUMANIZED_INPUT"# Person properties a flag is evaluated with beyond organization_id; resolve_run_arm drops any other key. The# PostHog preflight (scripts/check_run_arm_flags.py) reads this mapping to accept release conditions on them.@@ -101,3 +102,12 @@ def run_arm_enabled(flag: str, forced: bool) -> bool:return Falsepin = context.run_arms.get(flag)return pin is not None and pin[1] == "treatment"+++def current_run_arm(flag: str) -> RunArm | None:+ """The arm pinned for ``flag`` on the current run, or None when it was never resolved."""+ context = skyvern_context.current()+ if context is None:+ return None+ pin = context.run_arms.get(flag)+ return pin[1] if pin is not None else Nonediff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex c80a13327..7acdb8cab 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -55,6 +55,7 @@ from skyvern.forge.sdk.workflow.models.credential_release import (CredentialReleaseGuard,release_target_url,)+from skyvern.forge.taskv3 import input_dispatchfrom skyvern.forge.taskv3.loop import (ACTION_OUTCOME_DATA_KEY,FILL_TOOLS,@@ -543,8 +544,8 @@ SELECTION_REPORT_OPTION_WIDTH = 80def _selection_report(options: list[str]) -> str:shown = [- o[:SELECTION_REPORT_OPTION_WIDTH] + "…" if len(o) > SELECTION_REPORT_OPTION_WIDTH else o- for o in options[:SELECTION_REPORT_MAX_OPTIONS]+ m[:SELECTION_REPORT_OPTION_WIDTH] + "…" if len(m) > SELECTION_REPORT_OPTION_WIDTH else m+ for m in (_model_text(o, SELECTION_REPORT_OPTION_WIDTH + 1) for o in options[:SELECTION_REPORT_MAX_OPTIONS])]if len(shown) < len(options):return f"{shown!r} (showing {len(shown)} of {len(options)})"@@ -924,7 +925,7 @@ def _commit_gate(def _unproven_row_error(value: str, selector: str, seen: str, row: dict[str, Any]) -> ToolResult:- label = str(row.get("text") or "")[:60]+ label = _model_text(row.get("text") or "", 60)return ToolResult.error(f"{value!r} matched no option in {selector}'s list — {seen}, so select_combobox cannot prove {label!r} is "f'the only match; the field is NOT filled — if the visible row [data-tv3-menu="{row.get("n")}"] '@@ -4188,6 +4189,7 @@ _TYPE_TARGET_PROBE_JS = (+ _NATIVE_LABEL_JS+ r"""try { _q.all('[data-tv3-cover]').forEach((n) => n.removeAttribute('data-tv3-cover')); } catch (e) { /* best-effort */ }+ try { _q.all('[data-tv3-catcher]').forEach((n) => n.removeAttribute('data-tv3-catcher')); } catch (e) { /* best-effort */ }// A host-anchored selector's two halves straddle a shadow boundary, so no single root can match it// and a per-root lookup finds nothing -- which would read as "no field here" and skip the check on@@ -5175,6 +5177,15 @@ _TYPE_TARGET_PROBE_JS = (try { layer.setAttribute('data-tv3-cover', '1'); } catch (e) { /* best-effort */ }out.occluder = { selector: layerSelector, name: layerName, controls, truncated };out.occluder.layerKind = layerKind;+ // The walk can stop on a pinned ancestor of the field itself: a sticky header whose collapsed+ // search form lays a transparent click-catcher over its own inputs. There is nothing to dismiss.+ // An interactive hit (the form's own submit button) is not a catcher: clicking it could submit.+ const activates = """+ + json.dumps(_INTERACTIVE_HIT_SEL)+ + r""";+ if (related(layer, el) && ![hit, hitResult.top].some((n) => n && n.closest && n.closest(activates))) {+ try { hit.setAttribute('data-tv3-catcher', '1'); out.occluder.ownContainer = true; } catch (e) { /* best-effort */ }+ }// Whether a PERSON would see this layer at all. A leftover consent backdrop still intercepts the// pointer (elementFromPoint returned it) but can paint nothing -- fully transparent, no visible// control, heading or text -- so the field looks clear on screen and "dismiss the overlay you@@ -10285,7 +10296,7 @@ async def _click_stamped_row(page: Any, stamp: str, want: str, timeout: int) ->try:if not await handle.evaluate(_STAMPED_ROW_GUARD_JS, want):return False- await handle.click(timeout=timeout)+ await input_dispatch.click_handle(page, handle, timeout=timeout)finally:try:await handle.dispose()@@ -12193,6 +12204,10 @@ def build_browser_tools(# backdrop, and telling the model to press Escape on it abandons the verification.if str(occluder.get("challengeFrame") or "").strip():return "challenge"+ # Ahead of `invisible`: a transparent pinned ancestor paints nothing but the field it holds, yet its+ # catcher is still the thing to click.+ if occluder.get("ownContainer"):+ return "own_container"if occluder.get("invisible"):return "invisible"return "named"@@ -12222,7 +12237,7 @@ def build_browser_tools(# recorded -- it is page text, and these names carry personal data.parts = _named_controls(occluder)branch = _covered_branch(occluder)- _record_covered(occluder, branch, controls=parts)+ _record_covered(occluder, branch, controls=[] if branch == "own_container" else parts)if branch == "clipped":# A dismissal is the one instruction that cannot be carried out here: the dialog, overlay# and banner it would name are what this branch is entered BECAUSE the probe ruled out.@@ -12285,6 +12300,14 @@ def build_browser_tools(f"({challenge_frame}), so it cannot be {verb}{also}. Its controls: {controls_desc}.",error_class="covered",)+ if branch == "own_container":+ return ToolResult.error(+ f"{selector} cannot be {verb}: an element inside {layer_desc}, the container this field "+ "itself sits in, takes the pointer at the field's position. Do not try to dismiss it; it "+ 'holds the field. Click [data-tv3-catcher="1"] to activate the field, then retry '+ f"{selector}.",+ error_class="covered",+ )return ToolResult.error(f"{selector} is covered by {layer_desc}, so it cannot be {verb}{also}. "# The layer may be a general modal, not just a consent wall -- these are every control@@ -12750,6 +12773,7 @@ def build_browser_tools()else:await _require_single_target(page, selector)+ await input_dispatch.approach(page, selector)pre: dict[str, Any] | None = Nonetry:pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, await _probe_arg(page, selector))@@ -12762,7 +12786,7 @@ def build_browser_tools(# commit baseline must be the POST-hover fingerprint, or a mere highlight would read as# "its state changed" commit evidence on a no-op click.try:- await page.hover(selector, timeout=2000)+ await input_dispatch.hover(page, selector, timeout=2000)hovered = await page.evaluate(_MENU_AFTER_JS, await _probe_arg(page, selector))if isinstance(hovered, dict) and hovered.get("optState"):pre["optState"] = hovered["optState"]@@ -12940,7 +12964,7 @@ def build_browser_tools("different control, or use the native control directly if the page exposes one")try:- await _current_page().mouse.click(float(label_click["x"]), float(label_click["y"]))+ await input_dispatch.click_at(_current_page(), float(label_click["x"]), float(label_click["y"]))except Exception as e:return ToolResult.error(f"click on {selector} via its label failed ({type(e).__name__}) — the page may have "@@ -12951,13 +12975,7 @@ def build_browser_tools(# Resolved again here rather than reused: this evaluate is the action, not a probe, and a# selector naming a control through its host resolves ONLY through the handle -- a node the# page replaced while the probes ran would be clicked off-document, silently.- fired = await page.evaluate(- "(arg) => { const _q = "- + _ROOT_QUERY_JS- + "; const el = _q.find(arg.sel) || arg.el;"- + " if (!el || !el.isConnected) return false; el.click(); return true; }",- await _probe_arg(page, selector),- )+ fired = await input_dispatch.js_click(page, _ROOT_QUERY_JS, await _probe_arg(page, selector))if not fired:return ToolResult.error(f"{selector} left the page before the click could land — it was replaced by a "@@ -12976,9 +12994,9 @@ def build_browser_tools(# The probe already answered: the only thing "over" this control is its own label# (slotted, or a sibling for=id), which the driver's containment check would wait# 15s to reject.- await page.click(selector, timeout=_ACTION_TIMEOUT_MS, force=True)+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS, force=True)else:- await page.click(selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS)except Exception as e:gone = Falsetry:@@ -13051,7 +13069,7 @@ def build_browser_tools(return ToolResult.ok(f"{selector} is already selected — no change needed")try:await page.locator(selector).first.wait_for(state="visible", timeout=3000)- await page.click(selector, timeout=5000, force=True)+ await input_dispatch.click(page, selector, timeout=5000, force=True)except Exception:raise e from Noneelse:@@ -13194,7 +13212,7 @@ def build_browser_tools(return errorselector = args["selector"]await _require_single_target(page, selector)- await page.hover(selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.hover(page, selector, timeout=_ACTION_TIMEOUT_MS)return ToolResult.ok(f"hovered {selector}")async def _annotate_challenge_frame(realm: Any, top_page: Any, occluder: dict[str, Any] | None) -> None:@@ -13307,7 +13325,7 @@ def build_browser_tools(return Falserealm = realm.parent_frameawait page.evaluate("() => { window.__tv3_doc = 1; }")- await top.mouse.click(x, y)+ await input_dispatch.click_at(top, x, y)except Exception:return Falsereturn True@@ -13341,7 +13359,7 @@ def build_browser_tools(try:# Still focused explicitly: the press need not move the caret, and the display it landed on# may have no handler that forwards focus to the input.- await page.focus(selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.focus(page, selector, timeout=_ACTION_TIMEOUT_MS)held = await page.evaluate(_ACTIVE_IS_JS, await _probe_arg(page, selector))except Exception:held = None@@ -13371,7 +13389,7 @@ def build_browser_tools(# "is this still the same document" exactly -- the same technique the pre-snapshot uses.await page.evaluate("() => { window.__tv3_doc = 1; }")try:- await page.click(selector, timeout=_ACTION_TIMEOUT_MS, force=True)+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS, force=True)except Exception as exc:# Force skips the hit-target check but not the viewport one, which rejects any box of# at most one square pixel: a segment input kept sub-pixel under its own display layer.@@ -13398,7 +13416,7 @@ def build_browser_tools(return False, occluder, "click"else:try:- await page.click(selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.click(page, selector, timeout=_ACTION_TIMEOUT_MS)except Exception as exc:# A segmented control can keep its real input off-viewport with tabindex=-1 under an# aria-hidden display layer: the probe finds nothing on top of it, but the click's@@ -13413,7 +13431,7 @@ def build_browser_tools(if focused is False:# focus() needs no hit target, so it repairs a skin that swallowed the click without# forwarding it. Typing then goes to the field rather than wherever the caret was.- await page.focus(selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.focus(page, selector, timeout=_ACTION_TIMEOUT_MS)return True, None, reachdef _occluder_labels_hold(occluder: dict[str, Any] | None, value: str) -> bool:@@ -14096,7 +14114,9 @@ def build_browser_tools(# Short wait on purpose: this field was writable moments ago and visibly moved, so a# page that has since hidden or disabled it is not going to become actionable -- the# full timeout would just buy that refusal once per field.- await page.fill(f'[data-tv3-collateral="{tag}"]', was, timeout=_COLLATERAL_FILL_TIMEOUT_MS)+ await input_dispatch.fill(+ page, f'[data-tv3-collateral="{tag}"]', was, timeout=_COLLATERAL_FILL_TIMEOUT_MS+ )except Exception:LOG.debug("taskv3 collateral restore failed", tag=tag)# Count what actually went back, over the fields that MOVED. Counting over the whole scan would@@ -14125,7 +14145,7 @@ def build_browser_tools(if current.strip() and not any(current.strip().lower() == q.strip().lower() for q in typed):returntry:- await page.fill(selector, pre_value, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.fill(page, selector, pre_value, timeout=_ACTION_TIMEOUT_MS)except Exception:LOG.debug("taskv3 pre-type value restore failed", selector=selector)@@ -14223,7 +14243,7 @@ def build_browser_tools(# A page can replace the field during cleanup; writes and Escape must stay on the original node.# released_input is set only once a dispatch returned, so a fill that raised first stays uncharged.if current:- await offer.element.fill("", timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.fill(_current_page(), offer.element, "", timeout=_ACTION_TIMEOUT_MS)released_input = Trueif await offer.element.evaluate(read_value) != "":continue@@ -14265,6 +14285,7 @@ def build_browser_tools(collateral: list[list[str]] | None = None,query: str | None = None,press_enter: bool = False,+ secret: bool = False,) -> tuple[_TypeaheadPick, str | None, str | None, _Reach]:# Keystroke-type (so a widget's async suggestion fetch fires on real key events). Snapshot the# visible DOM BEFORE the focus click, not just before typing: a widget that opens its full list on@@ -14279,6 +14300,7 @@ def build_browser_tools(# Same reason: fill()/type() below open the widget's own list (aria-expanded=true) for the rest# of this attempt, and ownCommittedSurface reads nothing while it is open.pre_own = await _own_surface_text(page, selector)+ await input_dispatch.approach(page, selector)try:await page.evaluate(_PRESNAPSHOT_JS)except Exception:@@ -14306,8 +14328,10 @@ def build_browser_tools(# Only a field the click could not reach can misroute its keys, and only that path restores.# A checked click leaves the page untagged and pays nothing.collateral.extend(await _capture_collateral(page, selector))- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)- await page.type(selector, typed, delay=15, timeout=_typing_timeout_ms(typed))+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.type_keys(+ page, selector, typed, delay=15, timeout=_typing_timeout_ms(typed), secret=secret, replace=True+ )if collateral:collateral[:] = await _collateral_moved_while_typing(page, collateral)if not presnapshot_ok or reach != "click":@@ -14407,7 +14431,7 @@ def build_browser_tools(try:try:- await page.press(selector, "Enter", timeout=5000)+ await input_dispatch.press(page, selector, "Enter", timeout=5000)except Exception:return _settled(ToolResult.error(@@ -14500,9 +14524,9 @@ def build_browser_tools(return Nonetry:if element is None:- await page.press(selector, "Escape", timeout=5000)+ await input_dispatch.press(page, selector, "Escape", timeout=5000)else:- await element.press("Escape", timeout=5000)+ await input_dispatch.press(page, element, "Escape", timeout=5000)except Exception:LOG.debug("taskv3 lingering typeahead list close failed", selector=selector)return None@@ -14776,7 +14800,7 @@ def build_browser_tools(for _ in range(4):if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):return- await locator.press("Backspace")+ await input_dispatch.press(page, locator, "Backspace")returnheld = _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector))if held:@@ -14784,15 +14808,15 @@ def build_browser_tools(caret = await locator.evaluate(_CARET_TO_END_JS, timeout=2000)# End is only a caret key in a text box; an ARIA spinbutton takes it as "set to maximum".if caret == "end_key" and await locator.evaluate(_IS_TEXT_BOX_JS, timeout=2000):- await locator.press("End")+ await input_dispatch.press(page, locator, "End")elif caret == "end_key":# No caret key here, so Delete clears the digits after the press point.for _ in range(len(held.group(0))):- await locator.press("Delete")+ await input_dispatch.press(page, locator, "Delete")if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):returnfor _ in range(len(held.group(0)) if held else 0):- await locator.press("Backspace")+ await input_dispatch.press(page, locator, "Backspace")if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):returnexcept Exception:@@ -14888,7 +14912,7 @@ def build_browser_tools(except Exception:return "focus"try:- await locator.click(timeout=2000)+ await input_dispatch.click(page, locator, timeout=2000)except Exception:return "focus"return "click"@@ -14920,7 +14944,9 @@ def build_browser_tools(# Pressed through Playwright as the layer ELEMENT, never forced: its actionability and hit-target checks# refuse the press when the layer has been hidden or covered since the aim measured it.try:- await layer.click(position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500)+ await input_dispatch.click_handle(+ page, layer, position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500+ )except Exception:return "focus"return "mirror"@@ -14937,14 +14963,14 @@ def build_browser_tools(passaim = await _aim_date_segment(page, selector, locator)try:- await locator.focus(timeout=2000)+ await input_dispatch.focus(page, locator, timeout=2000)except Exception:return False, aimawait _clear_date_segment(page, selector, locator)# locator.fill() does not commit digits into a date spinbutton segment; only real keystrokes# advance it, so this and the element-focused fallback below both TYPE rather than fill.try:- await _current_page().keyboard.type(digits, delay=40)+ await input_dispatch.type_keys(_current_page(), None, digits, delay=40)except Exception:passif await _date_segment_holds(page, selector, digits):@@ -14952,9 +14978,9 @@ def build_browser_tools(try:if _date_segment_aim_on():aim = await _aim_date_segment(page, selector, locator)- await locator.focus(timeout=2000)+ await input_dispatch.focus(page, locator, timeout=2000)await _clear_date_segment(page, selector, locator)- await locator.press_sequentially(digits, delay=40)+ await input_dispatch.type_keys(page, locator, digits, delay=40)except Exception:return False, aimreturn await _date_segment_holds(page, selector, digits), aim@@ -15190,7 +15216,6 @@ def build_browser_tools(url_before: str | None,) -> ToolResult:n = len(text)- # Registered as a secret only where it came from one: the one-time-code resolver registers the whole code.url_before_fill = await _url(_current_page())field = f"the {n}-box code field at {selector}"@@ -15232,12 +15257,12 @@ def build_browser_tools(held: str | None = Nonetry:# fill() waits for the box to be enabled, so its timeout bounds the wait for the box before it.- await box.fill(char, timeout=_CODE_BOX_ENABLE_WAIT_MS)+ await input_dispatch.fill(page, box, char, timeout=_CODE_BOX_ENABLE_WAIT_MS)typed = i + 1held = await box.input_value(timeout=1000)if not held:# v1's fallback: a box that drops a programmatic value may still take the key.- await box.press_sequentially(char, timeout=1000)+ await input_dispatch.type_keys(page, box, char, timeout=1000, secret=secret)held = await box.input_value(timeout=1000)except Exception:held = None@@ -15296,7 +15321,7 @@ def build_browser_tools()if not press_enter:return ToolResult.ok(filled)- await page.press(f'[data-tv3-codebox="{n - 1}"]', "Enter")+ await input_dispatch.press(page, f'[data-tv3-codebox="{n - 1}"]', "Enter")return ToolResult.ok(f"{filled}, then pressed Enter", data={"url_before": url_before})kept = sum(bool(box_state.get("value")) for box_state in state)if remounted:@@ -15405,8 +15430,8 @@ def build_browser_tools()if not matched:return ToolResult.error(- f"the field at {selector} has {run} one-character boxes from that box on, but the text has {len(code)} "- "characters once spaces and dashes are dropped. The length does not match the box count, "+ f"the field at {selector} has {run} one-character boxes from that box on, but the text has "+ f"{len(code)} characters once spaces and dashes are dropped. The length does not match the box count, ""so nothing was typed. Re-observe the field and type one character for each box.",error_class="text_not_held",)@@ -15456,7 +15481,7 @@ def build_browser_tools(if not reachable:return _covered_error(selector, occluder)if clear:- await page.fill(selector, text, timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.fill(page, selector, text, timeout=_ACTION_TIMEOUT_MS)if box_text and await _read_field_value(page, selector) != text:return ToolResult.error(f"typed into {selector}, but it takes one character and does not hold the typed text "@@ -15491,7 +15516,9 @@ def build_browser_tools(at_end = Falseif at_end:# page.type() would focus the field again, and a fresh focus puts the caret back at the start.- sent = await _type_keys_before_deadline(text, press_end=at_end == "end_key")+ sent = await _type_keys_before_deadline(+ text, press_end=at_end == "end_key", secret=text != args.get("text", "")+ )if sent < len(text):progress = ("partway" if text != args.get("text", "") else f"after {sent} of {len(text)} characters"@@ -15503,7 +15530,9 @@ def build_browser_tools(error_class="text_not_held",)else:- await page.type(selector, text, timeout=_typing_timeout_ms(text))+ await input_dispatch.type_keys(+ page, selector, text, timeout=_typing_timeout_ms(text), secret=text != args.get("text", "")+ )not_held = await _appended_text_not_held(field, selector, text, before, text_is_secret=text != args.get("text", ""))@@ -15513,7 +15542,7 @@ def build_browser_tools(if not_held is not None:return not_heldif press_enter:- await page.press(selector, "Enter")+ await input_dispatch.press(page, selector, "Enter")return ToolResult.ok(f"typed into {selector}", data={"url_before": url_before})return ToolResult.ok(f"typed into {selector}")@@ -15543,7 +15572,7 @@ def build_browser_tools(# The editable element focus landed on after focusing the target, or None when it landed on none.# Fail-open: a probe error must not refuse a field type() could have filled.try:- await page.focus(selector, timeout=2000)+ await input_dispatch.focus(page, selector, timeout=2000)handle = await page.locator(selector).first.evaluate_handle(_FOCUS_IN_EDITABLE_JS, timeout=2000)except Exception:return _FOCUS_PROBE_FAILED@@ -15552,14 +15581,14 @@ def build_browser_tools(await handle.dispose()return element- async def _type_keys_before_deadline(text: str, *, press_end: bool) -> int:+ async def _type_keys_before_deadline(text: str, *, press_end: bool, secret: bool) -> int:# One key at a time: a single keyboard.type(text) keeps typing in the browser after its caller stops# waiting, into whatever field the next action focuses. The keyboard is the page's; a frame has none.- keyboard = _current_page().keyboard+ top = _current_page()deadline = time.monotonic() + _typing_timeout_ms(text) / 1000if press_end:try:- await asyncio.wait_for(keyboard.press("End"), deadline - time.monotonic())+ await asyncio.wait_for(input_dispatch.press(top, None, "End"), deadline - time.monotonic())except asyncio.TimeoutError:return 0for sent, key in enumerate(text):@@ -15567,7 +15596,7 @@ def build_browser_tools(if remaining <= 0:return senttry:- await asyncio.wait_for(keyboard.type(key), remaining)+ await asyncio.wait_for(input_dispatch.type_keys(top, None, key, secret=secret), remaining)except asyncio.TimeoutError:return sentreturn len(text)@@ -15662,7 +15691,7 @@ def build_browser_tools(collateral: list[list[str]] = []try:pick, pre_value, pre_own, reach = await _type_and_commit(- page, selector, text, rounds=3, focus_fallback=True, collateral=collateral+ page, selector, text, rounds=3, focus_fallback=True, collateral=collateral, secret=text_is_secret)except _FieldCovered as exc:return _covered_error(exc.selector, exc.occluder)@@ -15683,7 +15712,7 @@ def build_browser_tools(reacted = await _find_suggestion_rows(page, selector, text, any_region=True) is not Noneif not reacted:try:- await _current_page().keyboard.press("Tab")+ await input_dispatch.press(_current_page(), None, "Tab")except Exception:passheld = await _read_field_value(page, selector)@@ -15872,12 +15901,13 @@ def build_browser_tools(menu_open = Falseif menu_open:try:- await _current_page().keyboard.press("Escape")+ await input_dispatch.press(_current_page(), None, "Escape")await asyncio.sleep(0.1)except Exception:passasync def _open_and_enumerate() -> tuple[dict[str, Any] | None, ToolResult | None]:+ await input_dispatch.approach(page, selector)try:await page.evaluate(_PRESNAPSHOT_JS)except Exception:@@ -15893,7 +15923,7 @@ def build_browser_tools(# 5s, not the 15s a routine click waits: the control is already present (we just typed# into it, or it is a visible button), so it opens at once — a long wait here only delays# the error on a field that unmounted itself, which must fail loudly, not slowly.- await page.click(selector, timeout=5000)+ await input_dispatch.click(page, selector, timeout=5000)except Exception:return None, ToolResult.error(f"could not click {selector} to open its option list — the field is NOT filled; "@@ -16289,7 +16319,7 @@ def build_browser_tools(contenders = _value_contenders(value, row_texts)def _ambiguous_error() -> ToolResult:- named = "; ".join(repr(t[:60]) for t in contenders[:15])+ named = "; ".join(repr(_model_text(t, 60)) for t in contenders[:15])return ToolResult.error(f"{value!r} is ambiguous in {selector}'s list — it names more than one option "f"({named}); pass the one option's full text"@@ -16355,7 +16385,7 @@ def build_browser_tools(async def _take_back() -> ToolResult | None:try:- await page.fill(box_sel, str(box.get("prior") or ""), timeout=2000)+ await input_dispatch.fill(page, box_sel, str(box.get("prior") or ""), timeout=2000)except Exception:LOG.debug("taskv3 popup filter take-back failed", selector=selector)# A box that is not a filter (a key sink that commits the focused row) shows up here whatever@@ -16422,9 +16452,11 @@ def build_browser_tools(base_rows, _, _ = await _read_rows(mark=True)base_texts = [str(o.get("text") or "") for o in base_rows]try:- await page.fill(box_sel, "", timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.clear(page, box_sel, timeout=_ACTION_TIMEOUT_MS)cleared = await _box_value()- await page.type(box_sel, query[:1], delay=15, timeout=_typing_timeout_ms(query[:1]))+ await input_dispatch.type_keys(+ page, box_sel, query[:1], delay=15, timeout=_typing_timeout_ms(query[:1])+ )except Exception:return await _take_back()# A key sink swallows the keys it forwards to the list, so a box whose value does not newly show the@@ -16454,7 +16486,9 @@ def build_browser_tools()try:if query[1:]:- await page.type(box_sel, query[1:], delay=15, timeout=_typing_timeout_ms(query[1:]))+ await input_dispatch.type_keys(+ page, box_sel, query[1:], delay=15, timeout=_typing_timeout_ms(query[1:])+ )except Exception:return await _take_back()last: list[str] | None = None@@ -16534,7 +16568,9 @@ def build_browser_tools(several = list(dict.fromkeys(contenders + several))if len(several) < 2 or verdict == "incomplete":several = []- listed = "; ".join(repr(t[:60]) for t in (several or [str(o.get("text") or "") for o in filtered])[:15])+ listed = "; ".join(+ repr(_model_text(t, 60)) for t in (several or [str(o.get("text") or "") for o in filtered])[:15]+ )more = f"; +{len(filtered) - 15} more" if len(filtered) > 15 and not several else ""if visible is not None:return _unproven_row_error(@@ -16684,7 +16720,7 @@ def build_browser_tools(# itself.try:if await page.evaluate(_MENU_OPEN_JS, probe):- await _current_page().keyboard.press("Escape")+ await input_dispatch.press(_current_page(), None, "Escape")except Exception:passreturn ToolResult.ok(f"{matched!r} was already selected for {selector}; left it as is")@@ -16906,14 +16942,16 @@ def build_browser_tools(# anything (closed over from _commit_custom_combobox_attempt) — re-reading it here# would see the list THIS rung's own typing just opened and always read as changed.try:- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)await asyncio.sleep(0.2)# A rung is a fresh question, so it needs a fresh answer to "what appeared in# reaction": the snapshot from before the first attempt would mark the rows the# PREVIOUS query left on screen as pre-existing, and a widget that keeps its row# nodes across a re-search would then have no reaction to show at all.await page.evaluate(_PRESNAPSHOT_JS)- await page.type(selector, rung, delay=15, timeout=_typing_timeout_ms(rung))+ await input_dispatch.type_keys(+ page, selector, rung, delay=15, timeout=_typing_timeout_ms(rung), replace=True+ )except Exception:return []rung_pick = await _commit_typeahead(@@ -16962,7 +17000,7 @@ def build_browser_tools(# the focus pass so those rows are recorded as offered, exactly as if the field had opened# showing them — _FOCUS_OFFERED_LABELS_JS then reads them under the same own-list attribution.try:- await page.fill(selector, "", timeout=_ACTION_TIMEOUT_MS)+ await input_dispatch.clear(page, selector, timeout=_ACTION_TIMEOUT_MS)except Exception:returnawait asyncio.sleep(0.4)@@ -17254,16 +17292,20 @@ def build_browser_tools(# holds ['Alpha']` and told the model to re-pass a set it had never asked for.if label_list is not None:by_label, asked = True, label_list- await page.select_option(selector, label=label_list, timeout=_ACTION_TIMEOUT_MS, force=force)+ await input_dispatch.select_option(+ page, selector, label=label_list, timeout=_ACTION_TIMEOUT_MS, force=force+ )elif value_list is not None:by_label, asked = False, value_list- await page.select_option(selector, value=value_list, timeout=_ACTION_TIMEOUT_MS, force=force)+ await input_dispatch.select_option(+ page, selector, value=value_list, timeout=_ACTION_TIMEOUT_MS, force=force+ )elif label is not None:by_label, asked = True, [label]- await page.select_option(selector, label=label, timeout=_ACTION_TIMEOUT_MS, force=force)+ await input_dispatch.select_option(page, selector, label=label, timeout=_ACTION_TIMEOUT_MS, force=force)else:by_label, asked = False, ([value] if isinstance(value, str) else [])- await page.select_option(selector, value=value, timeout=_ACTION_TIMEOUT_MS, force=force)+ await input_dispatch.select_option(page, selector, value=value, timeout=_ACTION_TIMEOUT_MS, force=force)# A set-valued control is read back whether or not it was forced: without it a call that# discarded every prior selection reports the same bare success as one that added to them.if not force and not is_multi:@@ -17334,13 +17376,13 @@ def build_browser_tools(url_before = await _url(_current_page())if selector:await _require_single_target(page, selector)- await page.press(selector, key)+ await input_dispatch.press(page, selector, key)else:# Page-level, not realm-level: a Frame has no `keyboard`, and an unaddressed keypress goes# to whatever holds focus regardless of which document that is. Reached only with no# selector, so the realm is the page today -- written through _current_page() so it stays# correct if that ever stops being true.- await _current_page().keyboard.press(key)+ await input_dispatch.press(_current_page(), None, key)return ToolResult.ok(f"pressed {key}", data={"url_before": url_before})async def scroll(args: dict[str, Any]) -> ToolResult:@@ -17360,7 +17402,7 @@ def build_browser_tools(# Reachable WITH a selector -- a ref that resolved to a selector now matching nothing falls# through to here -- so `page` can be a child frame, and a Frame has no `mouse`. The wheel is a# viewport gesture either way, so it belongs to the page.- await _current_page().mouse.wheel(0, amount)+ await input_dispatch.wheel(_current_page(), 0, amount)return ToolResult.ok(f"scrolled {amount}px")async def wait(args: dict[str, Any]) -> ToolResult:@@ -17627,7 +17669,7 @@ def build_browser_tools(# picker it opens, as v1 does. The picker belongs to the page, whichever frame opened it.try:async with _current_page().expect_file_chooser(timeout=_FILE_CHOOSER_TIMEOUT_MS) as chooser_info:- await el.click(timeout=_FILE_CHOOSER_TIMEOUT_MS)+ await input_dispatch.click_handle(page, el, timeout=_FILE_CHOOSER_TIMEOUT_MS)chooser = await chooser_info.valueexcept Exception as exc:if not is_driver_timeout_error(exc):@@ -17639,10 +17681,10 @@ def build_browser_tools({**staged, "page_state_changed": True},error_class="no_file_input",)- await chooser.set_files([local_path])+ await input_dispatch.set_files(chooser, [local_path])file_input = chooser.elementelse:- await file_input.set_input_files([local_path])+ await input_dispatch.set_input_files(file_input, [local_path])populated = await _input_holds_file(file_input)# Settle + a small randomized delay so the upload and a following submit are not dispatched# in the same instant, matching v1's upload cadence (the engine that clears this step reliably).@@ -17674,7 +17716,7 @@ def build_browser_tools(new_lines = _newly_rendered_lines(text_before, text_after)new_text = "\n".join(new_lines)said = " | ".join(line for line in new_lines if _mentions_filename(line, local_path))- said = said[:_FILENAME_MENTION_CHARS]+ said = _model_text(said, _FILENAME_MENTION_CHARS)if probe.saw_upload() and not _UPLOAD_REJECTION_WORDS.search(new_text):LOG.info("taskv3 file_upload input cleared after attach but the page shows the uploaded file")return ToolResult.ok(diff --git a/skyvern/webeye/utils/captcha_solver.py b/skyvern/webeye/utils/captcha_solver.pyindex 66ff4f06b..2742a7bf8 100644--- a/skyvern/webeye/utils/captcha_solver.py+++ b/skyvern/webeye/utils/captcha_solver.py@@ -17,14 +17,16 @@ from __future__ import annotationsimport asyncioimport contextlibimport time+from collections.abc import Awaitable, Callablefrom dataclasses import dataclassfrom enum import StrEnumfrom typing import TYPE_CHECKING, Any, Literalfrom urllib.parse import urlparseimport structlog+from playwright.async_api import ElementHandlefrom playwright.async_api import Error as PlaywrightError-from playwright.async_api import Frame, Page+from playwright.async_api import Frame, Locator, Pagefrom skyvern.config import settingsfrom skyvern.forge import app@@ -48,6 +50,9 @@ class ChallengeStatus(StrEnum):ChallengeArm = Literal["vendor_handler", "dom_checkbox", "recaptcha_anchor_frame", "extension", "token"]+# How the ladder clicks a widget; a caller that routes its input elsewhere supplies its own.+ChallengeClick = Callable[[Locator], Awaitable[None]]+ChallengeHandleClick = Callable[[ElementHandle], Awaitable[None]]ChallengePageState = Literal["clear", "challenged", "not_rechecked"]@@ -313,6 +318,8 @@ async def solve_challenge_ladder(workflow_run_id: str | None = None,browser_session_id: str | None = None,probe_child_frames: bool = False,+ click: ChallengeClick | None = None,+ click_handle: ChallengeHandleClick | None = None,) -> bool:"""``solve_challenge`` as a bool: True when solved, False when no challenge is on the page, andCaptchaChallengeUnsolvedError when one is present but unsolved or unsupported."""@@ -322,6 +329,8 @@ async def solve_challenge_ladder(workflow_run_id=workflow_run_id,browser_session_id=browser_session_id,probe_child_frames=probe_child_frames,+ click=click,+ click_handle=click_handle,)if outcome.status in (ChallengeStatus.UNSOLVED, ChallengeStatus.UNSUPPORTED):raise CaptchaChallengeUnsolvedError("CAPTCHA could not be solved.")@@ -335,6 +344,8 @@ async def solve_challenge(workflow_run_id: str | None = None,browser_session_id: str | None = None,probe_child_frames: bool = False,+ click: ChallengeClick | None = None,+ click_handle: ChallengeHandleClick | None = None,) -> ChallengeOutcome:"""Solve a detected challenge inside one captcha-solver lifecycle scope and report what happened. A vendor pagegoes only to its handler and counts as cleared only when a fresh probe finds neither the vendor nor a widget."""@@ -378,6 +389,8 @@ async def solve_challenge(workflow_run_id=workflow_run_id,browser_session_id=browser_session_id,probe_child_frames=probe_child_frames,+ click=click or _click_directly,+ click_handle=click_handle or _click_directly,)except TimeoutError:LOG.info("CAPTCHA ladder stopped at its budget", arm="ladder")@@ -443,6 +456,10 @@ async def _rendered_frame_vendor(page: Page | RecordingPage) -> str | None:return await rendered_challenge_vendor([frame for frame in page.frames if frame.parent_frame is not None])+async def _click_directly(target: Locator | ElementHandle) -> None:+ await target.click()++async def _solve_challenge_ladder_impl(page: Page | RecordingPage,*,@@ -451,6 +468,8 @@ async def _solve_challenge_ladder_impl(workflow_run_id: str | None = None,browser_session_id: str | None = None,probe_child_frames: bool = False,+ click: ChallengeClick = _click_directly,+ click_handle: ChallengeHandleClick = _click_directly,) -> ChallengeArm | None:"""The arm that solved a detected widget, or None when the cheap probes find none and no solver is called.Raises CaptchaChallengeUnsolvedError when no arm, each clamped to the budget left since ``start``, clears it."""@@ -474,14 +493,14 @@ async def _solve_challenge_ladder_impl(candidate = checkbox.firsttry:if await candidate.is_visible() and await candidate.is_enabled():- await candidate.click()+ await click(candidate)await page.wait_for_timeout(100)if await candidate.is_checked() or await _bounded_locator_count(checkbox) == 0:continuation = page.locator(_CAPTCHA_CONTINUE_SELECTOR)if await _bounded_locator_count(continuation) == 1:continuation_candidate = continuation.firstif await continuation_candidate.is_visible() and await continuation_candidate.is_enabled():- await continuation_candidate.click()+ await click(continuation_candidate)await page.wait_for_timeout(100)if await _bounded_locator_count(checkbox) == 0:return "dom_checkbox"@@ -515,7 +534,7 @@ async def _solve_challenge_ladder_impl(if await candidate.get_attribute("aria-checked") == "true":breakpage_url_before_click = urlparse(page.url)._replace(fragment="").geturl()- await candidate.click()+ await click_handle(candidate)anchor_clicked = TrueLOG.info("CAPTCHA anchor frame clicked", arm="recaptcha_anchor_frame")await page.wait_for_timeout(_RECAPTCHA_ANCHOR_SETTLE_MS)diff --git a/tests/unit/test_browser_effect_sink_enumeration.py b/tests/unit/test_browser_effect_sink_enumeration.pyindex 5740f91af..d090234df 100644--- a/tests/unit/test_browser_effect_sink_enumeration.py+++ b/tests/unit/test_browser_effect_sink_enumeration.py@@ -1,4 +1,5 @@import ast+import refrom collections import Counterfrom pathlib import Path@@ -72,6 +73,24 @@ _DISCOVERED_BROWSER_API_CALLS = {{"clear": 1, "click": 2, "move": 2, "scroll_into_view_if_needed": 1, "type": 3, "wheel": 1}),"skyvern/forge/sdk/event/factory.py": Counter({"click": 1, "wheel": 1}),+ "skyvern/forge/taskv3/input_dispatch.py": Counter(+ {+ "click": 6,+ "evaluate": 2,+ "fill": 3,+ "focus": 2,+ "hover": 1,+ "insert_text": 1,+ "press": 3,+ "press_sequentially": 1,+ "scroll_into_view_if_needed": 1,+ "select_option": 2,+ "set_files": 1,+ "set_input_files": 1,+ "type": 2,+ "wheel": 1,+ }+ ),"skyvern/webeye/real_browser_state.py": Counter({"close": 6, "evaluate": 1, "goto": 1, "new_page": 3, "reload": 2}),}@@ -125,6 +144,8 @@ _EVALUATE_CALLERS = {}),"skyvern/webeye/real_browser_state.py": Counter({"stop_page_loading": 1}),+ # The settle wait after a treatment cursor approach, and the hidden-native-control click V3 fires in-page.+ "skyvern/forge/taskv3/input_dispatch.py": Counter({"approach": 1, "js_click": 1}),# OTP box/scope/document marker writes, visual masks and read-only hit-test geometry."skyvern/webeye/utils/dom.py": Counter({"apply_secret_visual_mask": 1, "mark_totp_box": 2, "blur": 1, "_pointer_interceptor_matches_label": 1}@@ -165,6 +186,7 @@ def _owned_source_paths() -> tuple[str, ...]:Path("skyvern/webeye/utils/dom.py"),*Path("skyvern/webeye/actions").glob("*.py"),*Path("skyvern/forge/sdk/event").glob("*.py"),+ Path("skyvern/forge/taskv3/input_dispatch.py"),}return tuple(sorted(path.as_posix() for path in paths))@@ -239,24 +261,142 @@ def test_discovered_browser_api_lower_bound_is_stable() -> None:}assert observed == _DISCOVERED_BROWSER_API_CALLS- assert sum(sum(methods.values()) for methods in observed.values()) == 184+ assert sum(sum(methods.values()) for methods in observed.values()) == 211handler_candidates = _candidate_signatures("skyvern/webeye/actions/handler.py", _CANDIDATE_METHODS)classified_non_browser = Counter({signature: count for signature, count in handler_candidates.items() if signature in _NON_BROWSER_CANDIDATES})assert classified_non_browser == _NON_BROWSER_CANDIDATESassert sum(_NON_BROWSER_CANDIDATES.values()) == 7- assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 177+ assert sum(sum(methods.values()) for methods in observed.values()) - sum(_NON_BROWSER_CANDIDATES.values()) == 204def test_every_raw_evaluate_call_is_classified() -> None:observed = {path: callers for path in _owned_source_paths() if (callers := _callers_for_method(path, "evaluate"))}assert observed == _EVALUATE_CALLERS- assert sum(sum(callers.values()) for callers in observed.values()) == 37+ assert sum(sum(callers.values()) for callers in observed.values()) == 39def test_every_cdp_dispatch_is_classified_by_exact_command() -> None:observed = {path: callers for path in _owned_source_paths() if (callers := _cdp_send_callers(path))}assert observed == _CDP_SENDS+++# Task V3 sends input only through input_dispatch, which picks the plain or the humanized transport per run.+_INPUT_DISPATCH = Path("skyvern/forge/taskv3/input_dispatch.py")+_TASKV3_BANNED_INPUT_METHODS = frozenset(+ """check click dblclick dispatch_event drag_and_drop drag_to fill focus hover insert_text press press_sequentially select_option+ select_text set_checked set_files set_input_files tap type uncheck""".split()+)+_TASKV3_BANNED_DEVICES = frozenset({"mouse", "keyboard"})+_IN_PAGE_INPUT_JS = re.compile(r"\.click\(\)|dispatchEvent\(|new (Mouse|Keyboard|Pointer)Event|Input\.dispatch")+# `clear` is also a list and dict method, so it is banned only on a receiver shaped like an element.+_ELEMENT_SOURCES = frozenset({"locator", "query_selector", "element_handle", "nth", "first", "last"})++# Where each input_dispatch gesture reaches the event strategy factory: a gesture that loses one sends treatment+# runs down the plain path without anything else going red.+_INPUT_DISPATCH_FACTORY_CALLS = Counter(+ {+ ("_move_to_element", "move_cursor"): 1,+ ("approach", "move_to_element"): 1,+ ("click", "click_element"): 1,+ ("click", "move_to_element"): 1,+ ("click_at", "move_cursor"): 1,+ ("hover", "move_to_element"): 1,+ ("clear", "clear_field"): 1,+ ("type_keys", "type_text"): 1,+ ("wheel", "scroll_by"): 1,+ ("arm_fields", "registered_profile"): 1,+ }+)+++def _enclosing_function(parents: dict[ast.AST, ast.AST], node: ast.AST) -> str:+ current = node+ while current in parents:+ current = parents[current]+ if isinstance(current, (ast.AsyncFunctionDef, ast.FunctionDef)):+ return current.name+ return "<module>"+++def _is_element(node: ast.expr, element_names: set[str]) -> bool:+ if isinstance(node, ast.Await):+ node = node.value+ if isinstance(node, ast.Call):+ node = node.func+ if isinstance(node, ast.Name):+ return node.id in element_names+ return isinstance(node, ast.Attribute) and (node.attr in _ELEMENT_SOURCES or node.attr.startswith("get_by_"))+++def _direct_input_sites(path: Path, only_function: str | None = None) -> list[str]:+ tree = ast.parse(path.read_text())+ parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)}+ element_names = {+ target.id+ for node in ast.walk(tree)+ if isinstance(node, ast.Assign) and _is_element(node.value, set())+ for target in node.targets+ if isinstance(target, ast.Name)+ }+ sites: list[str] = []+ for node in ast.walk(tree):+ found: str | None = None+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):+ receiver = node.func.value+ dispatched = isinstance(receiver, ast.Name) and receiver.id == "input_dispatch"+ if node.func.attr in _TASKV3_BANNED_INPUT_METHODS and not dispatched:+ found = f"{node.func.attr} on {ast.unparse(node.func.value)}"+ elif node.func.attr == "clear" and _is_element(receiver, element_names):+ found = f"clear on {ast.unparse(receiver)}"+ elif isinstance(node, ast.Attribute) and node.attr in _TASKV3_BANNED_DEVICES:+ found = f".{node.attr} on {ast.unparse(node.value)}"+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "getattr"+ and len(node.args) >= 2+ and isinstance(node.args[1], ast.Constant)+ and node.args[1].value in _TASKV3_BANNED_INPUT_METHODS | _TASKV3_BANNED_DEVICES+ ):+ found = f"getattr {node.args[1].value!r}"+ if isinstance(node, ast.Constant) and isinstance(node.value, str) and _IN_PAGE_INPUT_JS.search(node.value):+ found = "in-page input script"+ if found is None:+ continue+ function = _enclosing_function(parents, node)+ if only_function is not None and function != only_function:+ continue+ sites.append(f"{path}:{getattr(node, 'lineno', '?')} {function} {found}")+ return sites+++def test_taskv3_input_flows_only_through_input_dispatch() -> None:+ sites = [+ site+ for path in sorted(Path("skyvern/forge/taskv3").rglob("*.py"))+ if path != _INPUT_DISPATCH+ for site in _direct_input_sites(path)+ ]+ # The captcha ladder lives outside taskv3 and V3 hands it the click to use.+ sites += _direct_input_sites(Path("skyvern/webeye/utils/captcha_solver.py"), "_solve_challenge_ladder_impl")++ assert sites == []+++def test_input_dispatch_reaches_the_event_strategy_factory_from_every_humanized_gesture() -> None:+ tree = ast.parse(_INPUT_DISPATCH.read_text())+ parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)}+ observed = Counter(+ (_enclosing_function(parents, node), node.func.attr)+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "EventStrategyFactory"+ )++ assert observed == _INPUT_DISPATCH_FACTORY_CALLSdiff --git a/tests/unit/test_taskv3_captcha_tools.py b/tests/unit/test_taskv3_captcha_tools.pyindex f12006616..e412c6b72 100644--- a/tests/unit/test_taskv3_captcha_tools.py+++ b/tests/unit/test_taskv3_captcha_tools.py@@ -11,7 +11,7 @@ import pytestfrom skyvern.forge import appfrom skyvern.forge.agent_functions import AgentFunction-from skyvern.forge.taskv3 import captcha_tools+from skyvern.forge.taskv3 import captcha_tools, input_dispatchfrom skyvern.webeye.utils import captcha_solver as captcha_solver_modulefrom skyvern.webeye.utils.captcha_solver import CaptchaChallengeUnsolvedErrorfrom tests.unit.conftest import OcrRecordingAgentFunction, ScopeRecordingAgentFunction@@ -206,9 +206,20 @@ async def test_solve_captcha_threads_ids(monkeypatch: pytest.MonkeyPatch) -> Non_task(workflow_run_id="wr_9", browser_session_id="bs_9"), _provider(page), organization_id="o_9")await tools[0].handler({})- ladder.assert_awaited_once_with(- page, organization_id="o_9", workflow_run_id="wr_9", browser_session_id="bs_9", probe_child_frames=True- )+ ladder.assert_awaited_once()+ kwargs = dict(ladder.await_args.kwargs)+ # The ladder's widget clicks go through V3's input dispatch, bound to the page being solved.+ click = kwargs.pop("click")+ click_handle = kwargs.pop("click_handle")+ assert (click.func, click.args) == (input_dispatch.click, (page,))+ assert (click_handle.func, click_handle.args) == (input_dispatch.click_handle, (page,))+ assert ladder.await_args.args == (page,)+ assert kwargs == {+ "organization_id": "o_9",+ "workflow_run_id": "wr_9",+ "browser_session_id": "bs_9",+ "probe_child_frames": True,+ }@pytest.mark.asynciodiff --git a/tests/unit/test_taskv3_input_dispatch.py b/tests/unit/test_taskv3_input_dispatch.pynew file mode 100644index 000000000..77833f749--- /dev/null+++ b/tests/unit/test_taskv3_input_dispatch.py@@ -0,0 +1,384 @@+from __future__ import annotations++from collections.abc import Awaitable, Callable, Iterator+from typing import Any+from unittest.mock import AsyncMock, MagicMock++import pytest+from playwright.async_api import ElementHandle, Locator, Page++from skyvern.forge import app+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import RunArm, SkyvernContext+from skyvern.forge.sdk.event.base import CursorEventStrategy, InputEventStrategy, ScrollEventStrategy+from skyvern.forge.sdk.event.factory import EventStrategyFactory+from skyvern.forge.taskv3 import input_dispatch+from skyvern.forge.taskv3.run_arms import HUMANIZED_INPUT_FLAG, resolve_run_arm++Call = tuple[str, tuple[Any, ...], dict[str, Any]]+++class _Device:+ def __init__(self, log: list[Call], name: str) -> None:+ self._log = log+ self._name = name++ def __getattr__(self, attr: str) -> Callable[..., Awaitable[None]]:+ async def record(*args: Any, **kwargs: Any) -> None:+ self._log.append((f"{self._name}.{attr}", args, kwargs))++ return record+++class _FakeLocator(_Device):+ def __init__(self, page: _FakePage, selector: str) -> None:+ super().__init__(page.calls, f"locator({selector})")+ self.page = page+ self.first = self+++class _FakeHandle(_Device):+ def __init__(self, page: _FakePage) -> None:+ super().__init__(page.calls, "handle")++ async def bounding_box(self) -> dict[str, float]:+ return {"x": 10.0, "y": 20.0, "width": 4.0, "height": 6.0}+++class _FakePage(_Device):+ def __init__(self) -> None:+ self.calls: list[Call] = []+ super().__init__(self.calls, "page")+ self.keyboard = _Device(self.calls, "keyboard")+ self.mouse = _Device(self.calls, "mouse")++ def __getattr__(self, attr: str) -> Callable[..., Awaitable[None]]:+ # A top-level page has neither, which is how the dispatcher tells it from a frame.+ if attr in ("page", "parent_frame"):+ raise AttributeError(attr)+ return super().__getattr__(attr)++ def locator(self, selector: str) -> _FakeLocator:+ return _FakeLocator(self, selector)+++class _RecordingCursor(CursorEventStrategy):+ def __init__(self, log: list[Call]) -> None:+ self._log = log++ async def move_to(self, page: Page, x: float, y: float) -> None:+ self._log.append(("cursor.move_to", (x, y), {}))++ async def move_to_element(self, page: Page, locator: Locator) -> tuple[float, float]:+ self._log.append(("cursor.move_to_element", (locator,), {}))+ return 0.0, 0.0++ async def click(self, page: Page, locator: Locator, *, timeout: float | None = None) -> None:+ self._log.append(("cursor.click", (locator,), {"timeout": timeout}))+++class _RecordingInput(InputEventStrategy):+ def __init__(self, log: list[Call]) -> None:+ self._log = log++ async def type_text(+ self,+ page: Page,+ locator: Locator | None,+ text: str,+ *,+ timeout: float | None,+ delay: float | None = None,+ no_wait_after: bool | None = None,+ allow_batched_playwright: bool = False,+ ) -> None:+ self._log.append(("input.type_text", (locator, text), {"delay": delay}))++ async def clear_field(+ self,+ page: Page,+ locator: Locator,+ char_count: int,+ *,+ timeout: float | None,+ force: bool | None = None,+ no_wait_after: bool | None = None,+ ) -> None:+ self._log.append(("input.clear_field", (locator,), {"timeout": timeout}))+++class _RecordingScroll(ScrollEventStrategy):+ def __init__(self, log: list[Call]) -> None:+ self._log = log++ async def scroll_to_element(self, page: Page, locator: Locator) -> None:+ self._log.append(("scroll.scroll_to_element", (locator,), {}))++ async def scroll_by(self, page: Page, delta_y: float) -> None:+ self._log.append(("scroll.scroll_by", (delta_y,), {}))+++def page() -> _FakePage:+ return _FakePage()+++def humanized_strategies(page: _FakePage) -> Iterator[list[Call]]:+ """A humanized registration, as USE_EVENT_STRATEGIES makes it, logging into the page's own call list."""+ EventStrategyFactory.set_cursor_strategy(_RecordingCursor(page.calls))+ EventStrategyFactory.set_input_strategy(_RecordingInput(page.calls))+ EventStrategyFactory.set_scroll_strategy(_RecordingScroll(page.calls))+ EventStrategyFactory.record_profile("default")+ try:+ yield page.calls+ finally:+ EventStrategyFactory.reset()+++def _pin(arm: RunArm) -> Iterator[SkyvernContext]:+ context = SkyvernContext()+ context.run_arms = {HUMANIZED_INPUT_FLAG: ("wr_1", arm)}+ skyvern_context.set(context)+ try:+ yield context+ finally:+ skyvern_context.reset()+++def control() -> Iterator[SkyvernContext]:+ yield from _pin("control")+++def treatment() -> Iterator[SkyvernContext]:+ yield from _pin("treatment")+++def _names(calls: list[Call]) -> list[str]:+ return [name for name, _, _ in calls]+++ ("reader", "humanized"),+ [+ (AsyncMock(return_value="treatment"), True),+ (AsyncMock(return_value="control"), False),+ (AsyncMock(return_value=None), False),+ (AsyncMock(return_value="true"), False),+ (AsyncMock(side_effect=RuntimeError("flag service down")), False),+ ],+ ids=["treatment", "control", "no_variant", "unknown_variant", "flag_error"],+)+async def test_only_a_treatment_run_sends_its_clicks_through_the_registered_strategy(+ monkeypatch: pytest.MonkeyPatch,+ page: _FakePage,+ humanized_strategies: list[Call],+ reader: AsyncMock,+ humanized: bool,+) -> None:+ monkeypatch.setattr(app.EXPERIMENTATION_PROVIDER, "get_value_cached", reader)+ context = SkyvernContext()+ skyvern_context.set(context)+ try:+ await resolve_run_arm(context, HUMANIZED_INPUT_FLAG, distinct_id="wr_1", organization_id="o_1", forced=False)+ await input_dispatch.click(page, "#go", timeout=1234)+ finally:+ skyvern_context.reset()++ assert _names(page.calls) == (["cursor.click"] if humanized else ["page.click"])+++async def test_input_outside_a_resolved_run_is_plain_playwright(+ page: _FakePage, humanized_strategies: list[Call]+) -> None:+ skyvern_context.reset()++ await input_dispatch.click(page, "#go", timeout=1234)+ await input_dispatch.type_keys(page, "#q", "abc", delay=15, timeout=900)+ await input_dispatch.wheel(page, 0, 800)++ assert page.calls == [+ ("page.click", ("#go",), {"timeout": 1234}),+ ("page.type", ("#q", "abc"), {"delay": 15, "timeout": 900}),+ ("mouse.wheel", (0, 800), {}),+ ]+++async def test_control_sends_each_gesture_as_the_exact_playwright_call_it_replaced(+ control: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]+) -> None:+ locator = page.locator("#seg")+ await input_dispatch.approach(page, "#go")+ await input_dispatch.click(page, "#go", timeout=5000, force=True)+ await input_dispatch.click(page, locator, timeout=2000)+ await input_dispatch.click_at(page, 10.0, 20.0)+ await input_dispatch.hover(page, "#menu", timeout=2000)+ await input_dispatch.focus(page, "#f", timeout=2000)+ await input_dispatch.fill(page, "#f", "v", timeout=100)+ await input_dispatch.clear(page, "#f", timeout=100)+ await input_dispatch.type_keys(page, "#f", "ab", delay=15, timeout=900)+ await input_dispatch.type_keys(page, "#f", "ab", timeout=900)+ await input_dispatch.type_keys(page, None, "7", delay=40)+ await input_dispatch.type_keys(page, locator, "7", delay=40)+ await input_dispatch.press(page, "#f", "Enter")+ await input_dispatch.press(page, None, "Escape")+ await input_dispatch.press(page, locator, "Backspace")+ await input_dispatch.select_option(page, "#s", value=["a"], timeout=100, force=False)+ await input_dispatch.wheel(page, 0, -800)++ assert page.calls == [+ ("page.click", ("#go",), {"timeout": 5000, "force": True}),+ ("locator(#seg).click", (), {"timeout": 2000}),+ ("mouse.click", (10.0, 20.0), {}),+ ("page.hover", ("#menu",), {"timeout": 2000}),+ ("page.focus", ("#f",), {"timeout": 2000}),+ ("page.fill", ("#f", "v"), {"timeout": 100}),+ ("page.fill", ("#f", ""), {"timeout": 100}),+ ("page.type", ("#f", "ab"), {"delay": 15, "timeout": 900}),+ ("page.type", ("#f", "ab"), {"timeout": 900}),+ ("keyboard.type", ("7",), {"delay": 40}),+ ("locator(#seg).press_sequentially", ("7",), {"delay": 40}),+ ("page.press", ("#f", "Enter"), {}),+ ("keyboard.press", ("Escape",), {}),+ ("locator(#seg).press", ("Backspace",), {}),+ ("page.select_option", ("#s",), {"value": ["a"], "timeout": 100, "force": False}),+ ("mouse.wheel", (0, -800), {}),+ ]+++async def test_control_clicks_an_element_handle_with_only_the_arguments_given(control: SkyvernContext) -> None:+ handle = MagicMock(spec=ElementHandle)+ handle.click = AsyncMock()++ await input_dispatch.click_handle(_FakePage(), handle)++ handle.click.assert_awaited_once_with()+++async def test_treatment_routes_pointer_typing_and_scroll_through_the_registered_strategies(+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]+) -> None:+ await input_dispatch.click(page, "#go", timeout=1234)+ await input_dispatch.click(page, "#go", timeout=5000, force=True)+ await input_dispatch.click_at(page, 10.0, 20.0)+ await input_dispatch.hover(page, "#menu", timeout=2000)+ await input_dispatch.clear(page, "#f", timeout=100)+ await input_dispatch.wheel(page, 0, 800)++ assert _names(page.calls) == [+ "cursor.click",+ # The strategy has no forced click: it moves there, then Playwright clicks exactly as before.+ "cursor.move_to_element",+ "page.click",+ "cursor.move_to",+ "mouse.click",+ "cursor.move_to_element",+ "page.hover",+ "input.clear_field",+ "scroll.scroll_by",+ ]+ assert page.calls[2] == ("page.click", ("#go",), {"timeout": 5000, "force": True})+++async def test_treatment_clicks_a_locator_through_the_strategy_and_a_handle_after_a_cursor_move(+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]+) -> None:+ await input_dispatch.click(page, page.locator("#seg"), timeout=2000)+ await input_dispatch.click_handle(page, _FakeHandle(page), position={"x": 1.0, "y": 2.0}, timeout=1500)++ assert _names(page.calls) == ["cursor.click", "cursor.move_to", "handle.click"]+ assert page.calls[1:] == [+ ("cursor.move_to", (12.0, 23.0), {}),+ ("handle.click", (), {"position": {"x": 1.0, "y": 2.0}, "timeout": 1500}),+ ]+++async def test_treatment_types_through_the_keyboard_after_one_focus(+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]+) -> None:+ await input_dispatch.type_keys(page, "#f", "abc", delay=15, timeout=900)++ # The strategy gets no locator (given one, a humanized keyboard re-focuses the field before every key) and no+ # delay: as in v1, the registered strategy owns the cadence.+ assert page.calls == [+ ("page.focus", ("#f",), {"timeout": 900}),+ ("input.type_text", (None, "abc"), {"delay": None}),+ ]+++ ("replace", "head_write"),+ [+ (True, ("page.fill", ("#f", "Hello, "), {"timeout": 900})),+ (False, ("keyboard.insert_text", ("Hello, ",), {})),+ ],+ ids=["emptied_field_fills_the_head", "append_inserts_the_head_at_the_caret"],+)+async def test_treatment_humanizes_only_the_last_ten_characters_as_v1_does(+ treatment: SkyvernContext,+ page: _FakePage,+ humanized_strategies: list[Call],+ replace: bool,+ head_write: Call,+) -> None:+ await input_dispatch.type_keys(page, "#f", "Hello, 0123456789", delay=15, timeout=900, replace=replace)++ assert page.calls[-2:] == [head_write, ("input.type_text", (None, "0123456789"), {"delay": None})]++[email protected]("arm", ["control", "treatment"])+async def test_a_secret_is_typed_key_by_key_exactly_as_before_in_both_arms(+ page: _FakePage, humanized_strategies: list[Call], arm: RunArm+) -> None:+ # Split one-time-code boxes advance on each keystroke, so neither a fill nor a humanized keyboard may take over.+ for _ in _pin(arm):+ await input_dispatch.type_keys(page, "#code", "s3cret", delay=15, timeout=900, secret=True, replace=True)+ await input_dispatch.type_keys(page, page.locator("#box"), "7", timeout=1000, secret=True)++ assert page.calls == [+ ("page.type", ("#code", "s3cret"), {"delay": 15, "timeout": 900}),+ ("locator(#box).press_sequentially", ("7",), {"timeout": 1000}),+ ]+++async def test_treatment_approach_moves_then_waits_for_the_page_to_settle(+ treatment: SkyvernContext, page: _FakePage, humanized_strategies: list[Call]+) -> None:+ await input_dispatch.approach(page, "#go")++ assert _names(page.calls) == ["locator(#go).scroll_into_view_if_needed", "cursor.move_to_element", "page.evaluate"]+++def test_arm_fields_name_the_arm_whether_it_is_in_effect_and_the_registered_profile(+ treatment: SkyvernContext, humanized_strategies: list[Call]+) -> None:+ assert input_dispatch.arm_fields() == {+ "humanized_input_arm": "treatment",+ "humanized_input_in_effect": True,+ "global_event_strategy_profile": "default",+ }+++def test_registered_profile_tells_no_registration_from_an_unnamed_one() -> None:+ EventStrategyFactory.reset()+ try:+ assert EventStrategyFactory.registered_profile() == "none"+ EventStrategyFactory.set_cursor_strategy(_RecordingCursor([]))+ assert EventStrategyFactory.registered_profile() == "unrecorded"+ EventStrategyFactory.record_profile("kernel")+ assert EventStrategyFactory.registered_profile() == "kernel"+ finally:+ EventStrategyFactory.reset()+ assert EventStrategyFactory.registered_profile() == "none"diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex ac7f2995b..253569be1 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -6,9 +6,11 @@ operations (no task-ecosystem) with the right args, without a live browser.from __future__ import annotations+import astimport asyncioimport contextlibimport html+import inspectimport itertoolsimport jsonimport os@@ -42,6 +44,8 @@ from skyvern.exceptions import NO_ADDRESS_RECORD_NAV_ERROR_CODEfrom skyvern.forge import appfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.event.default import DefaultInputStrategy+from skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.forge.sdk.services import credentials as credentials_modulefrom skyvern.forge.sdk.workflow.context_manager import WorkflowContextManager, WorkflowRunContextfrom skyvern.forge.sdk.workflow.models.block import _recorded_task_nav_error_codes@@ -68,6 +72,7 @@ from skyvern.forge.taskv3.tools import (_MENU_OPTION_TEXTS_JS,_OPAQUE_ID_RUN_RE,_SEMANTIC_COMMIT_STATE_JS,+ _TYPE_TARGET_PROBE_JS,NAVIGATION_DEAD_END_STATUSES,OBSERVE_DISPLAY_WIDTHS,OBSERVE_RETAIN_WIDTH_MIN,@@ -20809,6 +20814,112 @@ async def test_covered_error_message_when_the_occluding_layer_has_no_controls_atassert "no controls were found on it" in r.content, r.content+class _CoveredProbeFakePage(_TypeaheadFakePage):+ """Answers the type-target probe with a fixed covered reading, so the message the typing path+ builds from it is checked without a browser."""++ def __init__(self, occluder: dict[str, Any]) -> None:+ super().__init__(field_type="email")+ self._occluder = occluder++ async def evaluate(self, js: str, arg: Any = None) -> Any:+ if js == _TYPE_TARGET_PROBE_JS:+ return {"exists": True, "occluded": True, "occluder": self._occluder}+ return await super().evaluate(js, arg)+++# The probe sets `ownContainer` when the named layer is an ancestor of the field and the element taking+# the pointer is not itself interactive: a sticky header holding the input, with a transparent+# click-catcher span over it that the probe stamps `data-tv3-catcher`.+_OWN_CONTAINER_OCCLUDER = {+ "selector": "#header",+ "name": "People Phone Search",+ "controls": [{"selector": "#tab-people", "label": "People"}],+ "truncated": False,+ "layerKind": "qualified",+ "ownContainer": True,+}+++async def test_a_cover_inside_the_fields_own_container_names_the_stamped_catcher_to_click() -> None:+ page = _CoveredProbeFakePage(_OWN_CONTAINER_OCCLUDER)+ taskv3_loop._COVERED_LAYER.set(None)+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(+ {"selector": "#name", "text": "Jane Doe"}+ )+ assert r.error_class == "covered", r.content+ assert 'Click [data-tv3-catcher="1"] to activate the field, then retry #name.' in r.content, r.content+ assert "Do not try to dismiss it" in r.content, r.content+ assert "closes or dismisses" not in r.content, r.content+ assert not any(call[0] in ("fill", "type") for call in page.calls), page.calls+ recorded = taskv3_loop._COVERED_LAYER.get() or {}+ assert recorded == {"branch": "own_container", "controls": 0, "layer_kind": "qualified"}, recorded+++async def test_a_cover_without_the_own_container_flag_keeps_the_dismissal_message() -> None:+ occluder = {**_OWN_CONTAINER_OCCLUDER, "selector": "#banner", "name": "Accept cookies"}+ del occluder["ownContainer"]+ page = _CoveredProbeFakePage(occluder)+ taskv3_loop._COVERED_LAYER.set(None)+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(+ {"selector": "#name", "text": "Jane Doe"}+ )+ assert r.content == (+ '#name is covered by "Accept cookies" (#banner), so it cannot be typed into — a person could not '+ 'click it either. Its controls: #tab-people "People". Pick whichever one actually closes or '+ "dismisses the layer, then retry #name."+ ), r.content+ recorded = taskv3_loop._COVERED_LAYER.get() or {}+ assert recorded == {"branch": "named", "controls": 1, "layer_kind": "qualified"}, recorded+++async def test_a_transparent_own_container_still_names_the_catcher_rather_than_a_ghost_cover() -> None:+ page = _CoveredProbeFakePage({**_OWN_CONTAINER_OCCLUDER, "controls": [], "invisible": True})+ taskv3_loop._COVERED_LAYER.set(None)+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(+ {"selector": "#name", "text": "Jane Doe"}+ )+ assert 'Click [data-tv3-catcher="1"] to activate the field, then retry #name.' in r.content, r.content+ assert (taskv3_loop._COVERED_LAYER.get() or {}).get("branch") == "own_container"+++_STICKY_FORM_HTML = """+<div id="header" style="position:sticky;top:0;width:400px;height:140px;background:#eee">+ <form><input id="name" type="text" style="display:block;margin:4px;width:200px">{over}</form>+</div>+"""+_SIBLING_BANNER_HTML = """+<input id="name" type="text" style="position:absolute;top:100px;left:100px;width:150px;height:30px">+<div id="banner" style="position:fixed;top:90px;left:90px;width:300px;height:50px;z-index:10;background:#f88">+ Accept cookies <button id="accept">Accept</button>+</div>+"""+++@_skip_no_browser+async def test_the_real_probe_names_a_non_interactive_catcher_but_not_a_submit_button_or_a_banner() -> None:+ catcher = '<span id="catcher" style="position:absolute;inset:0;z-index:1"></span>'+ submit = '<button id="go" type="submit" style="position:absolute;left:0;top:0;width:220px;height:40px">Go</button>'+ async with _content_page(_STICKY_FORM_HTML.format(over=catcher)) as page:+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(+ {"selector": "#name", "text": "Jane Doe"}+ )+ assert 'Click [data-tv3-catcher="1"] to activate the field' in r.content, r.content+ assert await page.locator('[data-tv3-catcher="1"]').evaluate("n => n.id") == "catcher"+ for markup in (_STICKY_FORM_HTML.format(over=submit), _SIBLING_BANNER_HTML):+ async with _content_page(markup) as page:+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "type").handler(+ {"selector": "#name", "text": "Jane Doe"}+ )+ assert r.error_class == "covered", r.content+ assert "Pick whichever one actually closes or dismisses the layer" in r.content, r.content+ assert await page.locator("[data-tv3-catcher]").count() == 0++# A cover that qualifies as NOTHING: not pinned, no layer role, no aria-modal, not view-sized, and# not an ancestor of the field. The walk finds no layer and names the hit element itself, which is# the production shape behind most zero-control refusals -- an option row or a value cell, which has@@ -35362,9 +35473,35 @@ def _straddle_renders() -> list[tuple[str, int, Callable[[str], str]]]:[{"n": 1, "text": "Jane Doe", "vals": [f"data-code={t}"]}, {"n": 2, "text": "Jane Doe"}]),),+ (+ "unproven-row",+ 60,+ lambda t: taskv3_tools._unproven_row_error(+ "John", "#who", "scrolled to the end", {"n": 1, "text": t}+ ).content,+ ),+ (+ "selection-report",+ taskv3_tools.SELECTION_REPORT_OPTION_WIDTH,+ lambda t: taskv3_tools._selection_report([t, "B"]),+ ),]+def test_every_row_taking_error_constructor_is_in_the_straddle_table() -> None:+ # A new refusal that renders page rows must join the table above, or its mask is never checked.+ tree = ast.parse(Path(taskv3_tools.__file__).read_text())+ constructors = {+ f.name+ for f in tree.body+ if isinstance(f, ast.FunctionDef)+ and f.name.endswith("_error")+ and {"row", "rows"} & {a.arg for a in f.args.args + f.args.kwonlyargs}+ }+ table = inspect.getsource(_straddle_renders)+ assert constructors and all(name in table for name in constructors), sorted(constructors)++@pytest.mark.parametrize(("cap", "render"), [pytest.param(c, r, id=i) for i, c, r in _straddle_renders()])def test_a_row_listing_masks_a_hidden_value_before_any_cut_so_no_fragment_of_it_survives(cap: int, render: Callable[[str], str]@@ -35461,6 +35598,38 @@ async def test_dom_a_typeahead_suggestion_masks_a_hidden_value_straddling_its_diskyvern_context.reset()+@_skip_no_browser[email protected]("shape", ["twin_far_down_the_list", "list_search_box"])+async def test_dom_a_select_combobox_refusal_masks_a_hidden_value_straddling_its_row_cap(shape: str) -> None:+ # The refusals select_combobox builds inline name the rows that made the value ambiguous, each cut to 60.+ secret = "Qz7Kp4Wm9Xr2Vt6Ny3Lb8Hc5"+ fragments = {secret[:k] for k in range(4, len(secret) + 1)}+ ctx = SkyvernContext(organization_id="o_1")+ ctx.register_secret_value(secret, hide_from_model=True)+ skyvern_context.set(ctx)+ try:+ for start in (60 - len(secret) + 4, 60 - len(secret) // 2, 60 - 4):+ label = "North " + "." * (start - len("North ")) + secret+ rows = [(f"Row {i:03d}", f"+{3000 + i}") for i in range(1, 71)]+ if shape == "twin_far_down_the_list":+ rows[5], rows[60] = (label, "+9001"), (label, "+9002")+ html = _cc_widget_html(rows, 30)+ else:+ rows[10], rows[50] = (label, "+9010"), ("North Dakota", "+9050")+ html = _cc_widget_html([*rows, ("", "")], 12, search_box=True)+ async with _content_page(html) as page:+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "select_combobox").handler(+ {"selector": "#cc", "value": "North"}+ )+ shown = ctx.hide_from_model(r.content)+ assert r.status == "error", (start, shown)+ leaked = sorted(f for f in fragments if f in shown)+ assert not leaked, (start, leaked, shown)+ finally:+ skyvern_context.reset()++@pytest.mark.asyncioasync def test_get_html_offset_normalization_survives_the_selector_guards_arg_copy() -> None:# `_with_selector_guard` hands the handler a COPY of args whenever a selector is present, and the@@ -37681,3 +37850,26 @@ async def test_the_failed_read_count_resets_on_a_success_and_on_navigation(monkeawait page.goto("http://shop.test/review?again=1")after = await act(1)assert after[4] is None and after[1] == 0, after+++@_skip_no_browser+async def test_an_appended_credential_never_reaches_the_humanized_keyboard(monkeypatch: pytest.MonkeyPatch) -> None:+ class _RefusingInput(DefaultInputStrategy):+ async def type_text(self, *args: Any, **kwargs: Any) -> None:+ raise AssertionError("a resolved credential was sent through the registered input strategy")++ monkeypatch.setattr(settings, "TASK_V3_HUMANIZED_INPUT", True)+ EventStrategyFactory.set_input_strategy(_RefusingInput())+ try:+ async with _content_page('<!doctype html><html><body><input id="t" value="id-"></body></html>') as page:+ tools = build_browser_tools(+ _fixed_page_provider(page),+ resolve_typed_text=lambda text: "real-secret" if text == "placeholder_abc" else text,+ )+ r = await _tool(tools, "type").handler({"selector": "#t", "text": "placeholder_abc", "clear": False})+ held = await page.evaluate("() => document.getElementById('t').value")+ finally:+ EventStrategyFactory.reset()+ assert r.status == "ok", r.content+ assert held == "id-real-secret"