Skip to content

Automate browser based workflows with AI

Resolve Workday education and certification fields from the profile keys production sends (#8241)

Trevor Cheung888348d56dc4
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index a8a9b5ee5..363fa079f 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -182,15 +182,21 @@ from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
Action,
ActionStatus,
+ ClickAction,
CompleteAction,
CompleteVerifyResult,
DecisiveAction,
DownloadFileAction,
ExtractAction,
GotoUrlAction,
+ HoverAction,
+ InputTextAction,
KeypressAction,
ReloadPageAction,
+ SelectOption,
+ SelectOptionAction,
TerminateAction,
+ UploadFileAction,
VerificationStatus,
WaitAction,
WebAction,
@@ -281,6 +287,64 @@ def mark_pdf_source_downloaded(pdf_src: str) -> None:
context.downloaded_pdf_sources.add(hashlib.sha256(pdf_src.encode()).hexdigest())
+_TASKV3_TOOL_ACTION_TYPES = {
+ "click": ActionType.CLICK,
+ "hover": ActionType.HOVER,
+ "type": ActionType.INPUT_TEXT,
+ "select_option": ActionType.SELECT_OPTION,
+ "select_combobox": ActionType.SELECT_OPTION,
+ "press_key": ActionType.KEYPRESS,
+ "file_upload": ActionType.UPLOAD_FILE,
+ "navigate": ActionType.GOTO_URL,
+ "scroll": ActionType.SCROLL,
+ "wait": ActionType.WAIT,
+ "solve_captcha": ActionType.SOLVE_CAPTCHA,
+}
+
+
+def _redact_tool_args(args: dict[str, Any], secret_values: set[str]) -> dict[str, Any]:
+ if not secret_values:
+ return args
+ return {key: _redact_tool_arg(value, secret_values) for key, value in args.items()}
+
+
+def _redact_tool_arg(value: Any, secret_values: set[str]) -> Any:
+ # Tool args are bare json.loads output and the builder stringifies scalars, so numbers and nested
+ # values are matched as text too (bool/None are left alone). Selector/url are scrubbed as well: a
+ # persisted element_id that diverges from the live action beats a secret at rest in the row.
+ if isinstance(value, dict):
+ return {key: _redact_tool_arg(item, secret_values) for key, item in value.items()}
+ if isinstance(value, list):
+ return [_redact_tool_arg(item, secret_values) for item in value]
+ if isinstance(value, (str, int, float)) and not isinstance(value, bool):
+ return redact_secrets_from_text(str(value), secret_values, boundary_all_lengths=True)
+ return value
+
+
+def _taskv3_action_for_tool_call(tool_name: str, args: dict[str, Any], **fields: Any) -> Action:
+ """Build the Action a v3 tool call persists as, carrying the fields its typed subclass requires so
+ the row hydrates as that subclass instead of falling back to base Action on every read."""
+ selector = str(args.get("selector") or "")
+ if tool_name == "click":
+ return ClickAction(element_id=selector, **fields)
+ if tool_name == "hover":
+ return HoverAction(element_id=selector, **fields)
+ if tool_name == "type":
+ return InputTextAction(element_id=selector, text=str(args.get("text") or ""), **fields)
+ if tool_name in ("select_option", "select_combobox"):
+ label, value = args.get("label"), args.get("value")
+ option = SelectOption(label=None if label is None else str(label), value=None if value is None else str(value))
+ return SelectOptionAction(element_id=selector, option=option, **fields)
+ if tool_name == "press_key":
+ key = args.get("key")
+ return KeypressAction(keys=[str(key)] if key else [], **fields)
+ if tool_name == "file_upload":
+ return UploadFileAction(element_id=selector, file_url=str(args.get("file") or ""), **fields)
+ if tool_name == "navigate":
+ return GotoUrlAction(url=str(args.get("url") or ""), **fields)
+ return Action(action_type=_TASKV3_TOOL_ACTION_TYPES.get(tool_name, ActionType.CLICK), **fields)
+
+
class _PromptCeilingExceeded(Exception):
"""Internal signal: the cached split-template prompt blew past the
PROMPT_HARD_CEILING_TOKENS budget. Raised inside the cached extract-action
@@ -1243,19 +1307,6 @@ class ForgeAgent:
# action round, and we persist one DB row per action + one screenshot per round, so the Task
# API's action_screenshot_urls and GET /tasks/{id}/actions are populated for v3. Additive and
# best-effort — billing still meters off outcome.billable_actions below.
- _tool_action_types = {
- "click": ActionType.CLICK,
- "hover": ActionType.HOVER,
- "type": ActionType.INPUT_TEXT,
- "select_option": ActionType.SELECT_OPTION,
- "select_combobox": ActionType.SELECT_OPTION,
- "press_key": ActionType.KEYPRESS,
- "file_upload": ActionType.UPLOAD_FILE,
- "navigate": ActionType.GOTO_URL,
- "scroll": ActionType.SCROLL,
- "wait": ActionType.WAIT,
- "solve_captcha": ActionType.SOLVE_CAPTCHA,
- }
_billable_tool_names = frozenset(
{"click", "hover", "type", "select_option", "select_combobox", "press_key", "file_upload"}
)
@@ -1272,11 +1323,16 @@ class ForgeAgent:
)
except Exception:
LOG.warning("task_v3 failed to capture post-action screenshot", task_id=task.task_id, exc_info=True)
+ secret_values: set[str] = set()
+ if app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled(task.workflow_run_id):
+ secret_values = app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run(task.workflow_run_id)
for name, args, succeeded in round_actions:
try:
- selector = args.get("selector", "") if isinstance(args, dict) else ""
- action = Action(
- action_type=_tool_action_types.get(name, ActionType.CLICK),
+ tool_args = _redact_tool_args(args if isinstance(args, dict) else {}, secret_values)
+ selector = tool_args.get("selector", "")
+ action = _taskv3_action_for_tool_call(
+ name,
+ tool_args,
status=ActionStatus.completed if succeeded else ActionStatus.failed,
organization_id=task.organization_id,
workflow_run_id=task.workflow_run_id,
@@ -1459,11 +1515,11 @@ class ForgeAgent:
extracted_information = _redact_extracted_information(extracted_information, secret_values)
# Emit one action-result per billable browser action so the per-step billing hook meters a
# task_v3 run per action, exactly like the step engine — no billing-model change. (Reuses the
- # _tool_action_types map defined above for per-action persistence.)
+ # _TASKV3_TOOL_ACTION_TYPES map for per-action persistence.)
billable_action_results: list[tuple[Action, list[ActionResult]]] = [
(
Action(
- action_type=_tool_action_types.get(name, ActionType.CLICK),
+ action_type=_TASKV3_TOOL_ACTION_TYPES.get(name, ActionType.CLICK),
task_id=task.task_id,
step_id=step.step_id,
description=f"task_v3 {name}",
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index e30507d42..cb9995ef7 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -32,6 +32,11 @@ PageProvider = Callable[[], Awaitable[Any]]
PAGE_UNAVAILABLE_ERROR = "browser page unavailable"
+# The exact selector shapes our own enrichment mints: data-tv3 by observe(), data-tv3-menu by the
+# click menu probe. Either exists only where we set it, so one that matches nothing now cannot
+# reappear without a fresh observe / menu-opening click.
+_TV3_MARKER_SELECTOR_RE = re.compile(r'^\[data-tv3(?:-menu)?="[^"\\]+"\]$')
+
# ARIA combobox signals — used by observe() only to add a hint that a field is a typeahead. This is a
# nudge for the model, not load-bearing: type() handles typeaheads behaviorally (see _FIND_SUGGESTION_JS),
# so a field with no ARIA (a plain <input> backed by a custom dropdown) is still handled correctly.
@@ -158,6 +163,180 @@ _VERIFY_COMMIT_JS = r"""(args) => {
return '';
}"""
+# Whether a selector currently resolves. `true` on a broken selector: existence is only ever used to
+# soften/enrich behavior, so an unparseable selector must take the normal (unenriched) path.
+_SELECTOR_EXISTS_JS = r"""(sel) => { try { return !!document.querySelector(sel); } catch (e) { return true; } }"""
+
+# Pre-click state for the dropdown-commit path: whether a click-opened menu (rows tagged
+# data-tv3-menu by _FIND_MENU_JS) is currently open, whether the click target IS one of its rows, and
+# that row's state fingerprint — aria checked/selected/pressed, class, child count, text — so an option
+# click on a multi-select menu (which commits WITHOUT closing) can be verified by its state change.
+# Also takes the visible-DOM pre-snapshot (data-tv3-pre) so a menu the click opens reads as a reaction.
+_CLICK_PRECHECK_JS = r"""(clicked) => {
+ const vis = (el) => {
+ const r = el.getBoundingClientRect();
+ if (r.width === 0 || r.height === 0) return false;
+ // A menu mid-close (opacity fade, pointer-events cut) still has a nonzero rect; reading it as
+ // "open" would turn a healthy committed selection into a false "did not commit" error.
+ try {
+ const s = getComputedStyle(el);
+ if (s.visibility === 'hidden' || Number(s.opacity) < 0.05 || s.pointerEvents === 'none') return false;
+ } catch (e) {}
+ return true;
+ };
+ const state = (el) => {
+ // .checked is a DOM property, not an attribute: a native-checkbox multi-select commits by
+ // flipping only it, with no aria/class/text change. Same for inline-style-only toggles.
+ let kids = '';
+ try { for (const i of el.querySelectorAll('input')) kids += i.checked ? '1' : '0'; } catch (e) {}
+ return [
+ el.getAttribute('aria-checked'), el.getAttribute('aria-selected'), el.getAttribute('aria-pressed'),
+ el.className, el.children.length, (el.innerText || '').trim(),
+ el.getAttribute('style'), kids,
+ ].join('|');
+ };
+ const openRows = [];
+ for (const el of document.querySelectorAll('[data-tv3-menu]')) if (vis(el)) openRows.push(el);
+ let target = null;
+ try { target = document.querySelector(clicked); } catch (e) { target = null; }
+ let isOption = false;
+ let containsMenu = false;
+ let optText = '';
+ let optState = '';
+ if (target && openRows.length) {
+ for (const el of openRows) {
+ // The target being the row or inside it is an option pick. The target merely CONTAINING rows
+ // (the card around the menu) is not — and since a center-point click on the card can land on
+ // an arbitrary row, that case is flagged so the handler makes no claims about it at all.
+ if (el === target || el.contains(target)) {
+ isOption = true;
+ optText = (el.innerText || '').trim().slice(0, 80);
+ optState = state(el);
+ break;
+ }
+ if (target.contains(el)) containsMenu = true;
+ }
+ }
+ document.querySelectorAll('[data-tv3-pre]').forEach((e) => e.removeAttribute('data-tv3-pre'));
+ for (const el of document.querySelectorAll('body *')) if (vis(el)) el.setAttribute('data-tv3-pre', '1');
+ return { menuOpen: openRows.length > 0, isOption, containsMenu, optText, optState };
+}"""
+
+# Post-click menu state: how many previously-tagged menu rows are still visible (a closed menu — nodes
+# destroyed or hidden — reads 0), plus the clicked row's current state fingerprint for the multi-select
+# commit check. Field names are distinct from _CLICK_PRECHECK_JS's on purpose (tests dispatch on them).
+_MENU_AFTER_JS = r"""(clicked) => {
+ const vis = (el) => {
+ const r = el.getBoundingClientRect();
+ if (r.width === 0 || r.height === 0) return false;
+ try {
+ const s = getComputedStyle(el);
+ if (s.visibility === 'hidden' || Number(s.opacity) < 0.05 || s.pointerEvents === 'none') return false;
+ } catch (e) {}
+ return true;
+ };
+ const state = (el) => {
+ // .checked is a DOM property, not an attribute: a native-checkbox multi-select commits by
+ // flipping only it, with no aria/class/text change. Same for inline-style-only toggles.
+ let kids = '';
+ try { for (const i of el.querySelectorAll('input')) kids += i.checked ? '1' : '0'; } catch (e) {}
+ return [
+ el.getAttribute('aria-checked'), el.getAttribute('aria-selected'), el.getAttribute('aria-pressed'),
+ el.className, el.children.length, (el.innerText || '').trim(),
+ el.getAttribute('style'), kids,
+ ].join('|');
+ };
+ let stillOpen = 0;
+ const rows = [];
+ for (const el of document.querySelectorAll('[data-tv3-menu]')) if (vis(el)) { stillOpen++; rows.push(el); }
+ let target = null;
+ try { target = document.querySelector(clicked); } catch (e) { target = null; }
+ let optState = '';
+ if (target) {
+ for (const el of rows) {
+ if (el === target || el.contains(target)) { optState = state(el); break; }
+ }
+ }
+ return { stillOpen, optState };
+}"""
+
+# Behavioral, site-agnostic menu finder: after a click (with the pre-snapshot taken first),
+# look for the option list the page rendered IN REACTION — a NEW container (not data-tv3-pre: a
+# pre-existing visible container whose rows merely changed, e.g. pagination refreshing a results list,
+# is never a menu) holding >=2 new, visible, row-sized, mostly-clickable leaf rows, positioned adjacent
+# to the clicked element. Keys off reaction + geometry — no CSS-class/ARIA/site vocabulary — mirroring
+# _FIND_SUGGESTION_JS. Unlike the typeahead finder this only REPORTS (the model does the clicking), so
+# navigational rows are listed too. Tags rows data-tv3-menu="1..N" (top-to-bottom) — in-DOM tags that
+# stay valid until the menu re-renders, so the model can pick an option without a re-observe re-minting
+# ids (the staging trace's staleness trap). Existing tags are cleared only when a new menu is tagged.
+_FIND_MENU_JS = r"""(clicked) => {
+ const vis = (r) => r.width > 0 && r.height > 0;
+ let trigger = null;
+ try { trigger = document.querySelector(clicked); } catch (e) { return null; }
+ if (!trigger) return null;
+ const tr = trigger.getBoundingClientRect();
+ const rows = [];
+ for (const el of document.querySelectorAll('body *')) {
+ if (el.hasAttribute('data-tv3-pre')) continue;
+ const tag = el.tagName;
+ if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'SCRIPT' || tag === 'STYLE' || tag === 'LABEL' || tag === 'FORM') continue;
+ if (el.children.length > 8) continue;
+ const r = el.getBoundingClientRect();
+ if (!vis(r) || r.height > 90) continue;
+ const txt = (el.innerText || '').trim();
+ if (!txt || txt.length > 80) continue;
+ // Options are individually actionable rows. Requiring it per-row keeps a dialog's title/body
+ // text from being listed as "options" (and a horizontal Confirm/Cancel button pair then fails
+ // the stacked-rows check below).
+ const role = el.getAttribute('role');
+ let ptr = false;
+ try { ptr = getComputedStyle(el).cursor === 'pointer'; } catch (e) { ptr = false; }
+ const clickable = tag === 'BUTTON' || tag === 'A' || role === 'option' || role === 'menuitem' || role === 'menuitemcheckbox' || role === 'menuitemradio' || ptr;
+ if (!clickable) continue;
+ rows.push({ el, r, txt });
+ }
+ if (rows.length < 2) return null;
+ const leaves = rows.filter((c) => !rows.some((o) => o.el !== c.el && c.el.contains(o.el)));
+ if (leaves.length < 2) return null;
+ // Group by parent AND grandparent so both flat menus (card > button*N) and nested ones
+ // (ul > li > button) find their shared container.
+ const groups = new Map();
+ for (const c of leaves) {
+ const p1 = c.el.parentElement;
+ for (const p of [p1, p1 ? p1.parentElement : null]) {
+ if (!p || p === document.body || p === document.documentElement) continue;
+ if (!groups.has(p)) groups.set(p, new Set());
+ groups.get(p).add(c);
+ }
+ }
+ let best = null;
+ for (const [p, set] of groups) {
+ const g = Array.from(set);
+ if (g.length < 2) continue;
+ if (p.hasAttribute('data-tv3-pre')) continue;
+ // A dialog is a page mode, not a menu — mislabeling it invites a wrong "pick an option" move.
+ try { if (p.closest('dialog,[role~="dialog"],[aria-modal="true"]')) continue; } catch (e) {}
+ const pr = p.getBoundingClientRect();
+ if (!vis(pr) || pr.height > 500) continue;
+ if (pr.top < tr.top - 200 || pr.top > tr.bottom + 400) continue;
+ if (pr.right < tr.left - 100 || pr.left > tr.right + 100) continue;
+ const tops = new Set(g.map((c) => Math.round(c.r.top)));
+ if (tops.size < 2) continue;
+ if (!best || g.length > best.g.length || (g.length === best.g.length && pr.height < best.h)) best = { p, g, h: pr.height };
+ }
+ if (!best) return null;
+ document.querySelectorAll('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
+ best.g.sort((a, b) => a.r.top - b.r.top || a.r.left - b.r.left);
+ const options = [];
+ let n = 0;
+ for (const c of best.g) {
+ n++;
+ c.el.setAttribute('data-tv3-menu', String(n));
+ if (options.length < 15) options.push({ n, text: c.txt.slice(0, 60) });
+ }
+ return { count: n, options };
+}"""
+
# Raw DOM perception: collect visible interactive elements with a stable selector each.
# Elements without a natural selector get a data-tv3 marker so later actions can target them.
_OBSERVE_JS = (
@@ -285,6 +464,21 @@ _OBSERVE_JS = (
)
+def _menu_open_note(found: dict[str, Any], selector: str) -> str:
+ count = int(found.get("count") or 0)
+ parts = []
+ for o in (found.get("options") or [])[:15]:
+ # option texts are page-controlled and land in the LLM transcript — same sanitation as filenames
+ text = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", str(o.get("text", "")))
+ parts.append(f'[data-tv3-menu="{o.get("n")}"] {text!r}')
+ overflow = f" (+{count - len(parts)} more — re-observe for the full list)" if count > len(parts) else ""
+ return (
+ f"This click opened a menu of {count} options: {'; '.join(parts)}{overflow}. To select one, click "
+ f'its [data-tv3-menu="N"] selector NOW — clicking {selector} again or elsewhere closes the menu '
+ "and destroys these options."
+ )
+
+
def _spec(
name: str, description: str, params: dict[str, Any], handler: Callable[[dict[str, Any]], Awaitable[ToolResult]]
) -> ToolSpec:
@@ -404,17 +598,170 @@ def build_browser_tools(
html = ""
else:
html = await page.content()
+ # The click/type reaction gate stamps data-tv3-pre on every visible element; it is internal
+ # bookkeeping, and left in place it costs a third of the truncation budget below in noise.
+ html = html.replace(' data-tv3-pre="1"', "")
if len(html) > 20000:
return ToolResult.ok(html[:20000] + "…[truncated at 20000 chars]")
return ToolResult.ok(html)
+ async def _click_reaction(
+ page: Any, selector: str, pre: dict[str, Any], url_before: str
+ ) -> tuple[str | None, str | None]:
+ # Returns (note, commit_error) — at most one set. Raises are the caller's to swallow (fail-open:
+ # a probe failure must degrade to the bare pre-feature ok, never fail the click).
+ opt = _DOWNLOAD_NOTICE_SANITIZE_RE.sub("", str(pre.get("optText") or "")) or selector
+ if pre.get("isOption"):
+ # Commit evidence, any one suffices: navigation, the menu closing, the option's own state
+ # changing vs the post-hover baseline (multi-select menus commit WITHOUT closing), or a
+ # submenu opening (a cascading option commits nothing yet — reporting the child menu beats
+ # a false error).
+ baseline = pre.get("optState") or ""
+
+ def _committed_state(after: dict[str, Any]) -> bool:
+ return bool(after.get("optState")) and after.get("optState") != baseline
+
+ async def _state_holds(state: str) -> bool:
+ # A real commit settles into its new state; self-updating row content (a countdown,
+ # a live price) keeps moving. Only a state that holds across two reads is evidence.
+ await asyncio.sleep(0.15)
+ try:
+ again_raw = await page.evaluate(_MENU_AFTER_JS, selector)
+ except Exception:
+ return True
+ again = again_raw if isinstance(again_raw, dict) else {}
+ return bool(again.get("optState") == state)
+
+ url_now = await _url(page)
+ if url_before and url_now and url_now != url_before:
+ return f"Selected option {opt!r} — the page navigated.", None
+ try:
+ after_raw = await page.evaluate(_MENU_AFTER_JS, selector)
+ except Exception:
+ # the page tearing down right after an option click is a navigation — commit evidence
+ return f"Selected option {opt!r} — the page navigated.", None
+ after = after_raw if isinstance(after_raw, dict) else {}
+ if not after.get("stillOpen"):
+ return f"Selected option {opt!r} — the menu closed.", None
+ if _committed_state(after) and await _state_holds(after.get("optState") or ""):
+ return f"Selected option {opt!r} — its state changed (the menu stayed open).", None
+ # Menus routinely close through a fade or an async server ack; declaring "did not commit"
+ # off the instantaneous read would turn those healthy commits into false errors. One
+ # bounded settle, only on this would-be-error path.
+ await asyncio.sleep(0.6)
+ try:
+ settled_raw = await page.evaluate(_MENU_AFTER_JS, selector)
+ except Exception:
+ return f"Selected option {opt!r} — the page navigated.", None
+ settled = settled_raw if isinstance(settled_raw, dict) else {}
+ if not settled.get("stillOpen"):
+ return f"Selected option {opt!r} — the menu closed.", None
+ if _committed_state(settled) and await _state_holds(settled.get("optState") or ""):
+ return f"Selected option {opt!r} — its state changed (the menu stayed open).", None
+ # No-commit evidence is already established: a crash of this last informational probe must
+ # not fall through to the caller's fail-open bare ok.
+ try:
+ submenu = await page.evaluate(_FIND_MENU_JS, selector)
+ except Exception:
+ submenu = None
+ if isinstance(submenu, dict) and submenu.get("count"):
+ return _menu_open_note(submenu, selector), None
+ return None, (
+ f"clicked option {opt!r} ({selector}) but the selection did not commit — the menu is "
+ "still open and unchanged. Do not repeat this click; press Enter on the option, or "
+ "re-observe and try a different control."
+ )
+ if pre.get("menuOpen"):
+ if pre.get("containsMenu"):
+ # Clicked the card AROUND the menu: the center-point click may have landed on an
+ # arbitrary row, so any open/closed/selected claim could be false. Say nothing.
+ return None, None
+ try:
+ after_raw = await page.evaluate(_MENU_AFTER_JS, selector)
+ except Exception:
+ return None, None
+ found = await page.evaluate(_FIND_MENU_JS, selector)
+ if isinstance(found, dict) and found.get("count"):
+ return _menu_open_note(found, selector), None
+ if isinstance(after_raw, dict) and not after_raw.get("stillOpen"):
+ return (
+ "Note: this click CLOSED the open menu — no option was selected. To select, click "
+ 'an option\'s [data-tv3-menu="N"] selector while the menu is open.'
+ ), None
+ return None, None
+ found = await page.evaluate(_FIND_MENU_JS, selector)
+ if isinstance(found, dict) and found.get("count"):
+ return _menu_open_note(found, selector), None
+ return None, None
+
async def click(args: dict[str, Any]) -> ToolResult:
page, error = await _resolve_page()
if error is not None:
return error
selector = args["selector"]
- await page.click(selector, timeout=15000)
- return ToolResult.ok(f"clicked {selector} — now at {await _url(page)}")
+ if _TV3_MARKER_SELECTOR_RE.match(selector.strip()):
+ missing = False
+ try:
+ missing = not await page.evaluate(_SELECTOR_EXISTS_JS, selector)
+ except Exception:
+ missing = False
+ if missing:
+ # An absent marker cannot reappear without a re-observe, so Playwright's full 15s
+ # actionability wait is pure loss (4x in the specimen trace). Short attach grace
+ # tolerates a framework re-attaching the same node mid-render.
+ try:
+ await page.wait_for_selector(selector, state="attached", timeout=1200)
+ except Exception:
+ return ToolResult.error(
+ f"{selector} no longer exists on the page — element markers vanish when the "
+ "page re-renders (a closed menu destroys its options). Re-observe and act on "
+ "fresh selectors from the new observation."
+ )
+ pre: dict[str, Any] | None = None
+ try:
+ pre_raw = await page.evaluate(_CLICK_PRECHECK_JS, selector)
+ if isinstance(pre_raw, dict):
+ pre = pre_raw
+ except Exception:
+ pre = None
+ if pre is not None and pre.get("isOption"):
+ # Playwright's click hovers first, and menus routinely restyle a row on hover — so the
+ # commit baseline must be the POST-hover fingerprint, or a mere highlight would read as
+ # "its state changed" commit evidence on a no-op click.
+ try:
+ await page.hover(selector, timeout=2000)
+ hovered = await page.evaluate(_MENU_AFTER_JS, selector)
+ if isinstance(hovered, dict) and hovered.get("optState"):
+ pre["optState"] = hovered["optState"]
+ except Exception:
+ pass
+ url_before = await _url(page)
+ try:
+ await page.click(selector, timeout=15000)
+ except Exception as e:
+ gone = False
+ try:
+ gone = not await page.evaluate(_SELECTOR_EXISTS_JS, selector)
+ except Exception:
+ gone = False
+ if gone:
+ return ToolResult.error(
+ f"click on {selector} failed: the element no longer exists on the page — it was "
+ "likely removed by a re-render (e.g. a menu closed and destroyed its options). "
+ f"Re-observe and act on fresh selectors. (original error: {type(e).__name__})"
+ )
+ raise
+ base = f"clicked {selector} — now at {await _url(page)}"
+ if pre is None:
+ return ToolResult.ok(base)
+ try:
+ note, commit_error = await _click_reaction(page, selector, pre, url_before)
+ except Exception:
+ LOG.debug("taskv3 click reaction probe failed", selector=selector, exc_info=True)
+ return ToolResult.ok(base)
+ if commit_error is not None:
+ return ToolResult.error(commit_error)
+ return ToolResult.ok(base + "\n" + note if note else base)
async def hover(args: dict[str, Any]) -> ToolResult:
page, error = await _resolve_page()
@@ -686,8 +1033,11 @@ def build_browser_tools(
),
_spec(
"click",
- "Click an element by CSS selector. If the click triggers a file download, the tool result "
- "reports it when detected.",
+ "Click an element by CSS selector. If the click opens a menu of options, the result lists "
+ 'them with [data-tv3-menu="N"] selectors — click one of those to select (verified: you get '
+ "a loud error, not a silent no-op, if the selection does not commit; do not blindly repeat "
+ "a failed click). If the click triggers a file download, the tool result reports it when "
+ "detected.",
_obj({"selector": {"type": "string"}}, ["selector"]),
click,
),
diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py
index 791a3a0fd..3d4f3a905 100644
--- a/tests/unit/helpers.py
+++ b/tests/unit/helpers.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
+from skyvern.webeye.actions.actions import ActionStatus, ActionType
class FakeLLMResponse:
@@ -308,3 +309,34 @@ def setup_parallel_verification_mocks(
def make_browser_state() -> tuple[MagicMock, MagicMock, MagicMock]:
return MagicMock(), MagicMock(), MagicMock()
+
+
+def make_action_row(**overrides: Any) -> SimpleNamespace:
+ """Build a minimal duck-typed stand-in for ActionModel."""
+ base: dict[str, Any] = {
+ "action_id": "act_test",
+ "action_type": ActionType.MOVE,
+ "status": ActionStatus.completed,
+ "source_action_id": None,
+ "organization_id": "o_test",
+ "workflow_run_id": "wr_test",
+ "task_id": "tsk_test",
+ "step_id": "stp_test",
+ "step_order": 0,
+ "action_order": 0,
+ "confidence_float": None,
+ "reasoning": None,
+ "intention": None,
+ "response": None,
+ "element_id": None,
+ "skyvern_element_hash": None,
+ "skyvern_element_data": None,
+ "screenshot_artifact_id": None,
+ "started_at": None,
+ "finished_at": None,
+ "created_at": datetime(2026, 5, 6, 0, 0, 0),
+ "modified_at": datetime(2026, 5, 6, 0, 0, 0),
+ "action_json": None,
+ }
+ base.update(overrides)
+ return SimpleNamespace(**base)
diff --git a/tests/unit/test_agent_task_v3.py b/tests/unit/test_agent_task_v3.py
index 68ed4cc7d..5a9504abf 100644
--- a/tests/unit/test_agent_task_v3.py
+++ b/tests/unit/test_agent_task_v3.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.artifact.manager import ArtifactManager
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
+from skyvern.forge.sdk.db.utils import hydrate_action
from skyvern.forge.sdk.experimentation.providers import BaseExperimentationProvider
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
@@ -41,8 +42,18 @@ from skyvern.forge.sdk.workflow.models.parameter import CredentialParameter, Out
from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.forge.taskv3.engine import MIN_ACTION_STEPS
from skyvern.forge.taskv3.loop import LoopOutcome
-from skyvern.webeye.actions.actions import ActionStatus, ActionType
-from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task
+from skyvern.webeye.actions.actions import (
+ ActionStatus,
+ ActionType,
+ ClickAction,
+ HoverAction,
+ InputTextAction,
+ KeypressAction,
+ SelectOptionAction,
+ SolveCaptchaAction,
+ UploadFileAction,
+)
+from tests.unit.helpers import make_action_row, make_browser_state, make_organization, make_step, make_task
async def _run_execute_task_v3(
@@ -2019,3 +2030,88 @@ async def test_execute_task_v3_page_fingerprint_peeks_without_recovery(
loop_mock.browser_state.get_working_page = lost_page_peek
assert await fingerprint() is None
lost_page_peek.assert_awaited_once()
+
+
+async def test_execute_task_v3_persists_typed_actions_that_hydrate_as_their_subclass(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A row carrying only a description fails typed-subclass validation on every read and falls back
+ # to base Action with a warning; the persisted action must carry the tool call's typed fields so
+ # hydrate_action returns the subclass its action_type names (SKY-14494).
+ from skyvern.forge import agent as agent_mod
+
+ rounds = [
+ [
+ ("click", {"selector": "#go"}, True),
+ ("hover", {"selector": "#menu"}, True),
+ ("type", {"selector": "#q", "text": "hello"}, True),
+ ("select_option", {"selector": "#plan", "label": "Pro"}, True),
+ ("select_combobox", {"selector": "#city", "value": "Lisbon"}, True),
+ ("press_key", {"key": "Enter"}, True),
+ ("file_upload", {"selector": "#cv", "file": "https://files.example/cv.pdf"}, True),
+ ("solve_captcha", {}, True),
+ ]
+ ]
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["click"])
+ await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=rounds,
+ workflow_run_id="wr_v3test",
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ persisted = [c.kwargs["action"] for c in agent_mod.app.DATABASE.workflow_params.create_action.await_args_list]
+ assert [a.description for a in persisted[:2]] == ["task_v3 click #go", "task_v3 hover #menu"]
+ click, hover, typed, selected, combobox, keypress, upload, captcha = (
+ hydrate_action(make_action_row(action_type=a.action_type, element_id=a.element_id, action_json=a.model_dump()))
+ for a in persisted
+ )
+ assert isinstance(click, ClickAction) and click.element_id == "#go"
+ assert isinstance(hover, HoverAction) and hover.element_id == "#menu"
+ assert isinstance(typed, InputTextAction) and (typed.element_id, typed.text) == ("#q", "hello")
+ assert isinstance(selected, SelectOptionAction) and (selected.element_id, selected.option.label) == ("#plan", "Pro")
+ assert isinstance(combobox, SelectOptionAction) and (combobox.element_id, combobox.option.value) == (
+ "#city",
+ "Lisbon",
+ )
+ assert isinstance(keypress, KeypressAction) and keypress.keys == ["Enter"]
+ assert isinstance(upload, UploadFileAction) and (upload.element_id, upload.file_url) == (
+ "#cv",
+ "https://files.example/cv.pdf",
+ )
+ assert isinstance(captcha, SolveCaptchaAction)
+
+
[email protected]("typed_code", ["482913", 482913, ["482913"]])
+async def test_execute_task_v3_redacts_registered_secrets_from_persisted_action_fields(
+ monkeypatch: pytest.MonkeyPatch, typed_code: str | int | list[str]
+) -> None:
+ # A verification code the model typed is a registered secret; persisting the typed field must
+ # scrub it under the same redaction gate the v3 path applies to extracted output - also when the
+ # model emits the code as a JSON number, since nothing coerces tool args to the declared schema.
+ from skyvern.forge import agent as agent_mod
+
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.artifact_redaction_enabled", lambda *_a, **_k: True
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.WORKFLOW_CONTEXT_MANAGER.get_secret_values_for_run", lambda *_a, **_k: {"482913"}
+ )
+ outcome = LoopOutcome(status="completed", reason="done", billable_actions=["type"])
+ await _run_execute_task_v3(
+ monkeypatch,
+ outcome,
+ action_rounds=[[("type", {"selector": "#otp", "text": typed_code}, True)]],
+ workflow_run_id="wr_v3test",
+ data_extraction_goal=None,
+ extracted_information_schema=None,
+ )
+
+ persisted = agent_mod.app.DATABASE.workflow_params.create_action.await_args.kwargs["action"]
+ assert isinstance(persisted, InputTextAction)
+ assert persisted.element_id == "#otp"
+ assert "482913" not in persisted.model_dump_json()
diff --git a/tests/unit/test_cdp_download_interceptor.py b/tests/unit/test_cdp_download_interceptor.py
index f1e4fba7d..2eabef47a 100644
--- a/tests/unit/test_cdp_download_interceptor.py
+++ b/tests/unit/test_cdp_download_interceptor.py
@@ -1229,7 +1229,7 @@ class TestCDPDownloadInterceptorProxyAuth:
stalled_session.detach.side_effect = stalled_detach
binding = asyncio.create_task(replacement.bind_to_context(browser_context))
try:
- await asyncio.wait_for(detach_started.wait(), timeout=5)
+ await asyncio.wait_for(detach_started.wait(), timeout=0.1)
await asyncio.wait_for(binding, timeout=0.7)
await asyncio.wait_for(detach_cancelled.wait(), timeout=0.1)
diff --git a/tests/unit/test_hydrate_action_resilience.py b/tests/unit/test_hydrate_action_resilience.py
index 5a910fb92..4e7e75b44 100644
--- a/tests/unit/test_hydrate_action_resilience.py
+++ b/tests/unit/test_hydrate_action_resilience.py
@@ -5,7 +5,6 @@ endpoint via ValidationError propagation.
"""
import inspect
-from datetime import datetime
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
@@ -22,6 +21,7 @@ from skyvern.webeye.actions.actions import (
MoveAction,
UploadFileAction,
)
+from tests.unit.helpers import make_action_row as _action_row
def _task() -> SimpleNamespace:
@@ -49,37 +49,6 @@ def _session_yielding(rows: list) -> Any:
return staticmethod(lambda *_args: _Session())
-def _action_row(**overrides: Any) -> SimpleNamespace:
- """Build a minimal duck-typed stand-in for ActionModel."""
- base: dict[str, Any] = {
- "action_id": "act_test",
- "action_type": ActionType.MOVE,
- "status": ActionStatus.completed,
- "source_action_id": None,
- "organization_id": "o_test",
- "workflow_run_id": "wr_test",
- "task_id": "tsk_test",
- "step_id": "stp_test",
- "step_order": 0,
- "action_order": 0,
- "confidence_float": None,
- "reasoning": None,
- "intention": None,
- "response": None,
- "element_id": None,
- "skyvern_element_hash": None,
- "skyvern_element_data": None,
- "screenshot_artifact_id": None,
- "started_at": None,
- "finished_at": None,
- "created_at": datetime(2026, 5, 6, 0, 0, 0),
- "modified_at": datetime(2026, 5, 6, 0, 0, 0),
- "action_json": None,
- }
- base.update(overrides)
- return SimpleNamespace(**base)
-
-
def test_hydrate_action_happy_path_returns_subclass() -> None:
row = _action_row(action_json={"x": 10, "y": 20})
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 76c2fd09a..6a815bf1b 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -1445,3 +1445,679 @@ async def test_download_signal_survives_compaction_end_to_end_through_loop(tmp_p
elided = [m for m in observe_msgs if m["content"].startswith("[superseded ")]
assert len(elided) >= 1 # compaction actually ran on an earlier observe
assert any("Downloaded: report.pdf" in m["content"] for m in observe_msgs) # notice survives on a live message
+
+
+# --- Commit-verified click-open dropdown selection. The staging specimen: a click-open
+# filter popover that toggles open/closed on trigger clicks and REMOUNTS its option nodes each open,
+# so ids from a prior observe go stale and every toggle click returns an uninformative ok. The click
+# tool must (1) report a menu it opened WITH stable in-DOM option tags, (2) verify an option click
+# committed (menu closed / navigation / option state change) and error loudly when it did not, and
+# (3) turn vanished-element timeouts into honest "re-observe" errors. Plain clicks stay byte-identical. ---
+
+
+class _ClickFakePage:
+ """Fake page for the click reaction-probe control flow. `evaluate` dispatches on distinctive
+ substrings of the real JS constants (mirroring _TypeaheadFakePage): 'return !!' => the
+ selector-exists probe, 'menuOpen' => the pre-click check, 'stillOpen' => the menu-state read
+ (consumed in call order from `after_states`, last one repeating — the handler reads it after
+ hover, after click, and after the settle), 'clickable' => the new-menu finder."""
+
+ def __init__(
+ self,
+ *,
+ exists: bool = True,
+ menu_open: bool = False,
+ is_option: bool = False,
+ opt_text: str = "",
+ opt_state: str = "",
+ after_states: list[dict[str, Any]] | None = None,
+ found_menu: dict[str, Any] | None = None,
+ probe_raises: bool = False,
+ find_raises: bool = False,
+ click_raises: Exception | None = None,
+ ) -> None:
+ self.url = "https://example.test/results"
+ self.calls: list[tuple[str, Any]] = []
+ self._exists = exists
+ self._menu_open = menu_open
+ self._is_option = is_option
+ self._opt_text = opt_text
+ self._opt_state = opt_state
+ self._after_states = after_states or [{"stillOpen": 0, "optState": ""}]
+ self._after_i = 0
+ self._found_menu = found_menu
+ self._probe_raises = probe_raises
+ self._find_raises = find_raises
+ self._click_raises = click_raises
+
+ async def evaluate(self, js: str, arg: Any = None) -> Any:
+ if "return !!" in js:
+ return self._exists
+ if self._probe_raises:
+ raise RuntimeError("probe boom")
+ if "menuOpen" in js:
+ return {
+ "menuOpen": self._menu_open,
+ "isOption": self._is_option,
+ "optText": self._opt_text,
+ "optState": self._opt_state,
+ }
+ if "stillOpen" in js:
+ state = self._after_states[min(self._after_i, len(self._after_states) - 1)]
+ self._after_i += 1
+ return state
+ if "clickable" in js:
+ if self._find_raises:
+ raise RuntimeError("finder boom")
+ return self._found_menu
+ return None
+
+ async def click(self, selector: str, timeout: int | None = None) -> None:
+ self.calls.append(("click", selector))
+ if self._click_raises is not None:
+ raise self._click_raises
+
+ async def hover(self, selector: str, timeout: int | None = None) -> None:
+ self.calls.append(("hover", selector))
+
+ async def wait_for_selector(self, selector: str, state: str = "visible", timeout: int | None = None) -> None:
+ self.calls.append(("wait_for_selector", (selector, timeout)))
+ if not self._exists:
+ raise TimeoutError(f"waiting for {selector}")
+
+
+async def test_click_plain_result_format_unchanged() -> None:
+ # Constraint: a click that is not a dropdown interaction gains nothing — the result is
+ # byte-identical to the pre-feature format (no notes, no errors, no latency-adding retries).
+ page = _ClickFakePage()
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#save"})
+ assert r.status == "ok"
+ assert r.content == "clicked #save — now at https://example.test/results"
+
+
+async def test_click_probe_failure_falls_back_to_bare_ok() -> None:
+ # Any reaction-probe failure must degrade to today's behavior, never fail the click.
+ page = _ClickFakePage(probe_raises=True)
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#save"})
+ assert r.status == "ok"
+ assert r.content == "clicked #save — now at https://example.test/results"
+ assert ("click", "#save") in page.calls
+
+
+async def test_click_reports_opened_menu_with_stable_tags() -> None:
+ page = _ClickFakePage(
+ found_menu={
+ "count": 7,
+ "options": [
+ {"n": 1, "text": "Relevance"},
+ {"n": 2, "text": "Most recent"},
+ {"n": 3, "text": "Most popular"},
+ {"n": 4, "text": "Highest rated"},
+ {"n": 5, "text": "Nearest"},
+ {"n": 6, "text": "Price low to high"},
+ {"n": 7, "text": "Price high to low"},
+ ],
+ }
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3="t82"]'})
+ assert r.status == "ok"
+ assert "opened a menu of 7 options" in r.content
+ assert '[data-tv3-menu="1"]' in r.content and "Relevance" in r.content
+ assert "Most popular" in r.content
+ # the model is told the options are volatile: re-clicking the trigger destroys them
+ assert "closes the menu" in r.content
+
+
+async def test_click_menu_listing_is_bounded_with_overflow_note() -> None:
+ # Click results are billable and NOT compactable — they live in the transcript forever, so a
+ # 40-option menu must not inflate a permanent transcript entry.
+ options = [{"n": i, "text": f"Option number {i}"} for i in range(1, 16)]
+ page = _ClickFakePage(found_menu={"count": 40, "options": options})
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#filters"})
+ assert r.status == "ok"
+ assert "opened a menu of 40 options" in r.content
+ assert "Option number 15" in r.content
+ assert "Option number 16" not in r.content
+ assert "+25 more" in r.content and "re-observe" in r.content
+
+
+async def test_click_option_commit_verified_by_menu_close() -> None:
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="Most popular",
+ opt_state="s0",
+ after_states=[{"stillOpen": 7, "optState": "s0"}, {"stillOpen": 0, "optState": ""}],
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="3"]'})
+ assert r.status == "ok"
+ assert "Selected option 'Most popular'" in r.content
+ assert "menu closed" in r.content
+ # commit judged against the POST-hover baseline: the handler hovered before clicking
+ assert ("hover", '[data-tv3-menu="3"]') in page.calls
+
+
+async def test_click_option_no_commit_errors_loud(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The click-open-family contract: an option click whose selection did not commit (menu still open
+ # even after the settle, option state unchanged, no navigation, no submenu) must error loudly —
+ # never a bare ok that fuels a 21-click loop.
+ import asyncio as _a
+
+ monkeypatch.setattr(_a, "sleep", _instant_sleep)
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="Most popular",
+ opt_state="s0",
+ after_states=[{"stillOpen": 7, "optState": "s0"}],
+ found_menu=None,
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="3"]'})
+ assert r.status == "error"
+ assert "did not commit" in r.content
+ assert "Most popular" in r.content
+ assert "Do not repeat" in r.content
+
+
+async def test_click_option_commit_after_async_close_settles_not_errors(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A healthy commit that closes the menu via a fade or an async server ack must not read as
+ # "did not commit" off the instantaneous probe — the settle re-probe accepts the late close.
+ import asyncio as _a
+
+ monkeypatch.setattr(_a, "sleep", _instant_sleep)
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="Most popular",
+ opt_state="s0",
+ after_states=[
+ {"stillOpen": 7, "optState": "s0"}, # post-hover baseline
+ {"stillOpen": 7, "optState": "s0"}, # instant read: close animation still running
+ {"stillOpen": 0, "optState": ""}, # settled: closed
+ ],
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="3"]'})
+ assert r.status == "ok"
+ assert "Selected option 'Most popular'" in r.content
+
+
+async def test_click_hover_highlight_is_not_commit_evidence(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Playwright's click hovers first, and menus restyle rows on hover. The pre-click fingerprint
+ # ('s0') differs from the hovered one ('hl') — commit must be judged against the POST-hover
+ # baseline, so an otherwise no-op click still errors instead of claiming "its state changed".
+ import asyncio as _a
+
+ monkeypatch.setattr(_a, "sleep", _instant_sleep)
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="Banana",
+ opt_state="s0",
+ after_states=[{"stillOpen": 7, "optState": "hl"}],
+ found_menu=None,
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="1"]'})
+ assert r.status == "error"
+ assert "did not commit" in r.content
+
+
+async def test_click_option_no_commit_error_survives_submenu_probe_crash(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Once no-commit evidence is established, a crash of the final informational submenu probe must
+ # not fall through to the fail-open bare ok — that would be a silent ok on the exact case the
+ # feature exists to make loud.
+ import asyncio as _a
+
+ monkeypatch.setattr(_a, "sleep", _instant_sleep)
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="Most popular",
+ opt_state="s0",
+ after_states=[{"stillOpen": 7, "optState": "s0"}],
+ find_raises=True,
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="3"]'})
+ assert r.status == "error"
+ assert "did not commit" in r.content
+
+
+async def test_click_multiselect_option_state_change_is_commit() -> None:
+ # Filter menus are disproportionately multi-select: a successful pick leaves the menu OPEN and
+ # marks the option (checkmark/aria-checked/class flip). That state change IS commit evidence —
+ # erroring here would fire exactly where the feature is aimed.
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="In stock",
+ opt_state="aria-checked=false",
+ after_states=[
+ {"stillOpen": 7, "optState": "aria-checked=false"}, # post-hover baseline
+ {"stillOpen": 7, "optState": "aria-checked=true"}, # post-click: marked
+ ],
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="2"]'})
+ assert r.status == "ok"
+ assert "Selected option 'In stock'" in r.content
+
+
+async def test_click_option_opening_submenu_is_not_a_false_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ # An option click that spawns a child menu (cascading filter, date-picker) commits nothing yet
+ # but is NOT a failure — report the submenu's options instead of a false "did not commit".
+ import asyncio as _a
+
+ monkeypatch.setattr(_a, "sleep", _instant_sleep)
+ page = _ClickFakePage(
+ menu_open=True,
+ is_option=True,
+ opt_text="More filters",
+ opt_state="s0",
+ after_states=[{"stillOpen": 7, "optState": "s0"}],
+ found_menu={
+ "count": 3,
+ "options": [{"n": 1, "text": "Colour"}, {"n": 2, "text": "Size"}, {"n": 3, "text": "Brand"}],
+ },
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="5"]'})
+ assert r.status == "ok"
+ assert "opened a menu of 3 options" in r.content
+
+
+async def test_click_trigger_reclick_reports_menu_closed_no_selection() -> None:
+ # The staging loop's core move: re-clicking the trigger toggles the menu closed. That must be
+ # named — 17 byte-identical bare oks are what kept the model looping.
+ page = _ClickFakePage(
+ menu_open=True, is_option=False, after_states=[{"stillOpen": 0, "optState": ""}], found_menu=None
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3="t82"]'})
+ assert r.status == "ok"
+ assert "CLOSED the open menu" in r.content
+ assert "no option was selected" in r.content
+
+
+async def test_click_stale_marker_fast_fails_without_15s_wait() -> None:
+ # A [data-tv3=...] marker is minted only by our own enrichment: if it matches nothing now, it can
+ # never appear without a re-observe — waiting Playwright's full 15s (4x in the staging trace) is
+ # pure loss. Fail fast and loud, and never dispatch the doomed click.
+ page = _ClickFakePage(exists=False)
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3="t157"]'})
+ assert r.status == "error"
+ assert "no longer exists" in r.content
+ assert "e-observe" in r.content
+ assert not any(c[0] == "click" for c in page.calls)
+ # the short attach grace was attempted (re-attach tolerance), not a bare instant fail
+ waited = [c for c in page.calls if c[0] == "wait_for_selector"]
+ assert waited and waited[0][1][1] is not None and waited[0][1][1] <= 2000
+
+
+async def test_click_timeout_on_vanished_element_reports_removal() -> None:
+ # Non-marker selector: the click itself timed out AND the element is gone — say so, instead of
+ # surfacing a generic Playwright timeout the model cannot act on.
+ page = _ClickFakePage(exists=False, click_raises=TimeoutError("Page.click: Timeout 15000ms exceeded"))
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#opt-old"})
+ assert r.status == "error"
+ assert "no longer exists" in r.content
+
+
+async def test_click_timeout_on_live_element_reraises_original() -> None:
+ # The element exists but was not actionable (covered/detached mid-render): that is a genuine
+ # timeout — keep today's behavior (the loop converts the raised error), no reinterpretation.
+ class _Boom(Exception):
+ pass
+
+ page = _ClickFakePage(exists=True, click_raises=_Boom("not actionable"))
+ tools = build_browser_tools(_fixed_page_provider(page))
+ with pytest.raises(_Boom):
+ await _tool(tools, "click").handler({"selector": "#covered"})
+
+
+# --- DOM-level tests: the REAL precheck/finder/after JS against live Chromium, on a faithful mimic
+# of the staging widget (conditional-render popover, option nodes REMOUNTED on every toggle). ---
+
+_MENU_FIXTURE_HTML = """
+<!doctype html><html><body style="margin:0">
+ <div style="height:40px">Results header</div>
+ <button id="sort-trigger" style="position:absolute;top:50px;left:600px;height:30px">Sort: Relevance</button>
+ <div id="results" style="position:absolute;top:120px;left:40px;width:500px">
+ <a href="#r1" style="display:block;height:40px">First result row</a>
+ <a href="#r2" style="display:block;height:40px">Second result row</a>
+ </div>
+ <div id="pager" style="position:absolute;top:220px;left:40px"><button id="next-page">Next</button></div>
+ <script>
+ window.__commits = 0;
+ const OPTS = ['Relevance','Most recent','Most popular','Highest rated','Nearest','Price low to high','Price high to low'];
+ document.getElementById('sort-trigger').addEventListener('click', () => {
+ const ex = document.getElementById('sort-menu');
+ if (ex) { ex.remove(); return; }
+ const card = document.createElement('div');
+ card.id = 'sort-menu';
+ card.setAttribute('style', 'position:absolute;top:82px;left:600px;width:180px;background:#fff;border:1px solid #ccc');
+ for (const t of OPTS) {
+ const b = document.createElement('button');
+ b.type = 'button'; b.textContent = t;
+ b.setAttribute('style', 'display:block;width:100%;height:28px;text-align:left');
+ if (window.__checkboxSelect) {
+ const cb = document.createElement('input');
+ cb.type = 'checkbox'; cb.setAttribute('style', 'pointer-events:none');
+ b.prepend(cb);
+ }
+ b.addEventListener('mouseenter', () => { b.className = 'hl'; });
+ b.addEventListener('click', (ev) => {
+ ev.stopPropagation();
+ if (window.__noCommit) return;
+ if (window.__checkboxSelect) {
+ const cb = b.querySelector('input');
+ cb.checked = !cb.checked;
+ window.__commits++;
+ return;
+ }
+ if (window.__multiSelect) {
+ b.setAttribute('aria-checked', b.getAttribute('aria-checked') === 'true' ? 'false' : 'true');
+ window.__commits++;
+ return;
+ }
+ document.getElementById('sort-trigger').textContent = 'Sort: ' + t;
+ window.__commits++;
+ if (window.__asyncClose) { setTimeout(() => card.remove(), 300); } else { card.remove(); }
+ });
+ card.appendChild(b);
+ }
+ document.body.appendChild(card);
+ });
+ document.getElementById('next-page').addEventListener('click', () => {
+ const res = document.getElementById('results');
+ res.innerHTML = '';
+ for (const t of ['Third result row', 'Fourth result row']) {
+ const a = document.createElement('a');
+ a.href = '#more'; a.textContent = t;
+ a.setAttribute('style', 'display:block;height:40px;cursor:pointer');
+ res.appendChild(a);
+ }
+ });
+ </script>
+</body></html>
+"""
+
+
+async def _menu_page() -> AsyncIterator[Any]:
+ from playwright.async_api import async_playwright # noqa: PLC0415
+
+ async with async_playwright() as pw:
+ browser = await pw.chromium.launch(headless=True)
+ try:
+ context = await browser.new_context(viewport={"width": 1024, "height": 900})
+ page = await context.new_page()
+ await page.set_content(_MENU_FIXTURE_HTML)
+ yield page
+ finally:
+ await browser.close()
+
+
+@_skip_no_browser
+async def test_dom_click_trigger_reports_menu_and_option_click_commits_in_two_actions() -> None:
+ # The staging shape, green contract: commit achieved within 2 clicks — never 21.
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert r1.status == "ok"
+ assert "opened a menu of 7 options" in r1.content
+ assert '[data-tv3-menu="3"]' in r1.content and "Most popular" in r1.content
+ r2 = await click.handler({"selector": '[data-tv3-menu="3"]'})
+ assert r2.status == "ok"
+ assert "Selected option 'Most popular'" in r2.content
+ label = await page.eval_on_selector("#sort-trigger", "e => e.textContent")
+ assert label == "Sort: Most popular"
+ assert await page.evaluate("() => window.__commits") == 1
+
+
+@_skip_no_browser
+async def test_dom_trigger_reclick_names_the_toggle_close() -> None:
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert "opened a menu of 7 options" in r1.content
+ r2 = await click.handler({"selector": "#sort-trigger"})
+ assert r2.status == "ok"
+ assert "CLOSED the open menu" in r2.content
+ assert await page.evaluate("() => window.__commits") == 0
+
+
+@_skip_no_browser
+async def test_dom_option_click_that_never_commits_errors_loud() -> None:
+ # The no-commit variant of the widget (click lands, nothing changes): loud error, not a bare ok.
+ async with _menu_page() as page:
+ await page.evaluate("() => { window.__noCommit = 1; }")
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert "opened a menu of 7 options" in r1.content
+ r2 = await click.handler({"selector": '[data-tv3-menu="3"]'})
+ assert r2.status == "error"
+ assert "did not commit" in r2.content
+
+
+@_skip_no_browser
+async def test_dom_multiselect_option_commits_by_state_change() -> None:
+ async with _menu_page() as page:
+ await page.evaluate("() => { window.__multiSelect = 1; }")
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ await click.handler({"selector": "#sort-trigger"})
+ r2 = await click.handler({"selector": '[data-tv3-menu="2"]'})
+ assert r2.status == "ok"
+ assert "Selected option 'Most recent'" in r2.content
+ assert await page.evaluate("() => window.__commits") == 1
+
+
+@_skip_no_browser
+async def test_dom_multiselect_option_commits_by_child_checkbox_property() -> None:
+ # A multi-select whose commit is ONLY a child checkbox's .checked DOM property (no aria, class,
+ # or text change) must read as committed — not return a false "did not commit" error.
+ async with _menu_page() as page:
+ await page.evaluate("() => { window.__checkboxSelect = 1; }")
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert "opened a menu of 7 options" in r1.content
+ r2 = await click.handler({"selector": '[data-tv3-menu="2"]'})
+ assert r2.status == "ok"
+ assert "Selected option" in r2.content and "Most recent" in r2.content
+ assert await page.evaluate("() => window.__commits") == 1
+
+
+@_skip_no_browser
+async def test_dom_self_mutating_row_text_is_not_commit_evidence() -> None:
+ # A row whose text updates on its own (countdown, live price) changes the state fingerprint
+ # without any commit; a no-op click on it must error loud, not read as "its state changed".
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert "opened a menu of 7 options" in r1.content
+ await page.evaluate(
+ "() => { window.__noCommit = 1; const row = document.querySelector('[data-tv3-menu=\"1\"]');"
+ " let n = 0; setInterval(() => { row.textContent = 'Trending ' + n++; }, 80); }"
+ )
+ r2 = await click.handler({"selector": '[data-tv3-menu="1"]'})
+ assert r2.status == "error"
+ assert "did not commit" in r2.content
+ assert await page.evaluate("() => window.__commits") == 0
+
+
+@_skip_no_browser
+async def test_dom_stale_menu_marker_click_fails_fast_and_loud() -> None:
+ # Menu-row tags die when the popover remounts — a click on a stale [data-tv3-menu] id (the
+ # staging trace's 4x-timeout move) must fail fast like a stale [data-tv3] marker, not eat 15s.
+ import time # noqa: PLC0415
+
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ r1 = await click.handler({"selector": "#sort-trigger"})
+ assert "opened a menu of 7 options" in r1.content
+ r2 = await click.handler({"selector": "#sort-trigger"})
+ assert "CLOSED the open menu" in r2.content
+ start = time.monotonic()
+ r3 = await click.handler({"selector": '[data-tv3-menu="3"]'})
+ elapsed = time.monotonic() - start
+ assert r3.status == "error"
+ assert "no longer exists" in r3.content
+ assert elapsed < 5.0 # not Playwright's 15s actionability wait
+
+
+@_skip_no_browser
+async def test_dom_stale_marker_click_fails_fast_and_loud() -> None:
+ import time # noqa: PLC0415
+
+ async with _menu_page() as page:
+ await page.evaluate(
+ "() => { const b = document.createElement('button'); b.setAttribute('data-tv3', 't9');"
+ " b.textContent = 'ephemeral'; document.body.appendChild(b); b.remove(); }"
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ start = time.monotonic()
+ r = await _tool(tools, "click").handler({"selector": '[data-tv3="t9"]'})
+ elapsed = time.monotonic() - start
+ assert r.status == "error"
+ assert "no longer exists" in r.content
+ assert elapsed < 5.0 # not Playwright's 15s actionability wait
+
+
+@_skip_no_browser
+ "selector",
+ ["#plain-btn", "#nav-link", "#the-checkbox", "#submit-btn", "#next-page"],
+ ids=["plain-button", "nav-link", "checkbox", "submit", "pagination"],
+)
+async def test_dom_fp_matrix_plain_interactions_pass_through_untouched(selector: str) -> None:
+ # The FP matrix from the brief: none of these are dropdown interactions; each must produce the
+ # exact bare pre-feature result — no menu notes, no errors. Pagination is the sharp case: it
+ # swaps rows inside a PRE-EXISTING visible container, which must not read as a menu opening.
+ extra = """
+ <button id="plain-btn" style="position:absolute;top:300px;left:40px">Save draft</button>
+ <a id="nav-link" href="#section-2" style="position:absolute;top:340px;left:40px;display:block">Jump to section</a>
+ <input id="the-checkbox" type="checkbox" style="position:absolute;top:380px;left:40px">
+ <form action="#done" style="position:absolute;top:420px;left:40px"><button id="submit-btn" type="submit">Submit</button></form>
+ """
+ async with _menu_page() as page:
+ await page.evaluate(
+ "(html) => { const d = document.createElement('div'); d.innerHTML = html; document.body.appendChild(d); }",
+ extra,
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": selector})
+ assert r.status == "ok"
+ assert r.content == f"clicked {selector} — now at {page.url}"
+
+
+@_skip_no_browser
+async def test_dom_async_close_commit_is_not_a_false_error() -> None:
+ # A menu that closes 300ms after the option click (server-ack pattern) is a healthy commit; the
+ # settle re-probe must accept it instead of erroring off the instantaneous read.
+ async with _menu_page() as page:
+ await page.evaluate("() => { window.__asyncClose = 1; }")
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ await click.handler({"selector": "#sort-trigger"})
+ r2 = await click.handler({"selector": '[data-tv3-menu="3"]'})
+ assert r2.status == "ok"
+ assert "Selected option 'Most popular'" in r2.content
+ assert await page.evaluate("() => window.__commits") == 1
+
+
+@_skip_no_browser
+async def test_dom_clicking_menu_container_is_not_an_option_pick() -> None:
+ # Clicking the card AROUND the menu is not picking an option — fabricating "Selected option"
+ # for it would tell the model a selection happened when none did.
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ click = _tool(tools, "click")
+ await click.handler({"selector": "#sort-trigger"})
+ r2 = await click.handler({"selector": "#sort-menu"})
+ # The center-point click lands on an arbitrary row (a real Playwright behavior), so any
+ # selected/closed claim could be false — the contract is NO claims at all.
+ assert r2.status == "ok"
+ assert r2.content == f"clicked #sort-menu — now at {page.url}"
+
+
+@_skip_no_browser
+async def test_dom_confirm_dialog_is_not_reported_as_menu() -> None:
+ # A confirm dialog is a page mode, not a menu: its title/body are not clickable "options", and
+ # its horizontal button pair must not be either. The bare pre-feature ok is the contract.
+ async with _menu_page() as page:
+ await page.evaluate(
+ """() => {
+ const d = document.createElement('div');
+ d.innerHTML = '<button id="del-btn" style="position:absolute;top:300px;left:40px">Delete</button>';
+ document.body.appendChild(d);
+ document.getElementById('del-btn').addEventListener('click', () => {
+ const m = document.createElement('div');
+ m.id = 'confirm-modal';
+ m.setAttribute('role', 'dialog');
+ m.setAttribute('style', 'position:absolute;top:280px;left:60px;width:280px;background:#fff;border:1px solid #333;padding:8px');
+ m.innerHTML = '<h2 style="height:24px;margin:0">Delete this item?</h2>'
+ + '<p style="height:20px;margin:4px 0">This cannot be undone.</p>'
+ + '<div><button style="width:100px">Cancel</button><button style="width:100px">Confirm</button></div>';
+ document.body.appendChild(m);
+ });
+ }"""
+ )
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#del-btn"})
+ assert r.status == "ok"
+ assert r.content == f"clicked #del-btn — now at {page.url}"
+
+
+@_skip_no_browser
+async def test_dom_get_html_strips_reaction_bookkeeping_attrs() -> None:
+ # The reaction gate stamps data-tv3-pre on every visible element per click; leaking it into
+ # get_html would burn ~a third of the 20k truncation budget on noise.
+ async with _menu_page() as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ await _tool(tools, "click").handler({"selector": "#sort-trigger"})
+ html = await _tool(tools, "get_html").handler({})
+ assert "data-tv3-pre" not in html.content