Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Emit autocomplete commit evidence on the LLM selection path via a measured control transition (#8345)
LawyZhengfa0a7783bc02
diff --git a/skyvern/services/action_service.py b/skyvern/services/action_service.pyindex 754e1674b..9a7a791c0 100644--- a/skyvern/services/action_service.py+++ b/skyvern/services/action_service.py@@ -55,6 +55,8 @@ async def get_action_history("upload_file_triggered","needs_followup","followup_message",+ "committed_option",+ "committed_value",},),}diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex df76ef9d0..ac83b45de 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -680,6 +680,148 @@ async def _verify_autocomplete_input_readback(return False+def _is_boundary_fragment(fragment: str, whole: str) -> bool:+ """Whether ``fragment`` occurs in ``whole`` as a word-boundary-delimited contiguous run.++ Both are expected to already be normalized. A fragment carrying no alphanumeric character can+ never anchor a meaningful boundary match, so it is rejected. Default Unicode ``\\w`` semantics+ apply, so accented Latin behaves and unsegmented CJK interiors fail closed.+ """+ if not any(ch.isalnum() for ch in fragment):+ return False+ return re.search(rf"(?<!\w){re.escape(fragment)}(?!\w)", whole) is not None+++def _autocomplete_commit_evidence(+ pre_value: str | None,+ post_value: str | None,+ option_label: str | None,+) -> tuple[str, str] | None:+ """Observational commit evidence for an autocomplete selection, or None.++ Emits ``(committed_option, committed_value)`` — both truncated to the shared field cap — only+ when the clicked option's label and both control read-backs are nonempty *after normalization*,+ the normalized post-click value differs from the normalized pre-click value, and BOTH the+ normalized pre and post are boundary-delimited fragments of the normalized option label. That+ last relation is what makes the transition selection-specific: it rejects unrelated blur,+ masking, formatting, validation, and restoration transforms whose output is not a fragment of+ the clicked option. The relation runs on the full normalized strings before any truncation.+ Whitespace-only fields normalize to empty and fail closed; equality (a no-op or highlight-only+ click) yields None. Secret suppression is the caller's responsibility.+ """+ normalized_pre = _normalize_select_shadow_text(pre_value)+ normalized_post = _normalize_select_shadow_text(post_value)+ normalized_label = _normalize_select_shadow_text(option_label)+ if not normalized_label or not normalized_pre or not normalized_post:+ return None+ if normalized_post == normalized_pre:+ return None+ if not _is_boundary_fragment(normalized_pre, normalized_label):+ return None+ if not _is_boundary_fragment(normalized_post, normalized_label):+ return None+ committed_option = _truncate_select_shadow_field(option_label)+ committed_value = _truncate_select_shadow_field(post_value)+ if not committed_option or not committed_value:+ return None+ return committed_option, committed_value+++async def _read_autocomplete_control_value(+ skyvern_element: SkyvernElement,+ engine_selection: BrowserEngineSelection | None = None,+) -> str | None:+ try:+ return await get_input_value(+ skyvern_element.get_tag_name(),+ skyvern_element.get_locator(),+ engine_selection=engine_selection,+ read_timeout_ms=settings.BROWSER_ACTION_TIMEOUT_MS,+ )+ except Exception:+ LOG.info("Failed to read autocomplete control value for commit evidence", exc_info=True)+ return None+++async def _read_clicked_option_label(+ *,+ skyvern_frame: SkyvernFrame,+ option_locator: Locator,+ option_static_element: dict | None,+) -> str | None:+ identity = await _read_autocomplete_option_identity(skyvern_frame=skyvern_frame, locator=option_locator)+ label = identity.get("label") if identity else None+ if not label and option_static_element:+ # Fall back to the scraped node text; it is tied to this exact option by construction.+ label = option_static_element.get("text")+ label = (label or "").strip()+ return label or None+++async def _click_autocomplete_option_with_commit_evidence(+ *,+ skyvern_element: SkyvernElement,+ option_locator: Locator,+ option_static_element: dict | None,+ skyvern_frame: SkyvernFrame,+ click: Callable[[], Awaitable[None]],+ is_secret_value: bool,+ engine_selection: BrowserEngineSelection | None = None,+) -> ActionResult:+ """Click the LLM-selected option and return ``ActionSuccess``, enriched with commit evidence+ only when the target control's read-back changes into a value that — like the pre-click value —+ is a boundary-delimited fragment of the clicked option's label (see+ ``_autocomplete_commit_evidence``) AND that transitioned value survives a single bounded settle+ reread. The settle guards against an optimistic control that paints the selected label and then+ reverts it after async validation/rerender, which would otherwise commit a transient value.++ Evidence capture is best-effort and fail-closed: a failed read (before, after, or on the settle+ reread), a missing label, an unchanged control, an unrelated transform, a value that drifts on+ the settled reread, or a secret value all leave a bare ``ActionSuccess``. Only the click itself+ can fail the action — every evidence read is exception-isolated so a successful click never+ regresses to ``ActionFailure`` because capture failed.+ """+ option_label: str | None = None+ pre_value: str | None = None+ if not is_secret_value:+ try:+ option_label = await _read_clicked_option_label(+ skyvern_frame=skyvern_frame,+ option_locator=option_locator,+ option_static_element=option_static_element,+ )+ except Exception:+ LOG.info("Failed to read clicked autocomplete option label for commit evidence", exc_info=True)+ pre_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)++ await click()++ if is_secret_value:+ return ActionSuccess()++ try:+ post_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)+ evidence = _autocomplete_commit_evidence(pre_value, post_value, option_label)+ if evidence is None:+ return ActionSuccess()+ # The candidate transition passed, but an optimistic control can paint the selected label and+ # then revert it after async validation/rerender. Settle once (no loop) and reconfirm the+ # value; only record evidence when the settled reread still holds the first post value, so a+ # transient paint cannot be committed as stale evidence.+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.commit")+ confirm_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)+ if confirm_value is None or _normalize_select_shadow_text(confirm_value) != _normalize_select_shadow_text(+ post_value+ ):+ return ActionSuccess()+ except Exception:+ LOG.info("Autocomplete commit-evidence capture failed after a successful click", exc_info=True)+ return ActionSuccess()++ committed_option, committed_value = evidence+ return ActionSuccess(committed_option=committed_option, committed_value=committed_value)++async def _reset_autocomplete_for_llm_fallback(*,current_incremental_scraped: IncrementalScrapePage,@@ -7472,6 +7614,7 @@ async def _handle_input_text_action(task=task,action=action,collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,+ is_secret_value=is_secret_value,):auto_complete_hacky_flag = Falsereturn [result]@@ -9912,6 +10055,8 @@ async def choose_auto_completion_dropdown(is_location_input: bool = False,collapse_autocomplete_fanout_enabled: bool = False,action: InputTextAction | None = None,+ *,+ is_secret_value: bool,) -> AutoCompletionResult:preserved_elements = preserved_elements or []clear_input = True@@ -10234,8 +10379,20 @@ async def choose_auto_completion_dropdown(static_element=incremental_scraped.id_to_element_dict.get(element_id, {}),engine_selection=engine_selection,)- await selected_element.scroll_into_view()- await selected_element.click(page=page, engine_selection=engine_selection)++ async def _click_selected_option() -> None:+ await selected_element.scroll_into_view()+ await selected_element.click(page=page, engine_selection=engine_selection)++ result.action_result = await _click_autocomplete_option_with_commit_evidence(+ skyvern_element=skyvern_element,+ option_locator=locator,+ option_static_element=incremental_scraped.id_to_element_dict.get(element_id),+ skyvern_frame=skyvern_frame,+ click=_click_selected_option,+ is_secret_value=is_secret_value,+ engine_selection=engine_selection,+ )clear_input = Falsereturn result@@ -10288,6 +10445,8 @@ async def input_or_auto_complete_input(task: Task,action: InputTextAction | None = None,collapse_autocomplete_fanout_enabled: bool = False,+ *,+ is_secret_value: bool,) -> ActionResult | None:LOG.info("Trigger auto completion",@@ -10329,9 +10488,10 @@ async def input_or_auto_complete_input(is_location_input=is_location,collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,action=action,+ is_secret_value=is_secret_value,)if isinstance(result.action_result, ActionSuccess):- return ActionSuccess()+ return result.action_resultif input_or_select_context.is_search_bar:LOG.info(@@ -10397,9 +10557,10 @@ async def input_or_auto_complete_input(is_location_input=is_location,collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,action=action,+ is_secret_value=is_secret_value,)if isinstance(result.action_result, ActionSuccess):- return ActionSuccess()+ return result.action_resulttried_values.append(value)whole_new_elements.extend(result.incremental_elements)@@ -13828,20 +13989,25 @@ async def click_listbox_option(async def get_input_value(- tag_name: str, locator: Locator, engine_selection: BrowserEngineSelection | None = None+ tag_name: str,+ locator: Locator,+ engine_selection: BrowserEngineSelection | None = None,+ read_timeout_ms: float | None = None,) -> str | None:# input_value() rejects non-<input>/<textarea>/<select> nodes and inner_text() rejects# non-HTMLElement nodes; the live node can disagree with the scraped tag_name after a# re-render. Treat an incompatible read as "value unknown" so the caller's own# element-type classification runs instead of a raw driver exception escaping here. The# incompatible-node identity is matched against THIS run's selected engine; missing selection- # keeps the stock Playwright identity (unchanged default).+ # keeps the stock Playwright identity (unchanged default). read_timeout_ms is opt-in: when unset+ # the read keeps Playwright's default wait; callers that must not stall pass an explicit bound.+ read_kwargs = {} if read_timeout_ms is None else {"timeout": read_timeout_ms}try:if tag_name in COMMON_INPUT_TAGS:- return await locator.input_value()+ return await locator.input_value(**read_kwargs)# for span, div, p or other tags:# we need to trim the unicode space for these tags- return (await locator.inner_text()).replace("\xa0", " ").strip()+ return (await locator.inner_text(**read_kwargs)).replace("\xa0", " ").strip()except Exception as exc:if not _is_selected_engine_error(exc, engine_selection):raisediff --git a/skyvern/webeye/actions/responses.py b/skyvern/webeye/actions/responses.pyindex 009dd8d84..8497967b9 100644--- a/skyvern/webeye/actions/responses.py+++ b/skyvern/webeye/actions/responses.py@@ -27,6 +27,12 @@ class ActionResult(BaseModel):# suppressed it because the control already held the desired state. Both shapes stop the raw# click, so callers mirroring the setup contract have no other way to tell them apart.setup_performed: bool = False+ # Observational commit evidence for an autocomplete selection: the clicked option's label and+ # the control value observed immediately afterward. Set together only when both are+ # boundary-delimited fragments of the option label (a selection-specific transition); both stay+ # None on a no-op, unrelated transform, ambiguous, failed, or secret selection.+ committed_option: str | None = None+ committed_value: str | None = Nonedef __str__(self) -> str:results = [f"ActionResult(success={self.success}"]@@ -55,6 +61,10 @@ class ActionResult(BaseModel):results.append(f"interacted_with_parent={self.interacted_with_parent}")if self.skip_remaining_actions is not None:results.append(f"skip_remaining_actions={self.skip_remaining_actions}")+ if self.committed_option is not None:+ results.append(f"committed_option={self.committed_option}")+ if self.committed_value is not None:+ results.append(f"committed_value={self.committed_value}")return ", ".join(results) + ")"@@ -71,6 +81,8 @@ class ActionSuccess(ActionResult):interacted_with_sibling: bool = False,interacted_with_parent: bool = False,setup_performed: bool = False,+ committed_option: str | None = None,+ committed_value: str | None = None,):super().__init__(success=True,@@ -80,6 +92,8 @@ class ActionSuccess(ActionResult):interacted_with_sibling=interacted_with_sibling,interacted_with_parent=interacted_with_parent,setup_performed=setup_performed,+ committed_option=committed_option,+ committed_value=committed_value,)diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.pyindex 6d9b09b06..1932f100a 100644--- a/tests/unit/test_auto_completion_discover_fallback.py+++ b/tests/unit/test_auto_completion_discover_fallback.py@@ -595,6 +595,7 @@ async def test_input_or_auto_complete_calls_discover_fallback_on_failure() -> Noskyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,)assert isinstance(result, ActionSuccess)@@ -636,6 +637,7 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,)# Search bar returns None early, never reaching discover fallbackdiff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.pyindex 487beb26a..b3b985afb 100644--- a/tests/unit/test_auto_completion_location.py+++ b/tests/unit/test_auto_completion_location.py@@ -129,6 +129,7 @@ async def test_location_single_option_skips_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -173,6 +174,7 @@ async def test_location_whitespace_normalized_still_matches() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -223,6 +225,7 @@ async def test_location_multiple_options_calls_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -272,6 +275,7 @@ async def test_non_location_single_option_calls_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=False,)@@ -308,6 +312,7 @@ async def test_location_fast_path_returns_action_success() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -358,6 +363,7 @@ async def test_location_fast_path_element_not_in_dom_falls_through() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -395,6 +401,7 @@ async def test_dropdown_cleanup_clear_failure_does_not_clobber_result() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,)skyvern_el.input_clear.assert_awaited_once()@@ -426,6 +433,7 @@ async def test_input_or_auto_complete_passes_is_location_input() -> None:skyvern_element=_mock_skyvern_element(),step=_STEP,task=_TASK,+ is_secret_value=False,)assert isinstance(result, ActionSuccess)@@ -452,6 +460,7 @@ async def test_input_or_auto_complete_passes_false_for_non_location() -> None:skyvern_element=_mock_skyvern_element(),step=_STEP,task=_TASK,+ is_secret_value=False,)assert isinstance(result, ActionSuccess)@@ -544,6 +553,7 @@ async def test_location_no_results_option_falls_through_to_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -593,6 +603,7 @@ async def test_location_unrelated_option_falls_through_to_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,is_location_input=True,)@@ -628,6 +639,7 @@ async def test_collapse_autocomplete_exact_match_skips_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -667,6 +679,7 @@ async def test_collapse_autocomplete_stem_match_skips_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -717,6 +730,7 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -766,6 +780,7 @@ async def test_collapse_autocomplete_ambiguous_exact_match_falls_back_to_llm() -skyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -832,6 +847,7 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> Noskyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -904,6 +920,7 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> Noskyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)@@ -976,6 +993,7 @@ async def test_collapse_autocomplete_detached_option_resets_before_llm() -> Noneskyvern_element=skyvern_el,step=_STEP,task=_TASK,+ is_secret_value=False,collapse_autocomplete_fanout_enabled=True,)diff --git a/tests/unit/test_commit_evidence.py b/tests/unit/test_commit_evidence.pynew file mode 100644index 000000000..2bb548ea8--- /dev/null+++ b/tests/unit/test_commit_evidence.py@@ -0,0 +1,661 @@+"""Commit-evidence for the LLM autocomplete option-selection path.++When the agent clicks the LLM-selected autocomplete option, evidence is captured only when the+target control's read-back changes and BOTH the pre-click and post-click values are+boundary-delimited fragments of the clicked option's label — i.e. the observed transition is+selection-specific, not an unrelated blur/masking/validation/restoration transform. The evidence+(the clicked option's label plus the control's post-click value) is serialized into action history+as an observation a later step or the completion verifier can lean on without reopening the+control. A no-op click, an identity display, an unrelated transform, an empty read-back, a failed+capture, or a secret value all leave a bare success with no evidence.+"""++from __future__ import annotations++from datetime import UTC, datetime+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.models import StepStatus+from skyvern.services.action_service import get_action_history+from skyvern.webeye.actions import handler+from skyvern.webeye.actions.actions import ClickAction, InputOrSelectContext, InputTextAction+from skyvern.webeye.actions.responses import ActionResult, ActionSuccess+from tests.unit.helpers import make_organization, make_step, make_task++_NOW = datetime.now(UTC)+_ORG = make_organization(_NOW)+_TASK = make_task(_NOW, _ORG, navigation_payload={"region": "California"})+_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)++_OPTION_TEXT = "Backend Engineer"+_COMMITTED_VALUE = "Backend Engineer (Remote)"++# A composite option whose visible label decorates the identity the user typed+# ("CA - California" clicked, control closes onto "CA - California"). The typed+# probe left "California" in the control, so the click produces a real transition.+_COMPOSITE_LABEL = "CA - California"+_TYPED_PRE = "California"+++def _control(tag: str = "input") -> MagicMock:+ element = MagicMock()+ element.get_tag_name.return_value = tag+ element.get_locator.return_value = MagicMock()+ return element+++async def _run_producer(+ *,+ pre_values: list[str | None],+ option_label: str | None,+ static_element: dict | None = None,+ is_secret_value: bool = False,+ click: AsyncMock | None = None,+ skyvern_frame: MagicMock | None = None,+) -> ActionResult:+ """Drive the real producer with light IO stubs; pre_values feeds get_input_value in order."""+ click = click or AsyncMock()+ if skyvern_frame is None:+ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ identity = {"label": option_label} if option_label is not None else None+ with (+ patch.object(handler, "get_input_value", AsyncMock(side_effect=pre_values)),+ patch.object(handler, "_read_autocomplete_option_identity", AsyncMock(return_value=identity)),+ ):+ return await handler._click_autocomplete_option_with_commit_evidence(+ skyvern_element=_control(),+ option_locator=MagicMock(),+ option_static_element=static_element,+ skyvern_frame=skyvern_frame,+ click=click,+ is_secret_value=is_secret_value,+ )+++async def _history_for(action_and_results: list) -> list[dict]:+ """Run the real get_action_history projection over a hand-built step output."""+ step = MagicMock()+ step.output = SimpleNamespace(actions_and_results=action_and_results)+ task = MagicMock(task_id="tsk_1", organization_id="o_1")+ with patch("skyvern.services.action_service.app") as app_mock:+ app_mock.DATABASE.tasks.get_task_steps = AsyncMock(return_value=[])+ return await get_action_history(task, current_step=step)+++class TestCommitEvidenceModel:+ def test_fields_default_absent(self) -> None:+ r = ActionResult(success=True)+ assert r.committed_option is None+ assert r.committed_value is None++ def test_bare_success_has_no_evidence(self) -> None:+ r = ActionSuccess()+ assert r.committed_option is None+ assert r.committed_value is None++ def test_success_carries_evidence(self) -> None:+ r = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)+ assert r.success is True+ assert r.committed_option == _OPTION_TEXT+ assert r.committed_value == _COMMITTED_VALUE++ def test_evidence_in_str_representation(self) -> None:+ s = str(ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE))+ assert "committed_option=" in s+ assert "committed_value=" in s+++class TestCommitEvidenceGate:+ """The pure transition gate: evidence only on a nonempty, normalized pre→post difference."""++ def test_emits_on_genuine_transition(self) -> None:+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL) == (+ _COMPOSITE_LABEL,+ _COMPOSITE_LABEL,+ )++ def test_emits_when_pre_and_post_are_boundary_fragments_of_composite_label(self) -> None:+ # The motivating shape: the account digits the user typed and the customer-name display the+ # control closes onto are BOTH boundary-delimited fragments of one composite option label.+ assert handler._autocomplete_commit_evidence("12345", "Acme Corp", "12345 - Acme Corp") == (+ "12345 - Acme Corp",+ "Acme Corp",+ )++ def test_emits_state_code_transition(self) -> None:+ # user enters "California"; control closes onto the state code "CA"; both fragments of label.+ assert handler._autocomplete_commit_evidence("California", "CA", "CA - California") == ("CA - California", "CA")++ @pytest.mark.parametrize(+ "pre, post, label",+ [+ # An unrelated blur/formatting transform (phone) yields a post that is not a fragment of the label.+ ("California", "(415) 555-1234", "CA - California"),+ # Restoration to a different stale/invalid value after a failed selection.+ ("California", "Texas", "CA - California"),+ # "CA" merely appears inside "California" — not a boundary-delimited fragment.+ ("California", "CA", "California"),+ # Numeric prefix: "1234" sits inside the digit run "12345", not a boundary fragment.+ ("1234", "Acme Corp", "12345 - Acme Corp"),+ # Punctuation-only post carries no alphanumeric, so it can never be selection-specific.+ ("California", "—", "CA — California"),+ # Blur-time masking of the typed value is not a fragment of the option label.+ ("1234567890", "••••7890", "Account 1234567890"),+ # CJK interior: adjacent CJK chars are word chars, so an interior fragment fails the boundary+ # lookarounds and evidence is withheld (documented fail-closed recall loss).+ ("東京", "東京都", "東京都"),+ ],+ )+ def test_non_selection_transitions_fail_closed(self, pre: str, post: str, label: str) -> None:+ # Each is a nonempty, differing pre->post that the old gate accepted; requiring both to be+ # boundary-delimited fragments of the clicked option label rejects them.+ assert handler._autocomplete_commit_evidence(pre, post, label) is None++ def test_no_evidence_when_post_equals_pre(self) -> None:+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _TYPED_PRE, _COMPOSITE_LABEL) is None++ def test_no_evidence_when_post_normalizes_to_pre(self) -> None:+ assert handler._autocomplete_commit_evidence("California", " california ", _COMPOSITE_LABEL) is None++ def test_no_evidence_when_post_empty(self) -> None:+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, "", _COMPOSITE_LABEL) is None++ def test_no_evidence_when_pre_empty(self) -> None:+ assert handler._autocomplete_commit_evidence("", _COMPOSITE_LABEL, _COMPOSITE_LABEL) is None++ def test_no_evidence_when_label_empty(self) -> None:+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _COMPOSITE_LABEL, "") is None++ @pytest.mark.parametrize(+ "pre, post, label",+ [+ (" ", _COMPOSITE_LABEL, _COMPOSITE_LABEL), # whitespace-only pre normalizes to empty+ (_TYPED_PRE, " ", _COMPOSITE_LABEL), # whitespace-only post normalizes to empty+ (_TYPED_PRE, _COMPOSITE_LABEL, " \t\n "), # whitespace-only label normalizes to empty+ ("\xa0", _COMPOSITE_LABEL, _COMPOSITE_LABEL), # non-breaking space only+ ],+ )+ def test_whitespace_only_fields_fail_closed(self, pre: str, post: str, label: str) -> None:+ # A field that survives the truthiness check but normalizes to empty must not mint evidence.+ assert handler._autocomplete_commit_evidence(pre, post, label) is None++ def test_fields_truncated_after_relation_runs_on_full_strings(self) -> None:+ # pre and post are valid boundary fragments of the full label; the fragment relation is+ # evaluated on the full normalized strings, and only the emitted fields are then capped.+ limit = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS+ pre = _TYPED_PRE+ post = "california " + "y" * (limit + 40)+ label = post + " suffix"+ evidence = handler._autocomplete_commit_evidence(pre, post, label)+ assert evidence is not None+ committed_option, committed_value = evidence+ assert committed_option == label[:limit] + "…"+ assert committed_value == post[:limit] + "…"+++class TestCommitEvidenceProducer:+ """The LLM-path producer: click, then evidence only on a measured control transition."""++ @pytest.mark.asyncio+ async def test_transition_emits_live_label_and_post_value(self) -> None:+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],+ option_label=_COMPOSITE_LABEL,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option == _COMPOSITE_LABEL+ assert result.committed_value == _COMPOSITE_LABEL++ @pytest.mark.asyncio+ async def test_noop_click_leaves_pre_unchanged_no_evidence(self) -> None:+ # Codex r3881398576 regression: the click does not commit, control still holds the probe.+ click = AsyncMock()+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _TYPED_PRE, _TYPED_PRE],+ option_label=_COMPOSITE_LABEL,+ click=click,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_masked_pre_differs_from_typed_text_noop_no_evidence(self) -> None:+ # The measured pre-click value is a masked form, not the typed text; a no-op click that+ # leaves that masked form in place must not emit (proves measured pre, not typed text).+ result = await _run_producer(+ pre_values=["(•••) masked", "(•••) masked", "(•••) masked"],+ option_label=_COMPOSITE_LABEL,+ )+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_readback_changes_to_value_not_in_label_no_evidence(self) -> None:+ # The click flips the control to an unrelated reformatted value (blur masking / validation /+ # restoration) that is not a fragment of the clicked option label; a differing post is no+ # longer accepted as selection-specific, so the click stays a bare success.+ result = await _run_producer(+ pre_values=[_TYPED_PRE, "(415) 555-1234"],+ option_label=_COMPOSITE_LABEL,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_empty_post_no_evidence(self) -> None:+ result = await _run_producer(+ pre_values=[_TYPED_PRE, ""],+ option_label=_COMPOSITE_LABEL,+ )+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_optimistic_label_that_reverts_after_settle_no_evidence(self) -> None:+ # Codex r3892628198 regression: the control optimistically paints the selected label (a real+ # transition off the typed probe, so the candidate gate passes) and then reverts to the probe+ # after async validation/rerender. The first post read looked committed; the settled reread+ # proves it was transient, so no stale evidence may be recorded.+ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _TYPED_PRE],+ option_label=_COMPOSITE_LABEL,+ skyvern_frame=skyvern_frame,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_timer_style_revert_within_settle_floor_withholds_evidence(self) -> None:+ # Codex r3892846834 (P1): a non-animation revert — a ~300ms timer or async validation, not a+ # CSS/Web Animation — leaves document.getAnimations() empty, so safe_wait_for_animation_end has+ # nothing to await and only its before_wait_sec floor holds the confirm read back. A virtual+ # clock advanced solely by that floor (the settle's own before_wait_sec) proves the outcome at+ # an entry point above the wait: without a bounded floor the settled reread fires while the+ # optimistic label is still painted and commits a value that vanishes at 300ms; a floor past+ # 300ms observes the reverted probe and withholds evidence.+ revert_at_ms = 300.0+ clock = SimpleNamespace(ms=0.0)+ reads = iter([_TYPED_PRE, _COMPOSITE_LABEL])++ async def settle(*, before_wait_sec: float = 0, caller: str = "") -> None:+ clock.ms += before_wait_sec * 1000++ def read_control(*_args: object, **_kwargs: object) -> str:+ try:+ return next(reads)+ except StopIteration:+ return _TYPED_PRE if clock.ms >= revert_at_ms else _COMPOSITE_LABEL++ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock(side_effect=settle)+ with (+ patch.object(handler, "get_input_value", AsyncMock(side_effect=read_control)),+ patch.object(+ handler, "_read_autocomplete_option_identity", AsyncMock(return_value={"label": _COMPOSITE_LABEL})+ ),+ ):+ result = await handler._click_autocomplete_option_with_commit_evidence(+ skyvern_element=_control(),+ option_locator=MagicMock(),+ option_static_element=None,+ skyvern_frame=skyvern_frame,+ click=AsyncMock(),+ is_secret_value=False,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_confirm_read_failure_after_candidate_no_evidence(self) -> None:+ # The candidate gate passes on the first post read, but the settled confirm read fails+ # (returns None). A value that cannot be reconfirmed after settling must fail closed.+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, RuntimeError("read boom")],+ option_label=_COMPOSITE_LABEL,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_confirm_drift_to_other_value_no_evidence(self) -> None:+ # The candidate passes, but the settled reread drifts to a different value than the first+ # post read. Emission requires the transitioned value to equal the first post, not merely to+ # differ from pre, so a drift withholds evidence.+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, "Texas"],+ option_label=_COMPOSITE_LABEL,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None++ @pytest.mark.asyncio+ async def test_stable_transition_survives_settle_emits(self) -> None:+ # The transitioned value is reread identically after the single bounded settle, so the+ # selection is stable and the evidence is recorded.+ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],+ option_label=_COMPOSITE_LABEL,+ skyvern_frame=skyvern_frame,+ )+ assert result.committed_option == _COMPOSITE_LABEL+ assert result.committed_value == _COMPOSITE_LABEL+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_label_read_failure_click_still_succeeds_no_evidence(self) -> None:+ click = AsyncMock()+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL],+ option_label=None, # identity read returns None+ static_element=None, # no scraped fallback text either+ click=click,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_static_element_text_is_label_fallback(self) -> None:+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],+ option_label=None,+ static_element={"text": _COMPOSITE_LABEL},+ )+ assert result.committed_option == _COMPOSITE_LABEL+ assert result.committed_value == _COMPOSITE_LABEL++ @pytest.mark.asyncio+ async def test_secret_value_never_emits_and_skips_reads(self) -> None:+ click = AsyncMock()+ skyvern_frame = MagicMock()+ get_input_value = AsyncMock(return_value=_COMPOSITE_LABEL)+ identity = AsyncMock(return_value={"label": _COMPOSITE_LABEL})+ with (+ patch.object(handler, "get_input_value", get_input_value),+ patch.object(handler, "_read_autocomplete_option_identity", identity),+ ):+ result = await handler._click_autocomplete_option_with_commit_evidence(+ skyvern_element=_control(),+ option_locator=MagicMock(),+ option_static_element={"text": _COMPOSITE_LABEL},+ skyvern_frame=skyvern_frame,+ click=click,+ is_secret_value=True,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()+ get_input_value.assert_not_called()+ identity.assert_not_called()++ @pytest.mark.asyncio+ async def test_overlong_label_and_post_capped(self) -> None:+ limit = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS+ long_post = "california " + "p" * (limit + 50)+ long_label = long_post + " suffix"+ result = await _run_producer(+ pre_values=[_TYPED_PRE, long_post, long_post],+ option_label=long_label,+ )+ assert result.committed_option == long_label[:limit] + "…"+ assert result.committed_value == long_post[:limit] + "…"++ @pytest.mark.asyncio+ async def test_capture_exception_after_click_stays_success(self) -> None:+ click = AsyncMock()+ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ with (+ patch.object(handler, "get_input_value", AsyncMock(side_effect=[_TYPED_PRE, _COMPOSITE_LABEL])),+ patch.object(+ handler, "_read_autocomplete_option_identity", AsyncMock(return_value={"label": _COMPOSITE_LABEL})+ ),+ patch.object(handler, "_autocomplete_commit_evidence", side_effect=RuntimeError("boom")),+ ):+ result = await handler._click_autocomplete_option_with_commit_evidence(+ skyvern_element=_control(),+ option_locator=MagicMock(),+ option_static_element=None,+ skyvern_frame=skyvern_frame,+ click=click,+ is_secret_value=False,+ )+ assert isinstance(result, ActionSuccess)+ assert result.success is True+ assert result.committed_option is None+ click.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_failed_click_propagates_not_swallowed(self) -> None:+ click = AsyncMock(side_effect=RuntimeError("click failed"))+ skyvern_frame = MagicMock()+ identity = AsyncMock(return_value={"label": _COMPOSITE_LABEL})+ with (+ patch.object(handler, "get_input_value", AsyncMock(return_value=_TYPED_PRE)),+ patch.object(handler, "_read_autocomplete_option_identity", identity),+ pytest.raises(RuntimeError),+ ):+ await handler._click_autocomplete_option_with_commit_evidence(+ skyvern_element=_control(),+ option_locator=MagicMock(),+ option_static_element=None,+ skyvern_frame=skyvern_frame,+ click=click,+ is_secret_value=False,+ )+++class TestWrapperPreservesEvidence:+ """input_or_auto_complete_input must forward the producer's enriched result, not a bare success."""++ @pytest.mark.asyncio+ async def test_wrapper_forwards_enriched_result_first_attempt(self) -> None:+ enriched = ActionSuccess(committed_option=_COMPOSITE_LABEL, committed_value=_COMPOSITE_LABEL)+ producer_result = handler.AutoCompletionResult(action_result=enriched)+ context = SimpleNamespace(is_location_input=False, is_search_bar=False)+ element = MagicMock()+ element.get_id.return_value = "region-combobox"+ with patch.object(handler, "choose_auto_completion_dropdown", AsyncMock(return_value=producer_result)):+ returned = await handler.input_or_auto_complete_input(+ input_or_select_context=context,+ scraped_page=MagicMock(),+ page=MagicMock(),+ dom=MagicMock(),+ text=_TYPED_PRE,+ skyvern_element=element,+ step=MagicMock(),+ task=MagicMock(),+ is_secret_value=False,+ )+ assert returned is not None+ assert returned.committed_option == _COMPOSITE_LABEL+ assert returned.committed_value == _COMPOSITE_LABEL+++class TestChooseDropdownSeamWiring:+ """The LLM-selected-option branch of the real choose_auto_completion_dropdown routes the+ clicked option through the commit-evidence producer. A rendered-autocomplete option (keyed+ by id+text, not a native <select>) is chosen by the LLM; a measured control transition emits+ evidence, and a secret value suppresses it while the click still lands. Locks the seam at the+ choose→producer hop (control-vs-option element, is_secret_value forwarding)."""++ @staticmethod+ def _wire(*, control_reads: list[str], option_label: str) -> dict:+ option = {"id": "opt-ca", "tag": "div", "text": option_label}++ option_locator = MagicMock()+ option_locator.count = AsyncMock(return_value=1)+ option_locator.element_handle = AsyncMock(return_value=MagicMock())++ frame = MagicMock()+ frame.locator.return_value = option_locator++ control_locator = MagicMock()+ control_locator.input_value = AsyncMock(side_effect=list(control_reads))++ control = MagicMock()+ control.get_id.return_value = "region-combobox"+ control.get_frame.return_value = frame+ control.get_frame_id.return_value = "frame-1"+ control.is_interactable.return_value = True+ control.get_tag_name.return_value = "input"+ control.get_locator.return_value = control_locator+ control.press_fill = AsyncMock()+ control.input_clear = AsyncMock()+ control.is_visible = AsyncMock(return_value=True)+ control.get_element_handler = AsyncMock(return_value=MagicMock())++ skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())+ skyvern_frame.read_autocomplete_option_identity = AsyncMock(return_value={"index": 0, "label": option_label})++ inc = MagicMock()+ inc.start_listen_dom_increment = AsyncMock()+ inc.stop_listen_dom_increment = AsyncMock()+ inc.get_incremental_element_tree = AsyncMock(return_value=[dict(option)])+ inc.id_to_element_dict = {"opt-ca": dict(option)}+ inc.build_html_tree.return_value = "<div>opt</div>"++ return {+ "control": control,+ "control_locator": control_locator,+ "skyvern_frame": skyvern_frame,+ "inc": inc,+ }++ async def _run_choose(self, wired: dict, *, is_secret_value: bool):+ clicked_option = MagicMock(scroll_into_view=AsyncMock(), click=AsyncMock())+ llm_response = {"relevance_float": 0.95, "id": "opt-ca", "direct_searching": False, "reasoning": "match"}+ with (+ patch(+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",+ new=AsyncMock(return_value=wired["skyvern_frame"]),+ ),+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=wired["inc"]),+ patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=clicked_option),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ ):+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)+ mock_app.AGENT_FUNCTION = MagicMock()+ mock_prompt.load_prompt.return_value = "prompt"+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)+ result = await handler.choose_auto_completion_dropdown(+ context=InputOrSelectContext(field="Region", is_search_bar=False),+ page=MagicMock(),+ scraped_page=MagicMock(),+ dom=MagicMock(),+ text=_TYPED_PRE,+ skyvern_element=wired["control"],+ step=_STEP,+ task=_TASK,+ is_secret_value=is_secret_value,+ )+ return result.action_result, clicked_option++ @pytest.mark.asyncio+ async def test_llm_selected_option_transition_emits_evidence(self) -> None:+ wired = self._wire(+ control_reads=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL], option_label=_COMPOSITE_LABEL+ )+ action_result, clicked_option = await self._run_choose(wired, is_secret_value=False)++ assert isinstance(action_result, ActionSuccess)+ assert action_result.committed_option == _COMPOSITE_LABEL+ assert action_result.committed_value == _COMPOSITE_LABEL+ clicked_option.click.assert_awaited_once()+ # Evidence must come from the control's read-back (pre, post, settled confirm), not the option node.+ assert wired["control_locator"].input_value.await_count == 3++ @pytest.mark.asyncio+ async def test_secret_value_suppresses_evidence_but_click_lands(self) -> None:+ wired = self._wire(control_reads=[_TYPED_PRE, _COMPOSITE_LABEL], option_label=_COMPOSITE_LABEL)+ action_result, clicked_option = await self._run_choose(wired, is_secret_value=True)++ assert isinstance(action_result, ActionSuccess)+ assert action_result.committed_option is None+ assert action_result.committed_value is None+ clicked_option.click.assert_awaited_once()+ wired["control_locator"].input_value.assert_not_called()+ wired["skyvern_frame"].read_autocomplete_option_identity.assert_not_called()+++class TestControlReadTimeoutBound:+ """The commit-evidence control read is bounded by a small explicit timeout so the added settle+ reread cannot inherit Playwright's 30s default wait; the global get_input_value stays unbounded."""++ @pytest.mark.asyncio+ async def test_control_read_bounded_by_explicit_timeout(self) -> None:+ locator = MagicMock()+ locator.input_value = AsyncMock(return_value=_TYPED_PRE)+ element = MagicMock()+ element.get_tag_name.return_value = "input"+ element.get_locator.return_value = locator++ result = await handler._read_autocomplete_control_value(element)++ assert result == _TYPED_PRE+ locator.input_value.assert_awaited_once_with(timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)++ @pytest.mark.asyncio+ async def test_get_input_value_unbounded_by_default(self) -> None:+ # Other callers keep stock Playwright behavior: no timeout is threaded unless asked for.+ locator = MagicMock()+ locator.input_value = AsyncMock(return_value="x")++ await handler.get_input_value("input", locator)++ locator.input_value.assert_awaited_once_with()+++class TestCommitEvidenceSerialization:+ @pytest.mark.asyncio+ async def test_positive_readback_serialized_into_history(self) -> None:+ action = InputTextAction(element_id="job-title", text="Backend")+ result = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)++ history = await _history_for([(action, [result])])++ assert len(history) == 1+ assert history[0]["result"]["committed_option"] == _OPTION_TEXT+ assert history[0]["result"]["committed_value"] == _COMMITTED_VALUE++ @pytest.mark.asyncio+ async def test_unverified_success_omits_evidence(self) -> None:+ action = InputTextAction(element_id="job-title", text="Backend")+ history = await _history_for([(action, [ActionSuccess()])])++ assert "committed_option" not in history[0]["result"]+ assert "committed_value" not in history[0]["result"]++ @pytest.mark.asyncio+ async def test_adjacent_action_history_unchanged(self) -> None:+ target = InputTextAction(element_id="job-title", text="Backend")+ target_result = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)+ neighbor = ClickAction(element_id="submit")+ neighbor_result = ActionSuccess()++ history = await _history_for([(neighbor, [neighbor_result]), (target, [target_result])])++ assert "committed_option" not in history[0]["result"]+ assert "committed_value" not in history[0]["result"]+ assert history[1]["result"]["committed_option"] == _OPTION_TEXT+ assert history[1]["result"]["committed_value"] == _COMMITTED_VALUEdiff --git a/tests/unit/test_element_engine_selection_threading.py b/tests/unit/test_element_engine_selection_threading.pyindex ae139c90f..7f2663fb4 100644--- a/tests/unit/test_element_engine_selection_threading.py+++ b/tests/unit/test_element_engine_selection_threading.py@@ -204,6 +204,7 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: pystep=MagicMock(),task=task,preserved_elements=[{"id": "preserved"}],+ is_secret_value=False,)await handler._reset_autocomplete_for_llm_fallback(current_incremental_scraped=incremental,diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex fa3d9d66c..3301fd608 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -471,6 +471,7 @@ async def test_auto_completion_call_path_forwards_committed_value_to_shadow_matcskyvern_element=skyvern_element,step=SimpleNamespace(step_id="step-1"),task=_shadow_task(),+ is_secret_value=False,)events = [log for log in logs if log.get("event") == "select_shadow_match"]