Skip to content

Automate browser based workflows with AI

Emit autocomplete commit evidence on the LLM selection path via a measured control transition (#8345)

LawyZhengfa0a7783bc02
diff --git a/skyvern/services/action_service.py b/skyvern/services/action_service.py
index 754e1674b..9a7a791c0 100644
--- a/skyvern/services/action_service.py
+++ b/skyvern/services/action_service.py
@@ -55,6 +55,8 @@ async def get_action_history(
"upload_file_triggered",
"needs_followup",
"followup_message",
+ "committed_option",
+ "committed_value",
},
),
}
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index df76ef9d0..ac83b45de 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -680,6 +680,148 @@ async def _verify_autocomplete_input_readback(
return False
+def _is_boundary_fragment(fragment: str, whole: str) -> bool:
+ """Whether ``fragment`` occurs in ``whole`` as a word-boundary-delimited contiguous run.
+
+ Both are expected to already be normalized. A fragment carrying no alphanumeric character can
+ never anchor a meaningful boundary match, so it is rejected. Default Unicode ``\\w`` semantics
+ apply, so accented Latin behaves and unsegmented CJK interiors fail closed.
+ """
+ if not any(ch.isalnum() for ch in fragment):
+ return False
+ return re.search(rf"(?<!\w){re.escape(fragment)}(?!\w)", whole) is not None
+
+
+def _autocomplete_commit_evidence(
+ pre_value: str | None,
+ post_value: str | None,
+ option_label: str | None,
+) -> tuple[str, str] | None:
+ """Observational commit evidence for an autocomplete selection, or None.
+
+ Emits ``(committed_option, committed_value)`` — both truncated to the shared field cap — only
+ when the clicked option's label and both control read-backs are nonempty *after normalization*,
+ the normalized post-click value differs from the normalized pre-click value, and BOTH the
+ normalized pre and post are boundary-delimited fragments of the normalized option label. That
+ last relation is what makes the transition selection-specific: it rejects unrelated blur,
+ masking, formatting, validation, and restoration transforms whose output is not a fragment of
+ the clicked option. The relation runs on the full normalized strings before any truncation.
+ Whitespace-only fields normalize to empty and fail closed; equality (a no-op or highlight-only
+ click) yields None. Secret suppression is the caller's responsibility.
+ """
+ normalized_pre = _normalize_select_shadow_text(pre_value)
+ normalized_post = _normalize_select_shadow_text(post_value)
+ normalized_label = _normalize_select_shadow_text(option_label)
+ if not normalized_label or not normalized_pre or not normalized_post:
+ return None
+ if normalized_post == normalized_pre:
+ return None
+ if not _is_boundary_fragment(normalized_pre, normalized_label):
+ return None
+ if not _is_boundary_fragment(normalized_post, normalized_label):
+ return None
+ committed_option = _truncate_select_shadow_field(option_label)
+ committed_value = _truncate_select_shadow_field(post_value)
+ if not committed_option or not committed_value:
+ return None
+ return committed_option, committed_value
+
+
+async def _read_autocomplete_control_value(
+ skyvern_element: SkyvernElement,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> str | None:
+ try:
+ return await get_input_value(
+ skyvern_element.get_tag_name(),
+ skyvern_element.get_locator(),
+ engine_selection=engine_selection,
+ read_timeout_ms=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+ except Exception:
+ LOG.info("Failed to read autocomplete control value for commit evidence", exc_info=True)
+ return None
+
+
+async def _read_clicked_option_label(
+ *,
+ skyvern_frame: SkyvernFrame,
+ option_locator: Locator,
+ option_static_element: dict | None,
+) -> str | None:
+ identity = await _read_autocomplete_option_identity(skyvern_frame=skyvern_frame, locator=option_locator)
+ label = identity.get("label") if identity else None
+ if not label and option_static_element:
+ # Fall back to the scraped node text; it is tied to this exact option by construction.
+ label = option_static_element.get("text")
+ label = (label or "").strip()
+ return label or None
+
+
+async def _click_autocomplete_option_with_commit_evidence(
+ *,
+ skyvern_element: SkyvernElement,
+ option_locator: Locator,
+ option_static_element: dict | None,
+ skyvern_frame: SkyvernFrame,
+ click: Callable[[], Awaitable[None]],
+ is_secret_value: bool,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> ActionResult:
+ """Click the LLM-selected option and return ``ActionSuccess``, enriched with commit evidence
+ only when the target control's read-back changes into a value that — like the pre-click value —
+ is a boundary-delimited fragment of the clicked option's label (see
+ ``_autocomplete_commit_evidence``) AND that transitioned value survives a single bounded settle
+ reread. The settle guards against an optimistic control that paints the selected label and then
+ reverts it after async validation/rerender, which would otherwise commit a transient value.
+
+ Evidence capture is best-effort and fail-closed: a failed read (before, after, or on the settle
+ reread), a missing label, an unchanged control, an unrelated transform, a value that drifts on
+ the settled reread, or a secret value all leave a bare ``ActionSuccess``. Only the click itself
+ can fail the action — every evidence read is exception-isolated so a successful click never
+ regresses to ``ActionFailure`` because capture failed.
+ """
+ option_label: str | None = None
+ pre_value: str | None = None
+ if not is_secret_value:
+ try:
+ option_label = await _read_clicked_option_label(
+ skyvern_frame=skyvern_frame,
+ option_locator=option_locator,
+ option_static_element=option_static_element,
+ )
+ except Exception:
+ LOG.info("Failed to read clicked autocomplete option label for commit evidence", exc_info=True)
+ pre_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)
+
+ await click()
+
+ if is_secret_value:
+ return ActionSuccess()
+
+ try:
+ post_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)
+ evidence = _autocomplete_commit_evidence(pre_value, post_value, option_label)
+ if evidence is None:
+ return ActionSuccess()
+ # The candidate transition passed, but an optimistic control can paint the selected label and
+ # then revert it after async validation/rerender. Settle once (no loop) and reconfirm the
+ # value; only record evidence when the settled reread still holds the first post value, so a
+ # transient paint cannot be committed as stale evidence.
+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.commit")
+ confirm_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)
+ if confirm_value is None or _normalize_select_shadow_text(confirm_value) != _normalize_select_shadow_text(
+ post_value
+ ):
+ return ActionSuccess()
+ except Exception:
+ LOG.info("Autocomplete commit-evidence capture failed after a successful click", exc_info=True)
+ return ActionSuccess()
+
+ committed_option, committed_value = evidence
+ return ActionSuccess(committed_option=committed_option, committed_value=committed_value)
+
+
async def _reset_autocomplete_for_llm_fallback(
*,
current_incremental_scraped: IncrementalScrapePage,
@@ -7472,6 +7614,7 @@ async def _handle_input_text_action(
task=task,
action=action,
collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,
+ is_secret_value=is_secret_value,
):
auto_complete_hacky_flag = False
return [result]
@@ -9912,6 +10055,8 @@ async def choose_auto_completion_dropdown(
is_location_input: bool = False,
collapse_autocomplete_fanout_enabled: bool = False,
action: InputTextAction | None = None,
+ *,
+ is_secret_value: bool,
) -> AutoCompletionResult:
preserved_elements = preserved_elements or []
clear_input = True
@@ -10234,8 +10379,20 @@ async def choose_auto_completion_dropdown(
static_element=incremental_scraped.id_to_element_dict.get(element_id, {}),
engine_selection=engine_selection,
)
- await selected_element.scroll_into_view()
- await selected_element.click(page=page, engine_selection=engine_selection)
+
+ async def _click_selected_option() -> None:
+ await selected_element.scroll_into_view()
+ await selected_element.click(page=page, engine_selection=engine_selection)
+
+ result.action_result = await _click_autocomplete_option_with_commit_evidence(
+ skyvern_element=skyvern_element,
+ option_locator=locator,
+ option_static_element=incremental_scraped.id_to_element_dict.get(element_id),
+ skyvern_frame=skyvern_frame,
+ click=_click_selected_option,
+ is_secret_value=is_secret_value,
+ engine_selection=engine_selection,
+ )
clear_input = False
return result
@@ -10288,6 +10445,8 @@ async def input_or_auto_complete_input(
task: Task,
action: InputTextAction | None = None,
collapse_autocomplete_fanout_enabled: bool = False,
+ *,
+ is_secret_value: bool,
) -> ActionResult | None:
LOG.info(
"Trigger auto completion",
@@ -10329,9 +10488,10 @@ async def input_or_auto_complete_input(
is_location_input=is_location,
collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,
action=action,
+ is_secret_value=is_secret_value,
)
if isinstance(result.action_result, ActionSuccess):
- return ActionSuccess()
+ return result.action_result
if input_or_select_context.is_search_bar:
LOG.info(
@@ -10397,9 +10557,10 @@ async def input_or_auto_complete_input(
is_location_input=is_location,
collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,
action=action,
+ is_secret_value=is_secret_value,
)
if isinstance(result.action_result, ActionSuccess):
- return ActionSuccess()
+ return result.action_result
tried_values.append(value)
whole_new_elements.extend(result.incremental_elements)
@@ -13828,20 +13989,25 @@ async def click_listbox_option(
async def get_input_value(
- tag_name: str, locator: Locator, engine_selection: BrowserEngineSelection | None = None
+ tag_name: str,
+ locator: Locator,
+ engine_selection: BrowserEngineSelection | None = None,
+ read_timeout_ms: float | None = None,
) -> str | None:
# input_value() rejects non-<input>/<textarea>/<select> nodes and inner_text() rejects
# non-HTMLElement nodes; the live node can disagree with the scraped tag_name after a
# re-render. Treat an incompatible read as "value unknown" so the caller's own
# element-type classification runs instead of a raw driver exception escaping here. The
# incompatible-node identity is matched against THIS run's selected engine; missing selection
- # keeps the stock Playwright identity (unchanged default).
+ # keeps the stock Playwright identity (unchanged default). read_timeout_ms is opt-in: when unset
+ # the read keeps Playwright's default wait; callers that must not stall pass an explicit bound.
+ read_kwargs = {} if read_timeout_ms is None else {"timeout": read_timeout_ms}
try:
if tag_name in COMMON_INPUT_TAGS:
- return await locator.input_value()
+ return await locator.input_value(**read_kwargs)
# for span, div, p or other tags:
# we need to trim the unicode space for these tags
- return (await locator.inner_text()).replace("\xa0", " ").strip()
+ return (await locator.inner_text(**read_kwargs)).replace("\xa0", " ").strip()
except Exception as exc:
if not _is_selected_engine_error(exc, engine_selection):
raise
diff --git a/skyvern/webeye/actions/responses.py b/skyvern/webeye/actions/responses.py
index 009dd8d84..8497967b9 100644
--- a/skyvern/webeye/actions/responses.py
+++ b/skyvern/webeye/actions/responses.py
@@ -27,6 +27,12 @@ class ActionResult(BaseModel):
# suppressed it because the control already held the desired state. Both shapes stop the raw
# click, so callers mirroring the setup contract have no other way to tell them apart.
setup_performed: bool = False
+ # Observational commit evidence for an autocomplete selection: the clicked option's label and
+ # the control value observed immediately afterward. Set together only when both are
+ # boundary-delimited fragments of the option label (a selection-specific transition); both stay
+ # None on a no-op, unrelated transform, ambiguous, failed, or secret selection.
+ committed_option: str | None = None
+ committed_value: str | None = None
def __str__(self) -> str:
results = [f"ActionResult(success={self.success}"]
@@ -55,6 +61,10 @@ class ActionResult(BaseModel):
results.append(f"interacted_with_parent={self.interacted_with_parent}")
if self.skip_remaining_actions is not None:
results.append(f"skip_remaining_actions={self.skip_remaining_actions}")
+ if self.committed_option is not None:
+ results.append(f"committed_option={self.committed_option}")
+ if self.committed_value is not None:
+ results.append(f"committed_value={self.committed_value}")
return ", ".join(results) + ")"
@@ -71,6 +81,8 @@ class ActionSuccess(ActionResult):
interacted_with_sibling: bool = False,
interacted_with_parent: bool = False,
setup_performed: bool = False,
+ committed_option: str | None = None,
+ committed_value: str | None = None,
):
super().__init__(
success=True,
@@ -80,6 +92,8 @@ class ActionSuccess(ActionResult):
interacted_with_sibling=interacted_with_sibling,
interacted_with_parent=interacted_with_parent,
setup_performed=setup_performed,
+ committed_option=committed_option,
+ committed_value=committed_value,
)
diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.py
index 6d9b09b06..1932f100a 100644
--- a/tests/unit/test_auto_completion_discover_fallback.py
+++ b/tests/unit/test_auto_completion_discover_fallback.py
@@ -595,6 +595,7 @@ async def test_input_or_auto_complete_calls_discover_fallback_on_failure() -> No
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
)
assert isinstance(result, ActionSuccess)
@@ -636,6 +637,7 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
)
# Search bar returns None early, never reaching discover fallback
diff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.py
index 487beb26a..b3b985afb 100644
--- a/tests/unit/test_auto_completion_location.py
+++ b/tests/unit/test_auto_completion_location.py
@@ -129,6 +129,7 @@ async def test_location_single_option_skips_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -173,6 +174,7 @@ async def test_location_whitespace_normalized_still_matches() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -223,6 +225,7 @@ async def test_location_multiple_options_calls_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -272,6 +275,7 @@ async def test_non_location_single_option_calls_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=False,
)
@@ -308,6 +312,7 @@ async def test_location_fast_path_returns_action_success() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -358,6 +363,7 @@ async def test_location_fast_path_element_not_in_dom_falls_through() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -395,6 +401,7 @@ async def test_dropdown_cleanup_clear_failure_does_not_clobber_result() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
)
skyvern_el.input_clear.assert_awaited_once()
@@ -426,6 +433,7 @@ async def test_input_or_auto_complete_passes_is_location_input() -> None:
skyvern_element=_mock_skyvern_element(),
step=_STEP,
task=_TASK,
+ is_secret_value=False,
)
assert isinstance(result, ActionSuccess)
@@ -452,6 +460,7 @@ async def test_input_or_auto_complete_passes_false_for_non_location() -> None:
skyvern_element=_mock_skyvern_element(),
step=_STEP,
task=_TASK,
+ is_secret_value=False,
)
assert isinstance(result, ActionSuccess)
@@ -544,6 +553,7 @@ async def test_location_no_results_option_falls_through_to_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -593,6 +603,7 @@ async def test_location_unrelated_option_falls_through_to_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
is_location_input=True,
)
@@ -628,6 +639,7 @@ async def test_collapse_autocomplete_exact_match_skips_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -667,6 +679,7 @@ async def test_collapse_autocomplete_stem_match_skips_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -717,6 +730,7 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -766,6 +780,7 @@ async def test_collapse_autocomplete_ambiguous_exact_match_falls_back_to_llm() -
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -832,6 +847,7 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> No
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -904,6 +920,7 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> No
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
@@ -976,6 +993,7 @@ async def test_collapse_autocomplete_detached_option_resets_before_llm() -> None
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_secret_value=False,
collapse_autocomplete_fanout_enabled=True,
)
diff --git a/tests/unit/test_commit_evidence.py b/tests/unit/test_commit_evidence.py
new file mode 100644
index 000000000..2bb548ea8
--- /dev/null
+++ b/tests/unit/test_commit_evidence.py
@@ -0,0 +1,661 @@
+"""Commit-evidence for the LLM autocomplete option-selection path.
+
+When the agent clicks the LLM-selected autocomplete option, evidence is captured only when the
+target control's read-back changes and BOTH the pre-click and post-click values are
+boundary-delimited fragments of the clicked option's label — i.e. the observed transition is
+selection-specific, not an unrelated blur/masking/validation/restoration transform. The evidence
+(the clicked option's label plus the control's post-click value) is serialized into action history
+as an observation a later step or the completion verifier can lean on without reopening the
+control. A no-op click, an identity display, an unrelated transform, an empty read-back, a failed
+capture, or a secret value all leave a bare success with no evidence.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.services.action_service import get_action_history
+from skyvern.webeye.actions import handler
+from skyvern.webeye.actions.actions import ClickAction, InputOrSelectContext, InputTextAction
+from skyvern.webeye.actions.responses import ActionResult, ActionSuccess
+from tests.unit.helpers import make_organization, make_step, make_task
+
+_NOW = datetime.now(UTC)
+_ORG = make_organization(_NOW)
+_TASK = make_task(_NOW, _ORG, navigation_payload={"region": "California"})
+_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)
+
+_OPTION_TEXT = "Backend Engineer"
+_COMMITTED_VALUE = "Backend Engineer (Remote)"
+
+# A composite option whose visible label decorates the identity the user typed
+# ("CA - California" clicked, control closes onto "CA - California"). The typed
+# probe left "California" in the control, so the click produces a real transition.
+_COMPOSITE_LABEL = "CA - California"
+_TYPED_PRE = "California"
+
+
+def _control(tag: str = "input") -> MagicMock:
+ element = MagicMock()
+ element.get_tag_name.return_value = tag
+ element.get_locator.return_value = MagicMock()
+ return element
+
+
+async def _run_producer(
+ *,
+ pre_values: list[str | None],
+ option_label: str | None,
+ static_element: dict | None = None,
+ is_secret_value: bool = False,
+ click: AsyncMock | None = None,
+ skyvern_frame: MagicMock | None = None,
+) -> ActionResult:
+ """Drive the real producer with light IO stubs; pre_values feeds get_input_value in order."""
+ click = click or AsyncMock()
+ if skyvern_frame is None:
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ identity = {"label": option_label} if option_label is not None else None
+ with (
+ patch.object(handler, "get_input_value", AsyncMock(side_effect=pre_values)),
+ patch.object(handler, "_read_autocomplete_option_identity", AsyncMock(return_value=identity)),
+ ):
+ return await handler._click_autocomplete_option_with_commit_evidence(
+ skyvern_element=_control(),
+ option_locator=MagicMock(),
+ option_static_element=static_element,
+ skyvern_frame=skyvern_frame,
+ click=click,
+ is_secret_value=is_secret_value,
+ )
+
+
+async def _history_for(action_and_results: list) -> list[dict]:
+ """Run the real get_action_history projection over a hand-built step output."""
+ step = MagicMock()
+ step.output = SimpleNamespace(actions_and_results=action_and_results)
+ task = MagicMock(task_id="tsk_1", organization_id="o_1")
+ with patch("skyvern.services.action_service.app") as app_mock:
+ app_mock.DATABASE.tasks.get_task_steps = AsyncMock(return_value=[])
+ return await get_action_history(task, current_step=step)
+
+
+class TestCommitEvidenceModel:
+ def test_fields_default_absent(self) -> None:
+ r = ActionResult(success=True)
+ assert r.committed_option is None
+ assert r.committed_value is None
+
+ def test_bare_success_has_no_evidence(self) -> None:
+ r = ActionSuccess()
+ assert r.committed_option is None
+ assert r.committed_value is None
+
+ def test_success_carries_evidence(self) -> None:
+ r = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)
+ assert r.success is True
+ assert r.committed_option == _OPTION_TEXT
+ assert r.committed_value == _COMMITTED_VALUE
+
+ def test_evidence_in_str_representation(self) -> None:
+ s = str(ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE))
+ assert "committed_option=" in s
+ assert "committed_value=" in s
+
+
+class TestCommitEvidenceGate:
+ """The pure transition gate: evidence only on a nonempty, normalized pre→post difference."""
+
+ def test_emits_on_genuine_transition(self) -> None:
+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL) == (
+ _COMPOSITE_LABEL,
+ _COMPOSITE_LABEL,
+ )
+
+ def test_emits_when_pre_and_post_are_boundary_fragments_of_composite_label(self) -> None:
+ # The motivating shape: the account digits the user typed and the customer-name display the
+ # control closes onto are BOTH boundary-delimited fragments of one composite option label.
+ assert handler._autocomplete_commit_evidence("12345", "Acme Corp", "12345 - Acme Corp") == (
+ "12345 - Acme Corp",
+ "Acme Corp",
+ )
+
+ def test_emits_state_code_transition(self) -> None:
+ # user enters "California"; control closes onto the state code "CA"; both fragments of label.
+ assert handler._autocomplete_commit_evidence("California", "CA", "CA - California") == ("CA - California", "CA")
+
+ @pytest.mark.parametrize(
+ "pre, post, label",
+ [
+ # An unrelated blur/formatting transform (phone) yields a post that is not a fragment of the label.
+ ("California", "(415) 555-1234", "CA - California"),
+ # Restoration to a different stale/invalid value after a failed selection.
+ ("California", "Texas", "CA - California"),
+ # "CA" merely appears inside "California" — not a boundary-delimited fragment.
+ ("California", "CA", "California"),
+ # Numeric prefix: "1234" sits inside the digit run "12345", not a boundary fragment.
+ ("1234", "Acme Corp", "12345 - Acme Corp"),
+ # Punctuation-only post carries no alphanumeric, so it can never be selection-specific.
+ ("California", "—", "CA — California"),
+ # Blur-time masking of the typed value is not a fragment of the option label.
+ ("1234567890", "••••7890", "Account 1234567890"),
+ # CJK interior: adjacent CJK chars are word chars, so an interior fragment fails the boundary
+ # lookarounds and evidence is withheld (documented fail-closed recall loss).
+ ("東京", "東京都", "東京都"),
+ ],
+ )
+ def test_non_selection_transitions_fail_closed(self, pre: str, post: str, label: str) -> None:
+ # Each is a nonempty, differing pre->post that the old gate accepted; requiring both to be
+ # boundary-delimited fragments of the clicked option label rejects them.
+ assert handler._autocomplete_commit_evidence(pre, post, label) is None
+
+ def test_no_evidence_when_post_equals_pre(self) -> None:
+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _TYPED_PRE, _COMPOSITE_LABEL) is None
+
+ def test_no_evidence_when_post_normalizes_to_pre(self) -> None:
+ assert handler._autocomplete_commit_evidence("California", " california ", _COMPOSITE_LABEL) is None
+
+ def test_no_evidence_when_post_empty(self) -> None:
+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, "", _COMPOSITE_LABEL) is None
+
+ def test_no_evidence_when_pre_empty(self) -> None:
+ assert handler._autocomplete_commit_evidence("", _COMPOSITE_LABEL, _COMPOSITE_LABEL) is None
+
+ def test_no_evidence_when_label_empty(self) -> None:
+ assert handler._autocomplete_commit_evidence(_TYPED_PRE, _COMPOSITE_LABEL, "") is None
+
+ @pytest.mark.parametrize(
+ "pre, post, label",
+ [
+ (" ", _COMPOSITE_LABEL, _COMPOSITE_LABEL), # whitespace-only pre normalizes to empty
+ (_TYPED_PRE, " ", _COMPOSITE_LABEL), # whitespace-only post normalizes to empty
+ (_TYPED_PRE, _COMPOSITE_LABEL, " \t\n "), # whitespace-only label normalizes to empty
+ ("\xa0", _COMPOSITE_LABEL, _COMPOSITE_LABEL), # non-breaking space only
+ ],
+ )
+ def test_whitespace_only_fields_fail_closed(self, pre: str, post: str, label: str) -> None:
+ # A field that survives the truthiness check but normalizes to empty must not mint evidence.
+ assert handler._autocomplete_commit_evidence(pre, post, label) is None
+
+ def test_fields_truncated_after_relation_runs_on_full_strings(self) -> None:
+ # pre and post are valid boundary fragments of the full label; the fragment relation is
+ # evaluated on the full normalized strings, and only the emitted fields are then capped.
+ limit = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS
+ pre = _TYPED_PRE
+ post = "california " + "y" * (limit + 40)
+ label = post + " suffix"
+ evidence = handler._autocomplete_commit_evidence(pre, post, label)
+ assert evidence is not None
+ committed_option, committed_value = evidence
+ assert committed_option == label[:limit] + "…"
+ assert committed_value == post[:limit] + "…"
+
+
+class TestCommitEvidenceProducer:
+ """The LLM-path producer: click, then evidence only on a measured control transition."""
+
+ @pytest.mark.asyncio
+ async def test_transition_emits_live_label_and_post_value(self) -> None:
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option == _COMPOSITE_LABEL
+ assert result.committed_value == _COMPOSITE_LABEL
+
+ @pytest.mark.asyncio
+ async def test_noop_click_leaves_pre_unchanged_no_evidence(self) -> None:
+ # Codex r3881398576 regression: the click does not commit, control still holds the probe.
+ click = AsyncMock()
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _TYPED_PRE, _TYPED_PRE],
+ option_label=_COMPOSITE_LABEL,
+ click=click,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+ click.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_masked_pre_differs_from_typed_text_noop_no_evidence(self) -> None:
+ # The measured pre-click value is a masked form, not the typed text; a no-op click that
+ # leaves that masked form in place must not emit (proves measured pre, not typed text).
+ result = await _run_producer(
+ pre_values=["(•••) masked", "(•••) masked", "(•••) masked"],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_readback_changes_to_value_not_in_label_no_evidence(self) -> None:
+ # The click flips the control to an unrelated reformatted value (blur masking / validation /
+ # restoration) that is not a fragment of the clicked option label; a differing post is no
+ # longer accepted as selection-specific, so the click stays a bare success.
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, "(415) 555-1234"],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_empty_post_no_evidence(self) -> None:
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, ""],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_optimistic_label_that_reverts_after_settle_no_evidence(self) -> None:
+ # Codex r3892628198 regression: the control optimistically paints the selected label (a real
+ # transition off the typed probe, so the candidate gate passes) and then reverts to the probe
+ # after async validation/rerender. The first post read looked committed; the settled reread
+ # proves it was transient, so no stale evidence may be recorded.
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _TYPED_PRE],
+ option_label=_COMPOSITE_LABEL,
+ skyvern_frame=skyvern_frame,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_timer_style_revert_within_settle_floor_withholds_evidence(self) -> None:
+ # Codex r3892846834 (P1): a non-animation revert — a ~300ms timer or async validation, not a
+ # CSS/Web Animation — leaves document.getAnimations() empty, so safe_wait_for_animation_end has
+ # nothing to await and only its before_wait_sec floor holds the confirm read back. A virtual
+ # clock advanced solely by that floor (the settle's own before_wait_sec) proves the outcome at
+ # an entry point above the wait: without a bounded floor the settled reread fires while the
+ # optimistic label is still painted and commits a value that vanishes at 300ms; a floor past
+ # 300ms observes the reverted probe and withholds evidence.
+ revert_at_ms = 300.0
+ clock = SimpleNamespace(ms=0.0)
+ reads = iter([_TYPED_PRE, _COMPOSITE_LABEL])
+
+ async def settle(*, before_wait_sec: float = 0, caller: str = "") -> None:
+ clock.ms += before_wait_sec * 1000
+
+ def read_control(*_args: object, **_kwargs: object) -> str:
+ try:
+ return next(reads)
+ except StopIteration:
+ return _TYPED_PRE if clock.ms >= revert_at_ms else _COMPOSITE_LABEL
+
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock(side_effect=settle)
+ with (
+ patch.object(handler, "get_input_value", AsyncMock(side_effect=read_control)),
+ patch.object(
+ handler, "_read_autocomplete_option_identity", AsyncMock(return_value={"label": _COMPOSITE_LABEL})
+ ),
+ ):
+ result = await handler._click_autocomplete_option_with_commit_evidence(
+ skyvern_element=_control(),
+ option_locator=MagicMock(),
+ option_static_element=None,
+ skyvern_frame=skyvern_frame,
+ click=AsyncMock(),
+ is_secret_value=False,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_confirm_read_failure_after_candidate_no_evidence(self) -> None:
+ # The candidate gate passes on the first post read, but the settled confirm read fails
+ # (returns None). A value that cannot be reconfirmed after settling must fail closed.
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, RuntimeError("read boom")],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_confirm_drift_to_other_value_no_evidence(self) -> None:
+ # The candidate passes, but the settled reread drifts to a different value than the first
+ # post read. Emission requires the transitioned value to equal the first post, not merely to
+ # differ from pre, so a drift withholds evidence.
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, "Texas"],
+ option_label=_COMPOSITE_LABEL,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+
+ @pytest.mark.asyncio
+ async def test_stable_transition_survives_settle_emits(self) -> None:
+ # The transitioned value is reread identically after the single bounded settle, so the
+ # selection is stable and the evidence is recorded.
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],
+ option_label=_COMPOSITE_LABEL,
+ skyvern_frame=skyvern_frame,
+ )
+ assert result.committed_option == _COMPOSITE_LABEL
+ assert result.committed_value == _COMPOSITE_LABEL
+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_label_read_failure_click_still_succeeds_no_evidence(self) -> None:
+ click = AsyncMock()
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL],
+ option_label=None, # identity read returns None
+ static_element=None, # no scraped fallback text either
+ click=click,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+ click.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_static_element_text_is_label_fallback(self) -> None:
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],
+ option_label=None,
+ static_element={"text": _COMPOSITE_LABEL},
+ )
+ assert result.committed_option == _COMPOSITE_LABEL
+ assert result.committed_value == _COMPOSITE_LABEL
+
+ @pytest.mark.asyncio
+ async def test_secret_value_never_emits_and_skips_reads(self) -> None:
+ click = AsyncMock()
+ skyvern_frame = MagicMock()
+ get_input_value = AsyncMock(return_value=_COMPOSITE_LABEL)
+ identity = AsyncMock(return_value={"label": _COMPOSITE_LABEL})
+ with (
+ patch.object(handler, "get_input_value", get_input_value),
+ patch.object(handler, "_read_autocomplete_option_identity", identity),
+ ):
+ result = await handler._click_autocomplete_option_with_commit_evidence(
+ skyvern_element=_control(),
+ option_locator=MagicMock(),
+ option_static_element={"text": _COMPOSITE_LABEL},
+ skyvern_frame=skyvern_frame,
+ click=click,
+ is_secret_value=True,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.committed_option is None
+ assert result.committed_value is None
+ click.assert_awaited_once()
+ get_input_value.assert_not_called()
+ identity.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test_overlong_label_and_post_capped(self) -> None:
+ limit = handler.SELECT_SHADOW_MATCH_FIELD_MAX_CHARS
+ long_post = "california " + "p" * (limit + 50)
+ long_label = long_post + " suffix"
+ result = await _run_producer(
+ pre_values=[_TYPED_PRE, long_post, long_post],
+ option_label=long_label,
+ )
+ assert result.committed_option == long_label[:limit] + "…"
+ assert result.committed_value == long_post[:limit] + "…"
+
+ @pytest.mark.asyncio
+ async def test_capture_exception_after_click_stays_success(self) -> None:
+ click = AsyncMock()
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+ with (
+ patch.object(handler, "get_input_value", AsyncMock(side_effect=[_TYPED_PRE, _COMPOSITE_LABEL])),
+ patch.object(
+ handler, "_read_autocomplete_option_identity", AsyncMock(return_value={"label": _COMPOSITE_LABEL})
+ ),
+ patch.object(handler, "_autocomplete_commit_evidence", side_effect=RuntimeError("boom")),
+ ):
+ result = await handler._click_autocomplete_option_with_commit_evidence(
+ skyvern_element=_control(),
+ option_locator=MagicMock(),
+ option_static_element=None,
+ skyvern_frame=skyvern_frame,
+ click=click,
+ is_secret_value=False,
+ )
+ assert isinstance(result, ActionSuccess)
+ assert result.success is True
+ assert result.committed_option is None
+ click.assert_awaited_once()
+
+ @pytest.mark.asyncio
+ async def test_failed_click_propagates_not_swallowed(self) -> None:
+ click = AsyncMock(side_effect=RuntimeError("click failed"))
+ skyvern_frame = MagicMock()
+ identity = AsyncMock(return_value={"label": _COMPOSITE_LABEL})
+ with (
+ patch.object(handler, "get_input_value", AsyncMock(return_value=_TYPED_PRE)),
+ patch.object(handler, "_read_autocomplete_option_identity", identity),
+ pytest.raises(RuntimeError),
+ ):
+ await handler._click_autocomplete_option_with_commit_evidence(
+ skyvern_element=_control(),
+ option_locator=MagicMock(),
+ option_static_element=None,
+ skyvern_frame=skyvern_frame,
+ click=click,
+ is_secret_value=False,
+ )
+
+
+class TestWrapperPreservesEvidence:
+ """input_or_auto_complete_input must forward the producer's enriched result, not a bare success."""
+
+ @pytest.mark.asyncio
+ async def test_wrapper_forwards_enriched_result_first_attempt(self) -> None:
+ enriched = ActionSuccess(committed_option=_COMPOSITE_LABEL, committed_value=_COMPOSITE_LABEL)
+ producer_result = handler.AutoCompletionResult(action_result=enriched)
+ context = SimpleNamespace(is_location_input=False, is_search_bar=False)
+ element = MagicMock()
+ element.get_id.return_value = "region-combobox"
+ with patch.object(handler, "choose_auto_completion_dropdown", AsyncMock(return_value=producer_result)):
+ returned = await handler.input_or_auto_complete_input(
+ input_or_select_context=context,
+ scraped_page=MagicMock(),
+ page=MagicMock(),
+ dom=MagicMock(),
+ text=_TYPED_PRE,
+ skyvern_element=element,
+ step=MagicMock(),
+ task=MagicMock(),
+ is_secret_value=False,
+ )
+ assert returned is not None
+ assert returned.committed_option == _COMPOSITE_LABEL
+ assert returned.committed_value == _COMPOSITE_LABEL
+
+
+class TestChooseDropdownSeamWiring:
+ """The LLM-selected-option branch of the real choose_auto_completion_dropdown routes the
+ clicked option through the commit-evidence producer. A rendered-autocomplete option (keyed
+ by id+text, not a native <select>) is chosen by the LLM; a measured control transition emits
+ evidence, and a secret value suppresses it while the click still lands. Locks the seam at the
+ choose→producer hop (control-vs-option element, is_secret_value forwarding)."""
+
+ @staticmethod
+ def _wire(*, control_reads: list[str], option_label: str) -> dict:
+ option = {"id": "opt-ca", "tag": "div", "text": option_label}
+
+ option_locator = MagicMock()
+ option_locator.count = AsyncMock(return_value=1)
+ option_locator.element_handle = AsyncMock(return_value=MagicMock())
+
+ frame = MagicMock()
+ frame.locator.return_value = option_locator
+
+ control_locator = MagicMock()
+ control_locator.input_value = AsyncMock(side_effect=list(control_reads))
+
+ control = MagicMock()
+ control.get_id.return_value = "region-combobox"
+ control.get_frame.return_value = frame
+ control.get_frame_id.return_value = "frame-1"
+ control.is_interactable.return_value = True
+ control.get_tag_name.return_value = "input"
+ control.get_locator.return_value = control_locator
+ control.press_fill = AsyncMock()
+ control.input_clear = AsyncMock()
+ control.is_visible = AsyncMock(return_value=True)
+ control.get_element_handler = AsyncMock(return_value=MagicMock())
+
+ skyvern_frame = MagicMock(safe_wait_for_animation_end=AsyncMock())
+ skyvern_frame.read_autocomplete_option_identity = AsyncMock(return_value={"index": 0, "label": option_label})
+
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ inc.get_incremental_element_tree = AsyncMock(return_value=[dict(option)])
+ inc.id_to_element_dict = {"opt-ca": dict(option)}
+ inc.build_html_tree.return_value = "<div>opt</div>"
+
+ return {
+ "control": control,
+ "control_locator": control_locator,
+ "skyvern_frame": skyvern_frame,
+ "inc": inc,
+ }
+
+ async def _run_choose(self, wired: dict, *, is_secret_value: bool):
+ clicked_option = MagicMock(scroll_into_view=AsyncMock(), click=AsyncMock())
+ llm_response = {"relevance_float": 0.95, "id": "opt-ca", "direct_searching": False, "reasoning": "match"}
+ with (
+ patch(
+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
+ new=AsyncMock(return_value=wired["skyvern_frame"]),
+ ),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=wired["inc"]),
+ patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=clicked_option),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
+ ):
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
+ mock_app.AGENT_FUNCTION = MagicMock()
+ mock_prompt.load_prompt.return_value = "prompt"
+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+ result = await handler.choose_auto_completion_dropdown(
+ context=InputOrSelectContext(field="Region", is_search_bar=False),
+ page=MagicMock(),
+ scraped_page=MagicMock(),
+ dom=MagicMock(),
+ text=_TYPED_PRE,
+ skyvern_element=wired["control"],
+ step=_STEP,
+ task=_TASK,
+ is_secret_value=is_secret_value,
+ )
+ return result.action_result, clicked_option
+
+ @pytest.mark.asyncio
+ async def test_llm_selected_option_transition_emits_evidence(self) -> None:
+ wired = self._wire(
+ control_reads=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL], option_label=_COMPOSITE_LABEL
+ )
+ action_result, clicked_option = await self._run_choose(wired, is_secret_value=False)
+
+ assert isinstance(action_result, ActionSuccess)
+ assert action_result.committed_option == _COMPOSITE_LABEL
+ assert action_result.committed_value == _COMPOSITE_LABEL
+ clicked_option.click.assert_awaited_once()
+ # Evidence must come from the control's read-back (pre, post, settled confirm), not the option node.
+ assert wired["control_locator"].input_value.await_count == 3
+
+ @pytest.mark.asyncio
+ async def test_secret_value_suppresses_evidence_but_click_lands(self) -> None:
+ wired = self._wire(control_reads=[_TYPED_PRE, _COMPOSITE_LABEL], option_label=_COMPOSITE_LABEL)
+ action_result, clicked_option = await self._run_choose(wired, is_secret_value=True)
+
+ assert isinstance(action_result, ActionSuccess)
+ assert action_result.committed_option is None
+ assert action_result.committed_value is None
+ clicked_option.click.assert_awaited_once()
+ wired["control_locator"].input_value.assert_not_called()
+ wired["skyvern_frame"].read_autocomplete_option_identity.assert_not_called()
+
+
+class TestControlReadTimeoutBound:
+ """The commit-evidence control read is bounded by a small explicit timeout so the added settle
+ reread cannot inherit Playwright's 30s default wait; the global get_input_value stays unbounded."""
+
+ @pytest.mark.asyncio
+ async def test_control_read_bounded_by_explicit_timeout(self) -> None:
+ locator = MagicMock()
+ locator.input_value = AsyncMock(return_value=_TYPED_PRE)
+ element = MagicMock()
+ element.get_tag_name.return_value = "input"
+ element.get_locator.return_value = locator
+
+ result = await handler._read_autocomplete_control_value(element)
+
+ assert result == _TYPED_PRE
+ locator.input_value.assert_awaited_once_with(timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)
+
+ @pytest.mark.asyncio
+ async def test_get_input_value_unbounded_by_default(self) -> None:
+ # Other callers keep stock Playwright behavior: no timeout is threaded unless asked for.
+ locator = MagicMock()
+ locator.input_value = AsyncMock(return_value="x")
+
+ await handler.get_input_value("input", locator)
+
+ locator.input_value.assert_awaited_once_with()
+
+
+class TestCommitEvidenceSerialization:
+ @pytest.mark.asyncio
+ async def test_positive_readback_serialized_into_history(self) -> None:
+ action = InputTextAction(element_id="job-title", text="Backend")
+ result = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)
+
+ history = await _history_for([(action, [result])])
+
+ assert len(history) == 1
+ assert history[0]["result"]["committed_option"] == _OPTION_TEXT
+ assert history[0]["result"]["committed_value"] == _COMMITTED_VALUE
+
+ @pytest.mark.asyncio
+ async def test_unverified_success_omits_evidence(self) -> None:
+ action = InputTextAction(element_id="job-title", text="Backend")
+ history = await _history_for([(action, [ActionSuccess()])])
+
+ assert "committed_option" not in history[0]["result"]
+ assert "committed_value" not in history[0]["result"]
+
+ @pytest.mark.asyncio
+ async def test_adjacent_action_history_unchanged(self) -> None:
+ target = InputTextAction(element_id="job-title", text="Backend")
+ target_result = ActionSuccess(committed_option=_OPTION_TEXT, committed_value=_COMMITTED_VALUE)
+ neighbor = ClickAction(element_id="submit")
+ neighbor_result = ActionSuccess()
+
+ history = await _history_for([(neighbor, [neighbor_result]), (target, [target_result])])
+
+ assert "committed_option" not in history[0]["result"]
+ assert "committed_value" not in history[0]["result"]
+ assert history[1]["result"]["committed_option"] == _OPTION_TEXT
+ assert history[1]["result"]["committed_value"] == _COMMITTED_VALUE
diff --git a/tests/unit/test_element_engine_selection_threading.py b/tests/unit/test_element_engine_selection_threading.py
index ae139c90f..7f2663fb4 100644
--- a/tests/unit/test_element_engine_selection_threading.py
+++ b/tests/unit/test_element_engine_selection_threading.py
@@ -204,6 +204,7 @@ async def test_autocomplete_threads_one_fresh_selection_snapshot(monkeypatch: py
step=MagicMock(),
task=task,
preserved_elements=[{"id": "preserved"}],
+ is_secret_value=False,
)
await handler._reset_autocomplete_for_llm_fallback(
current_incremental_scraped=incremental,
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index fa3d9d66c..3301fd608 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -471,6 +471,7 @@ async def test_auto_completion_call_path_forwards_committed_value_to_shadow_matc
skyvern_element=skyvern_element,
step=SimpleNamespace(step_id="step-1"),
task=_shadow_task(),
+ is_secret_value=False,
)
events = [log for log in logs if log.get("event") == "select_shadow_match"]