Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Confirm autocomplete selections on the next render (#8348)
LawyZheng3cdcba8f99e1
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex f15364e6d..b74eaf4f3 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -774,9 +774,10 @@ async def _click_autocomplete_option_with_commit_evidence("""Click the LLM-selected option and return ``ActionSuccess``, enriched with commit evidenceonly when the target control's read-back changes into a value that — like the pre-click value —is a boundary-delimited fragment of the clicked option's label (see- ``_autocomplete_commit_evidence``) AND that transitioned value survives a single bounded settle- reread. The settle guards against an optimistic control that paints the selected label and then- reverts it after async validation/rerender, which would otherwise commit a transient value.+ ``_autocomplete_commit_evidence``) AND that transitioned value survives a next-render settle+ reread. The render-driven settle guards against an optimistic control that paints the selected+ label and then reverts it after async validation/rerender, which would otherwise commit a+ transient value.Evidence capture is best-effort and fail-closed: a failed read (before, after, or on the settlereread), a missing label, an unchanged control, an unrelated transform, a value that drifts on@@ -808,10 +809,11 @@ async def _click_autocomplete_option_with_commit_evidence(if evidence is None:return ActionSuccess()# The candidate transition passed, but an optimistic control can paint the selected label and- # then revert it after async validation/rerender. Settle once (no loop) and reconfirm the- # value; only record evidence when the settled reread still holds the first post value, so a- # transient paint cannot be committed as stale evidence.- await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.commit")+ # then revert it after async validation/rerender. Reconcile on the next render turn via the+ # existing render-settle helper (double-rAF, with a 250ms liveness cap) and reread once; only+ # record evidence when the next-render reread still holds the first post value, so a transient+ # paint cannot be committed as stale evidence.+ await _wait_custom_select_render_settle(skyvern_element)confirm_value = await _read_autocomplete_control_value(skyvern_element, engine_selection)if confirm_value is None or _normalize_select_shadow_text(confirm_value) != _normalize_select_shadow_text(post_valuediff --git a/tests/unit/test_commit_evidence.py b/tests/unit/test_commit_evidence.pyindex 2bb548ea8..9bdc65bed 100644--- a/tests/unit/test_commit_evidence.py+++ b/tests/unit/test_commit_evidence.py@@ -54,23 +54,26 @@ async def _run_producer(static_element: dict | None = None,is_secret_value: bool = False,click: AsyncMock | None = None,- skyvern_frame: MagicMock | None = None,+ settle: AsyncMock | None = None,) -> ActionResult:- """Drive the real producer with light IO stubs; pre_values feeds get_input_value in order."""+ """Drive the real producer with light IO stubs; pre_values feeds get_input_value in order.++ The candidate-evidence confirmation reconciles on the next render turn via the existing+ render-settle helper, so that seam is stubbed to a no-op here; pass ``settle`` to assert on it.+ """click = click or AsyncMock()- if skyvern_frame is None:- skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ settle = settle or AsyncMock()identity = {"label": option_label} if option_label is not None else Nonewith (patch.object(handler, "get_input_value", AsyncMock(side_effect=pre_values)),patch.object(handler, "_read_autocomplete_option_identity", AsyncMock(return_value=identity)),+ patch.object(handler, "_wait_custom_select_render_settle", settle),):return await handler._click_autocomplete_option_with_commit_evidence(skyvern_element=_control(),option_locator=MagicMock(),option_static_element=static_element,- skyvern_frame=skyvern_frame,+ skyvern_frame=MagicMock(),click=click,is_secret_value=is_secret_value,)@@ -261,49 +264,35 @@ class TestCommitEvidenceProducer:async def test_optimistic_label_that_reverts_after_settle_no_evidence(self) -> None:# Codex r3892628198 regression: the control optimistically paints the selected label (a real# transition off the typed probe, so the candidate gate passes) and then reverts to the probe- # after async validation/rerender. The first post read looked committed; the settled reread+ # after async validation/rerender. The first post read looked committed; the next-render reread# proves it was transient, so no stale evidence may be recorded.- skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ settle = AsyncMock()result = await _run_producer(pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _TYPED_PRE],option_label=_COMPOSITE_LABEL,- skyvern_frame=skyvern_frame,+ settle=settle,)assert isinstance(result, ActionSuccess)assert result.committed_option is Noneassert result.committed_value is None- skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()+ settle.assert_awaited_once()@pytest.mark.asyncio- async def test_timer_style_revert_within_settle_floor_withholds_evidence(self) -> None:- # Codex r3892846834 (P1): a non-animation revert — a ~300ms timer or async validation, not a- # CSS/Web Animation — leaves document.getAnimations() empty, so safe_wait_for_animation_end has- # nothing to await and only its before_wait_sec floor holds the confirm read back. A virtual- # clock advanced solely by that floor (the settle's own before_wait_sec) proves the outcome at- # an entry point above the wait: without a bounded floor the settled reread fires while the- # optimistic label is still painted and commits a value that vanishes at 300ms; a floor past- # 300ms observes the reverted probe and withholds evidence.- revert_at_ms = 300.0- clock = SimpleNamespace(ms=0.0)- reads = iter([_TYPED_PRE, _COMPOSITE_LABEL])-- async def settle(*, before_wait_sec: float = 0, caller: str = "") -> None:- clock.ms += before_wait_sec * 1000-- def read_control(*_args: object, **_kwargs: object) -> str:- try:- return next(reads)- except StopIteration:- return _TYPED_PRE if clock.ms >= revert_at_ms else _COMPOSITE_LABEL-+ async def test_candidate_path_confirms_on_next_render_not_fixed_wait(self) -> None:+ # Codex r3892846834 (P1), re-scoped off the rejected fixed wall-clock horizon: the settled+ # confirm read is render-driven. The candidate-evidence path reconciles on the next render turn+ # via the existing render-settle helper — exactly one call, and never the fixed-wait animation+ # seam — so an optimistic label that has reverted to the probe by the next render withholds+ # evidence.+ settle = AsyncMock()skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock(side_effect=settle)+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()with (- patch.object(handler, "get_input_value", AsyncMock(side_effect=read_control)),+ patch.object(handler, "get_input_value", AsyncMock(side_effect=[_TYPED_PRE, _COMPOSITE_LABEL, _TYPED_PRE])),patch.object(handler, "_read_autocomplete_option_identity", AsyncMock(return_value={"label": _COMPOSITE_LABEL})),+ patch.object(handler, "_wait_custom_select_render_settle", settle),):result = await handler._click_autocomplete_option_with_commit_evidence(skyvern_element=_control(),@@ -316,6 +305,8 @@ class TestCommitEvidenceProducer:assert isinstance(result, ActionSuccess)assert result.committed_option is Noneassert result.committed_value is None+ settle.assert_awaited_once()+ skyvern_frame.safe_wait_for_animation_end.assert_not_awaited()@pytest.mark.asyncioasync def test_confirm_read_failure_after_candidate_no_evidence(self) -> None:@@ -344,18 +335,74 @@ class TestCommitEvidenceProducer:@pytest.mark.asyncioasync def test_stable_transition_survives_settle_emits(self) -> None:- # The transitioned value is reread identically after the single bounded settle, so the+ # The transitioned value is reread identically after the single render-driven settle, so the# selection is stable and the evidence is recorded.- skyvern_frame = MagicMock()- skyvern_frame.safe_wait_for_animation_end = AsyncMock()+ settle = AsyncMock()result = await _run_producer(pre_values=[_TYPED_PRE, _COMPOSITE_LABEL, _COMPOSITE_LABEL],option_label=_COMPOSITE_LABEL,- skyvern_frame=skyvern_frame,+ settle=settle,)assert result.committed_option == _COMPOSITE_LABELassert result.committed_value == _COMPOSITE_LABEL- skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()+ settle.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_pre_read_failure_click_still_succeeds_no_evidence(self) -> None:+ # The pre-click control read raises. Capture is best-effort, so the exception is swallowed to a+ # None pre-value and the click still lands; a missing pre-value withholds evidence, but the+ # successful click must never regress to ActionFailure. Evidence is None before the settle.+ click = AsyncMock()+ settle = AsyncMock()+ result = await _run_producer(+ pre_values=[RuntimeError("pre read boom"), _COMPOSITE_LABEL],+ option_label=_COMPOSITE_LABEL,+ click=click,+ settle=settle,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()+ settle.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_first_post_read_failure_click_still_succeeds_no_evidence(self) -> None:+ # Pre-read succeeds, but the first post-click control read raises. The exception is swallowed to+ # a None post-value, so no transition can be measured and the click stays a bare success; the+ # gate fails before any settle reread.+ click = AsyncMock()+ settle = AsyncMock()+ result = await _run_producer(+ pre_values=[_TYPED_PRE, RuntimeError("post read boom")],+ option_label=_COMPOSITE_LABEL,+ click=click,+ settle=settle,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()+ settle.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_settle_failure_after_candidate_gate_no_evidence(self) -> None:+ # The candidate gate passes (pre->post is a selection-specific transition), then the render+ # settle reconciliation raises. Settle runs inside the exception-isolated capture, so a+ # successful click stays a bare success rather than regressing to ActionFailure.+ click = AsyncMock()+ settle = AsyncMock(side_effect=RuntimeError("settle boom"))+ result = await _run_producer(+ pre_values=[_TYPED_PRE, _COMPOSITE_LABEL],+ option_label=_COMPOSITE_LABEL,+ click=click,+ settle=settle,+ )+ assert isinstance(result, ActionSuccess)+ assert result.committed_option is None+ assert result.committed_value is None+ click.assert_awaited_once()+ settle.assert_awaited_once()@pytest.mark.asyncioasync def test_label_read_failure_click_still_succeeds_no_evidence(self) -> None:@@ -550,6 +597,7 @@ class TestChooseDropdownSeamWiring:),patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=wired["inc"]),patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=clicked_option),+ patch("skyvern.webeye.actions.handler._wait_custom_select_render_settle", new=AsyncMock()),patch("skyvern.webeye.actions.handler.app") as mock_app,patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,