Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(RUS-5): thread selected engine through handler readbacks (#7839)
LawyZhenge2d213affb49
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex d95411029..9cf956b88 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -539,8 +539,11 @@ async def _verify_autocomplete_input_readback(skyvern_element: SkyvernElement,matched_index: int,matched_label: str,+ engine_selection: BrowserEngineSelection | None = None,) -> bool:- actual_value = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())+ actual_value = await get_input_value(+ skyvern_element.get_tag_name(), skyvern_element.get_locator(), engine_selection=engine_selection+ )if _normalize_select_shadow_text(actual_value) == _normalize_select_shadow_text(matched_label):return True@@ -1659,9 +1662,10 @@ async def verify_phone_input_digits(allow_nanp_country_prefix: bool = False,pattern: str | None = None,maxlength: str | None = None,+ engine_selection: BrowserEngineSelection | None = None,) -> None:# Compare normalized digits only — never the raw value, which may be a secret.- actual_value = await get_input_value(tag_name=tag_name, locator=locator)+ actual_value = await get_input_value(tag_name=tag_name, locator=locator, engine_selection=engine_selection)expected_digits = _phone_digits(expected_value)actual_digits = _phone_digits(actual_value)# A field rendering a literal "+1" over the typed digits asserts NANP for itself; trust it only@@ -1694,6 +1698,7 @@ async def _verify_tel_input_after_fill(allow_nanp_country_prefix: bool,pattern: str | None = None,maxlength: str | None = None,+ engine_selection: BrowserEngineSelection | None = None,) -> None:await verify_phone_input_digits(tag_name=tag_name,@@ -1702,6 +1707,7 @@ async def _verify_tel_input_after_fill(allow_nanp_country_prefix=allow_nanp_country_prefix,pattern=pattern,maxlength=maxlength,+ engine_selection=engine_selection,)@@ -1713,6 +1719,7 @@ async def _fill_nanp_tel_with_readback(e164_fallback: str | None,pattern: str | None = None,maxlength: str | None = None,+ engine_selection: BrowserEngineSelection | None = None,) -> PhoneNumberInputMismatch | None:"""Fill affirmative NANP digits and verify every attempt.Retry atomically with national digits before constraint-safe E.164 for the least invasive recovery.@@ -1736,6 +1743,7 @@ async def _fill_nanp_tel_with_readback(allow_nanp_country_prefix=e164_fallback is not None,pattern=pattern,maxlength=maxlength,+ engine_selection=engine_selection,)except PhoneNumberInputMismatch as mismatch:if attempt_index == len(attempts) - 1:@@ -1754,12 +1762,22 @@ async def _fill_nanp_tel_with_readback(async def _log_tel_fallback_fill_digit_counts(- *, skyvern_element: SkyvernElement, tag_name: str, expected_value: str, task_id: str | None, step_id: str | None+ *,+ skyvern_element: SkyvernElement,+ tag_name: str,+ expected_value: str,+ task_id: str | None,+ step_id: str | None,+ engine_selection: BrowserEngineSelection | None = None,) -> None:# Observability only: the LLM-fallback tel fill has no raising read-back, so a digit drop there is# otherwise invisible. Count-only (values may be secrets) and never fails the action.try:- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())+ actual_value = await get_input_value(+ tag_name=tag_name,+ locator=skyvern_element.get_locator(),+ engine_selection=engine_selection,+ )expected_digit_count = len(_phone_digits(expected_value))actual_digit_count = len(_phone_digits(actual_value))LOG.info(@@ -1862,19 +1880,28 @@ async def _is_card_number_field(skyvern_element: SkyvernElement) -> bool:async def _fill_card_number_with_readback(- *, skyvern_element: SkyvernElement, tag_name: str, text: str, expected_digits: str+ *,+ skyvern_element: SkyvernElement,+ tag_name: str,+ text: str,+ expected_digits: str,+ engine_selection: BrowserEngineSelection | None = None,) -> ActionFailure | None:# Type the card number, then read the rendered digits back. Character-by-character typing races an# auto-formatting field's caret restore and can scramble the value (SKY-11720); a single atomic# value-set formats once, without the race, so a mismatch is re-entered atomically before failing.await skyvern_element.input_sequentially(text=text)- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())+ actual_value = await get_input_value(+ tag_name=tag_name, locator=skyvern_element.get_locator(), engine_selection=engine_selection+ )if not _card_readback_is_mismatch(expected_digits, actual_value):return Noneawait skyvern_element.input_clear()await skyvern_element.input_fill(text=text)- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())+ actual_value = await get_input_value(+ tag_name=tag_name, locator=skyvern_element.get_locator(), engine_selection=engine_selection+ )# Success after re-entry must be positively confirmed: a clean digit match. An empty/masked/# unreadable or still-mismatched retry read-back is NOT success -- fail loudly rather than silently# proceed with a value we deleted-and-could-not-verify.@@ -1956,7 +1983,13 @@ def _secret_readback_matches(expected: str, actual_value: str | None) -> bool:async def _fill_secret_with_readback(- *, skyvern_element: SkyvernElement, tag_name: str, text: str, input_type: str, maxlength: str | None+ *,+ skyvern_element: SkyvernElement,+ tag_name: str,+ text: str,+ input_type: str,+ maxlength: str | None,+ engine_selection: BrowserEngineSelection | None = None,) -> ActionFailure | None:# Character-by-character credential entry can race a hardened field's caret restore and rotate the# value, or be dropped by a controlled field and truncate it, while the block still completes --@@ -1977,7 +2010,9 @@ async def _fill_secret_with_readback()return None- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())+ actual_value = await get_input_value(+ tag_name=tag_name, locator=skyvern_element.get_locator(), engine_selection=engine_selection+ )# Exact equality first: a value that round-trips exactly is confirmed, even one made only of mask-like# characters -- so an all-"*" secret is a match, never misclassified as an unreadable mask.if not _secret_readback_is_mismatch(text, actual_value):@@ -1992,7 +2027,9 @@ async def _fill_secret_with_readback(await skyvern_element.input_clear()await skyvern_element.input_fill(text=text)- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())+ actual_value = await get_input_value(+ tag_name=tag_name, locator=skyvern_element.get_locator(), engine_selection=engine_selection+ )if _secret_readback_matches(text, actual_value):return None@@ -4616,6 +4653,7 @@ async def handle_input_text_action(tag_name=tag_name,text=text,expected_digits=card_expected_digits,+ engine_selection=engine_selection,)if card_failure is not None:return [card_failure]@@ -4627,6 +4665,7 @@ async def handle_input_text_action(e164_fallback=tel_e164_fallback,pattern=tel_pattern,maxlength=tel_maxlength,+ engine_selection=engine_selection,)if phone_mismatch is not None:LOG.warning(@@ -4643,6 +4682,7 @@ async def handle_input_text_action(text=text,input_type=secret_input_type,maxlength=secret_maxlength,+ engine_selection=engine_selection,)if secret_failure is not None:return [secret_failure]@@ -4662,6 +4702,7 @@ async def handle_input_text_action(expected_value=text,task_id=task.task_id,step_id=step.step_id,+ engine_selection=engine_selection,)incremental_element = await incremental_scraped.get_incremental_element_tree(@@ -6651,9 +6692,8 @@ async def choose_auto_completion_dropdown(current_frame = skyvern_element.get_frame()skyvern_frame = await SkyvernFrame.create_instance(current_frame)- incremental_scraped = IncrementalScrapePage(- skyvern_frame=skyvern_frame, engine_selection=resolve_engine_selection_for_task(task)- )+ engine_selection = resolve_engine_selection_for_task(task)+ incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame, engine_selection=engine_selection)await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())try:@@ -6760,6 +6800,7 @@ async def choose_auto_completion_dropdown(skyvern_element=skyvern_element,matched_index=matched_index,matched_label=matched_label,+ engine_selection=engine_selection,):clear_input = Falseresult.action_result = ActionSuccess()diff --git a/tests/unit/test_cc_input_digit_drop.py b/tests/unit/test_cc_input_digit_drop.pyindex be6ee8ac7..1f40e5d1c 100644--- a/tests/unit/test_cc_input_digit_drop.py+++ b/tests/unit/test_cc_input_digit_drop.py@@ -1,6 +1,6 @@from __future__ import annotations-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -176,6 +176,26 @@ async def test_fill_card_number_fails_when_retry_readback_masked() -> None:element.input_fill.assert_awaited_once_with(text=VISA_16)+async def test_fill_card_number_threads_engine_selection_to_both_readbacks() -> None:+ selection = object()+ element = _make_card_element([])+ with patch(+ "skyvern.webeye.actions.handler.get_input_value",+ new=AsyncMock(side_effect=["4539 5876 6214 6837", VISA_16]),+ ) as get_input_value:+ result = await _fill_card_number_with_readback(+ skyvern_element=element,+ tag_name="input",+ text=VISA_16,+ expected_digits=VISA_16,+ engine_selection=selection,+ )++ assert result is None+ assert [call.kwargs["engine_selection"] for call in get_input_value.await_args_list] == [selection, selection]++@pytest.mark.asyncio@pytest.mark.parametrize("attrs,expected",diff --git a/tests/unit/test_input_secret_readback.py b/tests/unit/test_input_secret_readback.pyindex 188682a4d..3479487dc 100644--- a/tests/unit/test_input_secret_readback.py+++ b/tests/unit/test_input_secret_readback.py@@ -352,3 +352,24 @@ async def test_fill_secret_logs_carry_no_secret_material() -> None:assert str(len(SECRET)) not in loggedassert result.exception_type == "SecretInputMismatch"assert SECRET not in (result.exception_message or "")+++async def test_fill_secret_threads_engine_selection_to_both_readbacks() -> None:+ selection = object()+ element = _make_secret_element([])+ with patch(+ "skyvern.webeye.actions.handler.get_input_value",+ new=AsyncMock(side_effect=[ROTATED, SECRET]),+ ) as get_input_value:+ result = await _fill_secret_with_readback(+ skyvern_element=element,+ tag_name="input",+ text=SECRET,+ input_type="password",+ maxlength=None,+ engine_selection=selection,+ )++ assert result is None+ assert [call.kwargs["engine_selection"] for call in get_input_value.await_args_list] == [selection, selection]diff --git a/tests/unit/test_input_text_phone_readback.py b/tests/unit/test_input_text_phone_readback.pyindex 9ec6afc57..9989da79d 100644--- a/tests/unit/test_input_text_phone_readback.py+++ b/tests/unit/test_input_text_phone_readback.py@@ -1,8 +1,9 @@from __future__ import annotations-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from playwright.async_api import Error as PlaywrightErrorfrom skyvern.exceptions import PhoneNumberInputMismatchfrom skyvern.webeye.actions.handler import (@@ -10,9 +11,30 @@ from skyvern.webeye.actions.handler import (_log_tel_fallback_fill_digit_counts,verify_phone_input_digits,)+from skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelectionNANP_DIGITS = "2245550199"NANP_E164 = f"+1{NANP_DIGITS}"+_NOT_INPUTELEMENT_ERROR = "Node is not an HTMLInputElement, HTMLTextAreaElement or HTMLSelectElement"+++class _EngineError(Exception):+ pass+++async def _never_start() -> None: # pragma: no cover+ raise AssertionError("start_driver must not be called")+++def _engine_selection() -> BrowserEngineSelection:+ return BrowserEngineSelection(+ name="engine-a",+ start_driver=_never_start,+ error_type=_EngineError,+ timeout_error_type=_EngineError,+ metadata=BrowserEngineMetadata(name="engine-a", version="0.0.0"),+ selection_reason="test",+ )def _make_element(locator: MagicMock) -> MagicMock:@@ -134,6 +156,64 @@ async def test_phone_readback_digit_drop_raises() -> None:assert exc.value.actual_digit_count == 9+async def test_phone_readback_threads_selected_engine_incompatible_error() -> None:+ locator = MagicMock()+ locator.input_value = AsyncMock(side_effect=_EngineError(_NOT_INPUTELEMENT_ERROR))++ with pytest.raises(PhoneNumberInputMismatch):+ await verify_phone_input_digits(+ tag_name="input",+ locator=locator,+ expected_value=NANP_DIGITS,+ engine_selection=_engine_selection(),+ )+++async def test_phone_readback_propagates_foreign_engine_error() -> None:+ locator = MagicMock()+ locator.input_value = AsyncMock(side_effect=PlaywrightError(_NOT_INPUTELEMENT_ERROR))++ with pytest.raises(PlaywrightError):+ await verify_phone_input_digits(+ tag_name="input",+ locator=locator,+ expected_value=NANP_DIGITS,+ engine_selection=_engine_selection(),+ )+++async def test_tel_fill_and_fallback_log_thread_engine_selection() -> None:+ selection = _engine_selection()+ element = _make_element(MagicMock())+ with patch(+ "skyvern.webeye.actions.handler.get_input_value",+ new=AsyncMock(side_effect=[NANP_DIGITS, NANP_DIGITS]),+ ) as get_input_value:+ assert (+ await _fill_nanp_tel_with_readback(+ skyvern_element=element,+ tag_name="input",+ national_digits=NANP_DIGITS,+ e164_fallback=NANP_E164,+ engine_selection=selection,+ )+ is None+ )+ await _log_tel_fallback_fill_digit_counts(+ skyvern_element=element,+ tag_name="input",+ expected_value=NANP_DIGITS,+ task_id="tsk_1",+ step_id="stp_1",+ engine_selection=selection,+ )++ assert [call.kwargs["engine_selection"] for call in get_input_value.await_args_list] == [selection, selection]++@pytest.mark.asyncioasync def test_nanp_fill_clean_first_attempt_skips_fallbacks() -> None:element = _make_fill_element(["(224) 555-0199"])diff --git a/tests/unit/test_input_text_tel_card_routing.py b/tests/unit/test_input_text_tel_card_routing.pyindex de0bb3c42..e41036624 100644--- a/tests/unit/test_input_text_tel_card_routing.py+++ b/tests/unit/test_input_text_tel_card_routing.py@@ -131,6 +131,7 @@ async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs:tag_name="input",text=VISA_16,expected_digits=VISA_16,+ engine_selection=None,)phone_format.assert_not_awaited()tel_verify.assert_not_awaited()@@ -153,6 +154,7 @@ async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> Noneallow_nanp_country_prefix=False,pattern=None,maxlength=None,+ engine_selection=None,)card_readback.assert_not_awaited()phone_format.assert_not_awaited()@@ -249,6 +251,7 @@ async def test_blocking_tel_input_rechecks_constraints_before_readback() -> Noneallow_nanp_country_prefix=False,pattern="[0-9]{10}",maxlength="10",+ engine_selection=None,)@@ -272,6 +275,7 @@ async def test_secret_tel_value_uses_tel_verifier_not_secret_readback() -> None:allow_nanp_country_prefix=False,pattern=None,maxlength=None,+ engine_selection=None,)secret_readback.assert_not_awaited()card_readback.assert_not_awaited()@@ -368,6 +372,7 @@ async def test_secret_in_exact_value_input_uses_readback(input_type: str | None)text="mysecretvalue",input_type=input_type or "",maxlength=None,+ engine_selection=None,)el.input_sequentially.assert_not_awaited()@@ -425,6 +430,7 @@ async def test_secret_readback_runs_on_retargeted_element_type() -> None:text="mysecretvalue",input_type="password",maxlength=None,+ engine_selection=None,)diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.pyindex 5f587648f..c29a3f14b 100644--- a/tests/unit/test_select_shadow_match.py+++ b/tests/unit/test_select_shadow_match.py@@ -1103,3 +1103,21 @@ async def test_converted_route_reports_input_text_converted(monkeypatch: pytest.MagicMock(),)assert select.await_args.kwargs["entry_action_type"] == "input_text_converted"+++async def test_autocomplete_readback_threads_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:+ selection = object()+ element = MagicMock()+ element.get_tag_name.return_value = "input"+ element.get_locator.return_value = MagicMock()+ get_input_value = AsyncMock(return_value="Choice")+ monkeypatch.setattr(handler, "get_input_value", get_input_value)++ assert await handler._verify_autocomplete_input_readback(+ skyvern_element=element,+ matched_index=0,+ matched_label="Choice",+ engine_selection=selection,+ )+ get_input_value.assert_awaited_once_with("input", element.get_locator(), engine_selection=selection)