Skip to content

Automate browser based workflows with AI

19.0 KB
Raw Blame
from __future__ import annotations

from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, call

import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError

from skyvern.constants import TEXT_INPUT_DELAY
from skyvern.forge.sdk.event.default import DefaultInputStrategy
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.actions.handler import (
    _is_tel_digit_fix_enabled,
    _nanp_e164_fallback,
    _nanp_national_digits,
    _plan_tel_text,
)
from tests.unit.helpers import make_organization, make_task


@pytest.mark.parametrize(
    "value,expected",
    [
        ("+1 (224) 555-0199", "2245550199"),
        ("+12245550199", "2245550199"),
        ("1-224-555-0199", "2245550199"),
        ("1 (224) 555-0199", "2245550199"),
        ("(224) 555-0199", "2245550199"),
        ("224-555-0199", "2245550199"),
        ("224.555.0199", "2245550199"),
        ("224 555 0199", "2245550199"),
        ("12245550199", None),  # bare leading 1 is not written as a country code
        ("13912345678", None),  # 11-digit non-NANP mobile whose first digit is 1
        ("15012345678", None),
        ("13987654321", None),
        ("(13) 98765-4321", None),  # 2-digit area code + 9-digit subscriber, strips to 1 + 10
        ("0412345678", None),
        ("0412 345 678", None),
        ("1234567890", None),
        ("2245550199", None),
        ("4155550123", None),
        ("(024) 555-0199", None),
        ("(224) 155-0199", None),
        ("+44 20 7946 0958", None),
        ("224-555-0199 x123", None),
        ("224-555-019", None),
        ("", None),
    ],
)
def test_nanp_national_digits(value: str, expected: str | None) -> None:
    assert _nanp_national_digits(value) == expected


@pytest.mark.parametrize(
    "source_value,pattern,maxlength,expected",
    [
        ("+1 (224) 555-0199", None, None, "+12245550199"),
        ("1-224-555-0199", r"\+1\d{10}", "12", "+12245550199"),
        ("(224) 555-0199", None, None, None),
        ("224-555-0199", None, None, None),
        ("+1 (224) 555-0199", r"\d{10}", None, None),
        ("+1 (224) 555-0199", "[", None, None),
        ("+1 (224) 555-0199", None, "10", None),
        ("+1 (224) 555-0199", None, "-1", None),
        ("+1 (224) 555-0199", None, "not-a-number", None),
    ],
)
def test_nanp_e164_fallback_respects_live_field_constraints(
    source_value: str, pattern: str | None, maxlength: str | None, expected: str | None
) -> None:
    assert _nanp_e164_fallback(source_value, pattern=pattern, maxlength=maxlength) == expected


def test_plan_tel_text_strips_secret_resolved_formatted_nanp() -> None:
    # A secret with an affirmative NANP country code is typed as national digits and is never sent to
    # the format-check LLM.
    text, used_bare, run_format_check = _plan_tel_text(
        is_tel=True, is_secret=True, value="+1 (224) 555-0199", pattern=None
    )
    assert text == "2245550199"
    assert used_bare is True
    assert run_format_check is False


@pytest.mark.parametrize(
    "is_secret,expected_format_check",
    [(False, True), (True, False)],
)
def test_plan_tel_text_ineligible_value_falls_back(is_secret: bool, expected_format_check: bool) -> None:
    value = "0412 345 678"
    assert _plan_tel_text(is_tel=True, is_secret=is_secret, value=value, pattern=None) == (
        value,
        False,
        expected_format_check,
    )


def test_plan_tel_text_self_formatting_field_uses_bare_digits() -> None:
    # A permissive pattern that accepts bare digits keeps the bare-digit fast path.
    text, used_bare, run_format_check = _plan_tel_text(
        is_tel=True, is_secret=False, value="(224) 555-0199", pattern=r"[+0-9().\- ]{7,20}"
    )
    assert text == "2245550199"
    assert used_bare is True
    assert run_format_check is False


def test_plan_tel_text_masked_field_keeps_format_check() -> None:
    # A field whose pattern requires a specific mask (bare digits don't match) is not stripped; the
    # non-secret format-check path still runs.
    text, used_bare, run_format_check = _plan_tel_text(
        is_tel=True, is_secret=False, value="(224) 555-0199", pattern=r"\(\d{3}\) \d{3}-\d{4}"
    )
    assert text == "(224) 555-0199"
    assert used_bare is False
    assert run_format_check is True


def test_plan_tel_text_masked_secret_skips_llm() -> None:
    # A masked field carrying a secret: not stripped, and the LLM is never called for secrets.
    assert _plan_tel_text(is_tel=True, is_secret=True, value="(224) 555-0199", pattern=r"\(\d{3}\) \d{3}-\d{4}") == (
        "(224) 555-0199",
        False,
        False,
    )


def test_plan_tel_text_non_tel_passthrough() -> None:
    assert _plan_tel_text(is_tel=False, is_secret=False, value="(224) 555-0199", pattern=None) == (
        "(224) 555-0199",
        False,
        False,
    )


@pytest.mark.asyncio
async def test_input_sequentially_does_not_fill_split_ten_digits(monkeypatch: pytest.MonkeyPatch) -> None:
    """Bare 10-digit values are typed in one pass — no fill()-split that an auto-formatting tel field
    would mangle into a dropped digit."""
    typed: list[str] = []
    monkeypatch.setattr(
        handler_utils.EventStrategyFactory,
        "type_text",
        AsyncMock(side_effect=lambda page, locator, text: typed.append(text)),
    )
    locator = MagicMock()
    locator.fill = AsyncMock()
    locator.page = MagicMock()

    await handler_utils.input_sequentially(locator, "2245550199")

    locator.fill.assert_not_called()
    assert typed == ["2245550199"]


@pytest.mark.asyncio
async def test_input_sequentially_fill_splits_separator_formatted_value(monkeypatch: pytest.MonkeyPatch) -> None:
    """A 14-char separator-formatted value gets fill()-split into the half-open '(224', which a live
    AsYouType formatter collapses — the failure mode the bare-digit path avoids."""
    typed: list[str] = []
    monkeypatch.setattr(
        handler_utils.EventStrategyFactory,
        "type_text",
        AsyncMock(side_effect=lambda page, locator, text: typed.append(text)),
    )
    locator = MagicMock()
    locator.fill = AsyncMock()
    locator.page = MagicMock()

    await handler_utils.input_sequentially(locator, "(224) 555-0199")

    locator.fill.assert_awaited_once()
    assert locator.fill.await_args.args[0] == "(224"
    assert typed == [") 555-0199"]


@pytest.mark.asyncio
async def test_strategy_aware_input_clears_short_replacement_through_active_strategy(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    deadline_active = False

    async def clear_field(*_args: object, **_kwargs: object) -> None:
        assert deadline_active

    async def type_text(*_args: object, **_kwargs: object) -> None:
        assert deadline_active

    clear_field_mock = AsyncMock(side_effect=clear_field)
    type_text_mock = AsyncMock(side_effect=type_text)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", clear_field_mock)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text_mock)
    timeout_calls: list[float | None] = []

    @asynccontextmanager
    async def record_timeout(delay: float | None) -> AsyncIterator[None]:
        nonlocal deadline_active
        timeout_calls.append(delay)
        deadline_active = True
        try:
            yield
        finally:
            deadline_active = False

    monkeypatch.setattr(handler_utils.asyncio, "timeout", record_timeout)

    await handler_utils.strategy_aware_input(locator, "Noor", clear=True, timeout=4321)

    clear_field_mock.assert_awaited_once_with(locator.page, locator, char_count=0, timeout=4321)
    type_text_mock.assert_awaited_once_with(
        locator.page,
        locator,
        "Noor",
        timeout=4321,
        allow_batched_playwright=True,
    )
    locator.fill.assert_not_awaited()
    assert timeout_calls == [4.321]


@pytest.mark.asyncio
async def test_strategy_aware_input_dispatches_best_effort_commit_events_after_typing(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    operations: list[str] = []

    async def clear_field(*_args: object, **_kwargs: object) -> None:
        operations.append("clear")

    async def type_text(*_args: object, **_kwargs: object) -> None:
        operations.append("type")

    async def dispatch_event(event_name: str, **_kwargs: object) -> None:
        operations.append(event_name)
        if event_name == "change":
            raise RuntimeError("dispatch failed")

    locator = MagicMock()
    locator.fill = AsyncMock()
    locator.dispatch_event = AsyncMock(side_effect=dispatch_event)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", AsyncMock(side_effect=clear_field))
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", AsyncMock(side_effect=type_text))

    await handler_utils.strategy_aware_input(
        locator,
        "Noor",
        clear=True,
        timeout=4321,
        dispatch_change_and_blur=True,
    )

    assert operations == ["clear", "type", "change", "blur"]
    assert locator.dispatch_event.await_args_list[0].kwargs == {"timeout": 4321}
    assert locator.dispatch_event.await_args_list[1].kwargs == {"timeout": 4321}


@pytest.mark.asyncio
async def test_strategy_aware_input_preserves_explicit_null_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    clear_field = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", clear_field)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)
    timeout_calls: list[float | None] = []

    @asynccontextmanager
    async def record_timeout(delay: float | None) -> AsyncIterator[None]:
        timeout_calls.append(delay)
        yield

    monkeypatch.setattr(handler_utils.asyncio, "timeout", record_timeout)

    await handler_utils.strategy_aware_input(locator, "Noor", clear=True, timeout=None)

    assert timeout_calls == [None]
    clear_field.assert_awaited_once_with(locator.page, locator, char_count=0, timeout=None)
    type_text.assert_awaited_once_with(
        locator.page,
        locator,
        "Noor",
        timeout=None,
        allow_batched_playwright=True,
    )


@pytest.mark.asyncio
async def test_strategy_aware_input_preserves_zero_as_no_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    clear_field = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", clear_field)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)
    timeout_calls: list[float | None] = []

    @asynccontextmanager
    async def record_timeout(delay: float | None) -> AsyncIterator[None]:
        timeout_calls.append(delay)
        yield

    monkeypatch.setattr(handler_utils.asyncio, "timeout", record_timeout)

    await handler_utils.strategy_aware_input(locator, "Noor", clear=True, timeout=0)

    assert timeout_calls == [None]
    clear_field.assert_awaited_once_with(locator.page, locator, char_count=0, timeout=0)
    type_text.assert_awaited_once_with(
        locator.page,
        locator,
        "Noor",
        timeout=0,
        allow_batched_playwright=True,
    )


@pytest.mark.asyncio
async def test_strategy_aware_force_fill_preserves_atomic_playwright_semantics(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    clear_field = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", clear_field)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)

    await handler_utils.strategy_aware_input(
        locator,
        "Noor",
        clear=True,
        timeout=4321,
        force=True,
        no_wait_after=True,
    )

    locator.fill.assert_awaited_once_with("Noor", timeout=4321, force=True, no_wait_after=True)
    clear_field.assert_not_awaited()
    type_text.assert_not_awaited()


@pytest.mark.asyncio
@pytest.mark.parametrize(
    ("input_type", "value"),
    [
        pytest.param("range", "42", id="range"),
        pytest.param("date", "2026-08-30", id="date"),
        pytest.param("datetime-local", "2026-08-30T12:34", id="datetime-local"),
        pytest.param("month", "2026-08", id="month"),
        pytest.param("time", "12:34", id="time"),
        pytest.param("week", "2026-W35", id="week"),
    ],
)
async def test_strategy_aware_fill_preserves_native_value_set_input_semantics(
    monkeypatch: pytest.MonkeyPatch,
    input_type: str,
    value: str,
) -> None:
    locator = MagicMock()
    locator.evaluate = AsyncMock(return_value=input_type)
    locator.fill = AsyncMock()
    locator.dispatch_event = AsyncMock()
    clear_field = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", clear_field)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)

    await handler_utils.strategy_aware_input(
        locator,
        value,
        clear=True,
        timeout=4321,
        force=False,
        no_wait_after=True,
        dispatch_change_and_blur=True,
    )

    locator.fill.assert_awaited_once_with(value, timeout=4321, force=False, no_wait_after=True)
    clear_field.assert_not_awaited()
    type_text.assert_not_awaited()
    assert [call.args[0] for call in locator.dispatch_event.await_args_list] == ["change", "blur"]


@pytest.mark.asyncio
async def test_default_input_strategy_forwards_timeout_to_every_locator_operation() -> None:
    page = MagicMock()
    locator = MagicMock()
    locator.type = AsyncMock()
    locator.clear = AsyncMock()
    strategy = DefaultInputStrategy()

    await strategy.clear_field(page, locator, char_count=0, timeout=0)
    await strategy.type_text(page, locator, "Noor", timeout=0)

    locator.clear.assert_awaited_once_with(timeout=0)
    assert locator.type.await_args_list == [call(char, delay=TEXT_INPUT_DELAY, timeout=0) for char in "Noor"]


@pytest.mark.asyncio
async def test_default_input_strategy_batches_when_copilot_surface_allows_it() -> None:
    page = MagicMock()
    locator = MagicMock()
    locator.type = AsyncMock()
    strategy = DefaultInputStrategy()

    await strategy.type_text(
        page,
        locator,
        "Noor",
        timeout=4321,
        allow_batched_playwright=True,
    )

    locator.type.assert_awaited_once_with("Noor", delay=TEXT_INPUT_DELAY, timeout=4321)


@pytest.mark.asyncio
async def test_default_input_strategy_preserves_supported_playwright_input_options() -> None:
    page = MagicMock()
    locator = MagicMock()
    locator.type = AsyncMock()
    locator.clear = AsyncMock()
    strategy = DefaultInputStrategy()

    await strategy.clear_field(page, locator, char_count=0, timeout=1234, force=True, no_wait_after=True)
    await strategy.type_text(page, locator, "No", timeout=2345, delay=17, no_wait_after=True)

    locator.clear.assert_awaited_once_with(timeout=1234, force=True, no_wait_after=True)
    assert locator.type.await_args_list == [
        call("N", delay=17, timeout=2345, no_wait_after=True),
        call("o", delay=17, timeout=2345, no_wait_after=True),
    ]


@pytest.mark.asyncio
async def test_strategy_aware_input_raises_playwright_timeout_for_aggregate_deadline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "clear_field", AsyncMock(side_effect=TimeoutError))

    with pytest.raises(PlaywrightTimeoutError, match="4321ms"):
        await handler_utils.strategy_aware_input(locator, "Noor", clear=True, timeout=4321)


@pytest.mark.asyncio
async def test_strategy_aware_input_keeps_long_prefix_and_tail_under_one_deadline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    deadline_active = False

    async def fill(*_args: object, **_kwargs: object) -> None:
        assert deadline_active

    async def type_text(*_args: object, **_kwargs: object) -> None:
        assert deadline_active

    locator = MagicMock()
    locator.fill = AsyncMock(side_effect=fill)
    type_text_mock = AsyncMock(side_effect=type_text)
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text_mock)
    timeout_calls: list[float | None] = []

    @asynccontextmanager
    async def record_timeout(delay: float | None) -> AsyncIterator[None]:
        nonlocal deadline_active
        timeout_calls.append(delay)
        deadline_active = True
        try:
            yield
        finally:
            deadline_active = False

    monkeypatch.setattr(handler_utils.asyncio, "timeout", record_timeout)
    text = "x" * (handler_utils.TEXT_PRESS_MAX_LENGTH + 1)

    await handler_utils.strategy_aware_input(locator, text, clear=True, timeout=4321)

    locator.fill.assert_awaited_once_with("x", timeout=4321)
    type_text_mock.assert_awaited_once_with(
        locator.page,
        locator,
        text[-handler_utils.TEXT_PRESS_MAX_LENGTH :],
        timeout=4321,
        allow_batched_playwright=True,
    )
    assert timeout_calls == [4.321]


@pytest.mark.asyncio
async def test_input_sequentially_keeps_legacy_strategy_timeout_behavior(monkeypatch: pytest.MonkeyPatch) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)
    timeout_scope = MagicMock(side_effect=AssertionError("legacy input must not gain an aggregate deadline"))
    monkeypatch.setattr(handler_utils.asyncio, "timeout", timeout_scope)

    await handler_utils.input_sequentially(locator, "Noor", timeout=4321)

    timeout_scope.assert_not_called()
    type_text.assert_awaited_once_with(locator.page, locator, "Noor")


@pytest.mark.asyncio
async def test_strategy_aware_input_appends_without_atomic_fill(monkeypatch: pytest.MonkeyPatch) -> None:
    locator = MagicMock()
    locator.fill = AsyncMock()
    type_text = AsyncMock()
    monkeypatch.setattr(handler_utils.EventStrategyFactory, "type_text", type_text)
    text = "x" * (handler_utils.TEXT_PRESS_MAX_LENGTH + 1)

    await handler_utils.strategy_aware_input(locator, text, clear=False, timeout=4321)

    locator.fill.assert_not_awaited()
    type_text.assert_awaited_once_with(
        locator.page,
        locator,
        text,
        timeout=4321,
        allow_batched_playwright=True,
    )


@pytest.mark.asyncio
async def test_is_tel_digit_fix_enabled_uses_org_keyed_rollout(monkeypatch: pytest.MonkeyPatch) -> None:
    now = datetime.now(UTC)
    organization = make_organization(now)
    task = make_task(now, organization)
    provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True))
    monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXPERIMENTATION_PROVIDER", provider)

    assert await _is_tel_digit_fix_enabled(task) is True
    provider.is_feature_enabled_cached.assert_awaited_once_with(
        "FIX_TEL_INPUT_DIGIT_DROP",
        organization.organization_id,
        properties={"organization_id": organization.organization_id},
    )