Skip to content

Automate browser based workflows with AI

22.7 KB
Raw Blame
from __future__ import annotations

from datetime import UTC, datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, call

import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError

from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.services.run_cancellation import RunCancellation
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import InputOrSelectContext, SelectOption, SelectOptionAction
from tests.unit.helpers import make_organization, make_task


class _FakeSelectElement:
    def __init__(self, options: list[dict], locator: MagicMock, selected_attr: str | None = None) -> None:
        self._options = options
        self._locator = locator
        self.get_attr = AsyncMock(return_value=selected_attr)
        self.refresh_select_options = AsyncMock(return_value=(options, ""))

    def get_locator(self) -> MagicMock:
        return self._locator

    def get_options(self) -> list[dict]:
        return self._options

    def build_HTML(self) -> str:
        return "<select></select>"


def _select_action(label: str) -> SelectOptionAction:
    return SelectOptionAction(
        element_id="select-1",
        option=SelectOption(label=label),
        input_or_select_context=InputOrSelectContext(field="Department", is_required=True),
    )


def _task() -> object:
    now = datetime.now(UTC)
    organization = make_organization(now)
    return make_task(now, organization)


def _selected_option_readback(
    options: list[dict],
    selected_value: str,
    *,
    selected_label: str | None = None,
    selected_index: int | None = None,
) -> dict:
    for index, option in enumerate(options):
        if option.get("value") == selected_value:
            return {
                "index": option.get("optionIndex", index) if selected_index is None else selected_index,
                "label": option.get("text") if selected_label is None else selected_label,
                "value": selected_value,
            }
    return {"index": selected_index, "label": selected_label or selected_value, "value": selected_value}


def _locator(
    options: list[dict],
    selected_value: str,
    *,
    selected_label: str | None = None,
    selected_index: int | None = None,
    evaluate_side_effect: BaseException | None = None,
) -> MagicMock:
    locator = MagicMock()
    locator.click = AsyncMock()
    locator.select_option = AsyncMock()
    locator.input_value = AsyncMock(return_value=selected_value)
    if evaluate_side_effect is not None:
        locator.evaluate = AsyncMock(side_effect=evaluate_side_effect)
    else:
        locator.evaluate = AsyncMock(
            return_value=_selected_option_readback(
                options,
                selected_value,
                selected_label=selected_label,
                selected_index=selected_index,
            )
        )
    return locator


async def _run_normal_select(
    monkeypatch: pytest.MonkeyPatch,
    *,
    feature_enabled: bool,
    action: SelectOptionAction,
    options: list[dict],
    selected_value: str,
    selected_attr: str | None = None,
    selected_label: str | None = None,
    selected_index: int | None = None,
    llm_response: dict | None = None,
    evaluate_side_effect: BaseException | None = None,
    collapse_gate: AsyncMock | None = None,
    agent_function: object | None = None,
    normal_select_llm: AsyncMock | None = None,
    cancellation_reader: AsyncMock | None = None,
) -> tuple[list, MagicMock, AsyncMock]:
    provider = SimpleNamespace(
        is_feature_enabled_cached=AsyncMock(return_value=feature_enabled),
        resolve_feature_enabled_unrecorded=AsyncMock(return_value=feature_enabled),
    )
    normal_select_llm = normal_select_llm or AsyncMock(
        return_value=llm_response or {"value": "fallback", "index": None}
    )

    if collapse_gate is None:
        monkeypatch.setattr(handler.app, "EXPERIMENTATION_PROVIDER", provider)
    else:
        monkeypatch.setattr(handler, "_is_collapse_select_fanout_enabled", collapse_gate)
    monkeypatch.setattr(handler.app, "AGENT_FUNCTION", agent_function or AgentFunction())
    monkeypatch.setattr(handler.app, "NORMAL_SELECT_AGENT_LLM_API_HANDLER", normal_select_llm)
    monkeypatch.setattr(handler, "read_run_cancellation", cancellation_reader or AsyncMock(return_value=None))
    monkeypatch.setattr(handler.prompt_engine, "load_prompt", MagicMock(return_value="prompt"))
    monkeypatch.setattr(handler.skyvern_context, "ensure_context", MagicMock(return_value=SimpleNamespace(tz_info=UTC)))

    locator = _locator(
        options,
        selected_value,
        selected_label=selected_label,
        selected_index=selected_index,
        evaluate_side_effect=evaluate_side_effect,
    )
    result = await handler.normal_select(
        action=action,
        skyvern_element=_FakeSelectElement(options, locator, selected_attr),  # type: ignore[arg-type]
        task=_task(),  # type: ignore[arg-type]
        step=MagicMock(),
        builder=MagicMock(),
    )
    return result, locator, normal_select_llm


@pytest.mark.parametrize(("cancel_during_llm", "selects"), [(True, False), (False, True)])
@pytest.mark.asyncio
async def test_normal_select_does_not_select_after_cancel_during_llm(
    monkeypatch: pytest.MonkeyPatch,
    cancel_during_llm: bool,
    selects: bool,
) -> None:
    llm_finished = False

    async def llm_side_effect(**_kwargs: object) -> dict[str, str | int | None]:
        nonlocal llm_finished
        llm_finished = True
        return {"value": "US", "index": None}

    cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
    cancellation_reader = AsyncMock(
        side_effect=lambda *_args, **_kwargs: cancellation if llm_finished and cancel_during_llm else None
    )
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=False,
        action=_select_action("United States"),
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_value="US",
        normal_select_llm=AsyncMock(side_effect=llm_side_effect),
        cancellation_reader=cancellation_reader,
    )

    assert locator.select_option.await_count == (1 if selects else 0)
    normal_select_llm.assert_awaited_once()
    assert handler._is_terminal_custom_select_failure(result[-1]) is (not selects)
    if not selects:
        cancellation_reader.assert_awaited_once()


@pytest.mark.asyncio
async def test_normal_select_does_not_deterministically_select_after_cancel_during_context_parse(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    context_parse_finished = False

    async def parse_context(**_kwargs: object) -> InputOrSelectContext:
        nonlocal context_parse_finished
        context_parse_finished = True
        return InputOrSelectContext(field="Department", is_required=True)

    context_parser = AsyncMock(side_effect=parse_context)
    monkeypatch.setattr(handler, "_get_input_or_select_context", context_parser)
    cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
    cancellation_reader = AsyncMock(
        side_effect=lambda *_args, **_kwargs: cancellation if context_parse_finished else None
    )
    agent_function = SimpleNamespace(
        resolve_field_option=AsyncMock(
            return_value=SimpleNamespace(
                fallback_to_llm=False,
                matched_index=0,
                matched_label="United States",
                matched_value="US",
            )
        )
    )
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=_select_action("United States"),
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_value="US",
        collapse_gate=AsyncMock(return_value=True),
        agent_function=agent_function,
        cancellation_reader=cancellation_reader,
    )

    assert handler._is_terminal_custom_select_failure(result[-1])
    locator.select_option.assert_not_awaited()
    normal_select_llm.assert_not_awaited()
    context_parser.assert_awaited_once()
    cancellation_reader.assert_awaited_once()


@pytest.mark.asyncio
async def test_normal_select_does_not_deterministically_select_after_cancel_during_focus(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    focus_finished = False

    async def focus_before_select(**_kwargs: object) -> None:
        nonlocal focus_finished
        focus_finished = True

    monkeypatch.setattr(handler, "_best_effort_focus_click_before_select", focus_before_select)
    cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
    cancellation_reader = AsyncMock(side_effect=lambda *_args, **_kwargs: cancellation if focus_finished else None)
    agent_function = SimpleNamespace(
        resolve_field_option=AsyncMock(
            return_value=SimpleNamespace(
                fallback_to_llm=False,
                matched_index=0,
                matched_label="United States",
                matched_value="US",
            )
        )
    )

    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=_select_action("United States"),
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_value="US",
        collapse_gate=AsyncMock(return_value=True),
        agent_function=agent_function,
        cancellation_reader=cancellation_reader,
    )

    assert handler._is_terminal_custom_select_failure(result[-1])
    locator.select_option.assert_not_awaited()
    normal_select_llm.assert_not_awaited()
    assert cancellation_reader.await_count == 2


@pytest.mark.asyncio
@pytest.mark.parametrize("feature_enabled", [False, True])
async def test_normal_select_does_not_try_fallback_after_cancel_between_attempts(
    monkeypatch: pytest.MonkeyPatch,
    feature_enabled: bool,
) -> None:
    cancellation = RunCancellation(TaskStatus.canceled, from_workflow_run=False)
    cancellation_reader = AsyncMock(side_effect=[None, None, cancellation])

    async def focus_before_select(*, locator: MagicMock, **_kwargs: object) -> None:
        locator.select_option.side_effect = Exception("timed out")

    monkeypatch.setattr(handler, "_best_effort_focus_click_before_select", focus_before_select)
    agent_function = SimpleNamespace(
        resolve_field_option=AsyncMock(
            return_value=SimpleNamespace(
                fallback_to_llm=False,
                matched_index=0,
                matched_label="United States",
                matched_value="US",
            )
        )
    )
    result, locator, _ = await _run_normal_select(
        monkeypatch,
        feature_enabled=feature_enabled,
        action=_select_action("United States"),
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_value="US",
        llm_response={"value": "US", "index": 0},
        collapse_gate=AsyncMock(return_value=feature_enabled),
        agent_function=agent_function,
        cancellation_reader=cancellation_reader,
    )

    assert handler._is_terminal_custom_select_failure(result[-1])
    assert locator.select_option.await_args_list == [
        call(value="US", timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)
    ]
    assert cancellation_reader.await_count == 3


@pytest.mark.parametrize(
    ("target", "options", "selected_value"),
    [
        (
            "United States",
            [
                {"optionIndex": 0, "text": "Canada", "value": "CA"},
                {"optionIndex": 1, "text": "United States", "value": "US"},
            ],
            "US",
        ),
        (
            "Departments",
            [
                {"optionIndex": 0, "text": "Department", "value": "department"},
                {"optionIndex": 1, "text": "Role", "value": "role"},
            ],
            "department",
        ),
    ],
)
@pytest.mark.asyncio
async def test_normal_select_exact_or_stem_match_skips_llm(
    monkeypatch: pytest.MonkeyPatch,
    target: str,
    options: list[dict],
    selected_value: str,
) -> None:
    gate = AsyncMock(return_value=True)
    action = _select_action(target)
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=options,
        selected_value=selected_value,
        collapse_gate=gate,
    )

    assert handler._normal_select_successful(result)
    assert not action.has_mini_agent
    gate.assert_awaited_once()
    locator.select_option.assert_awaited_once_with(
        value=selected_value,
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )
    normal_select_llm.assert_not_awaited()


@pytest.mark.asyncio
async def test_normal_select_duplicate_value_selects_matched_label(monkeypatch: pytest.MonkeyPatch) -> None:
    action = _select_action("Target Department")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=[
            {"optionIndex": 0, "text": "Wrong Department", "value": "shared"},
            {"optionIndex": 1, "text": "Target Department", "value": "shared"},
        ],
        selected_value="shared",
        selected_label="Target Department",
        selected_index=1,
    )

    assert handler._normal_select_successful(result)
    locator.select_option.assert_awaited_once_with(
        label="Target Department",
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )
    locator.evaluate.assert_awaited_once()
    normal_select_llm.assert_not_awaited()


@pytest.mark.asyncio
async def test_normal_select_duplicate_value_wrong_label_falls_back_to_llm(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    action = _select_action("Target Department")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=[
            {"optionIndex": 0, "text": "Wrong Department", "value": "shared"},
            {"optionIndex": 1, "text": "Target Department", "value": "shared"},
        ],
        selected_value="shared",
        selected_label="Wrong Department",
        selected_index=0,
        llm_response={"value": "Target Department", "index": 1},
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    assert locator.select_option.await_args_list[0] == call(
        label="Target Department",
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )
    locator.evaluate.assert_awaited_once()
    normal_select_llm.assert_awaited_once()


@pytest.mark.parametrize(
    ("target", "options", "llm_value"),
    [
        (
            "Senior",
            [
                {"optionIndex": 0, "text": "Senior Vice President", "value": "svp"},
                {"optionIndex": 1, "text": "Associate", "value": "associate"},
            ],
            "svp",
        ),
        (
            "United States",
            [
                {"optionIndex": 0, "text": "United States", "value": "US"},
                {"optionIndex": 1, "text": "United States", "value": "USA"},
            ],
            "USA",
        ),
    ],
)
@pytest.mark.asyncio
async def test_normal_select_unsafe_match_falls_back_to_llm(
    monkeypatch: pytest.MonkeyPatch,
    target: str,
    options: list[dict],
    llm_value: str,
) -> None:
    gate = AsyncMock(return_value=True)
    action = _select_action(target)
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=options,
        selected_value=llm_value,
        llm_response={"value": llm_value, "index": None},
        collapse_gate=gate,
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    gate.assert_awaited_once()
    locator.select_option.assert_awaited_once_with(value=llm_value, timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)
    normal_select_llm.assert_awaited_once()


@pytest.mark.asyncio
async def test_normal_select_readback_value_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:
    action = _select_action("United States")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=[
            {"optionIndex": 0, "text": "Canada", "value": "CA"},
            {"optionIndex": 1, "text": "United States", "value": "US"},
        ],
        selected_value="CA",
        llm_response={"value": "US", "index": None},
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    locator.select_option.assert_has_awaits(
        [
            call(value="US", timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS),
            call(value="US", timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS),
        ]
    )
    normal_select_llm.assert_awaited_once()


@pytest.mark.parametrize("scraped_label", [" United States", "United States "])
@pytest.mark.asyncio
async def test_normal_select_edge_whitespace_label_verifies_without_llm(
    monkeypatch: pytest.MonkeyPatch,
    scraped_label: str,
) -> None:
    """Read-back trims the label but the scraped option text does not (domUtils.removeMultipleSpaces
    collapses runs without trimming), so an edge-space-only difference must not read as a mismatch."""
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=_select_action("United States"),
        options=[
            {"optionIndex": 0, "text": "Canada", "value": "CA"},
            {"optionIndex": 1, "text": scraped_label, "value": "US"},
        ],
        selected_value="US",
        selected_label="United States",
        selected_index=1,
    )

    assert handler._normal_select_successful(result)
    normal_select_llm.assert_not_awaited()
    locator.select_option.assert_awaited_once_with(
        value="US",
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )


@pytest.mark.parametrize("scraped_value", [" US", "US "])
@pytest.mark.asyncio
async def test_normal_select_edge_whitespace_value_verifies_without_llm(
    monkeypatch: pytest.MonkeyPatch,
    scraped_value: str,
) -> None:
    """domUtils.removeMultipleSpaces() leaves option.value untrimmed too, so the value arm needs the
    same normalization as the label arm."""
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=_select_action("United States"),
        options=[
            {"optionIndex": 0, "text": "Canada", "value": "CA"},
            {"optionIndex": 1, "text": "United States", "value": scraped_value},
        ],
        selected_value="US",
        selected_label="United States",
        selected_index=1,
    )

    assert handler._normal_select_successful(result)
    normal_select_llm.assert_not_awaited()
    locator.select_option.assert_awaited_once_with(
        value=scraped_value,
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )


@pytest.mark.asyncio
async def test_normal_select_readback_timeout_keeps_success_and_skips_stale_locator(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """select_option already committed the value; a select-triggered rerender detaches the element so
    the read-back times out. That must not spend the Playwright default 30s, and must not drive the LLM
    fallback through the now-stale locator."""
    action = _select_action("United States")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=[
            {"optionIndex": 0, "text": "Canada", "value": "CA"},
            {"optionIndex": 1, "text": "United States", "value": "US"},
        ],
        selected_value="US",
        evaluate_side_effect=PlaywrightTimeoutError("Timeout 30000ms exceeded waiting for locator"),
        llm_response={"value": "US", "index": None},
    )

    assert handler._normal_select_successful(result)
    normal_select_llm.assert_not_awaited()
    assert not action.has_mini_agent
    locator.select_option.assert_awaited_once_with(
        value="US",
        timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS,
    )
    assert locator.evaluate.await_args.kwargs["timeout"] == handler.settings.BROWSER_ACTION_TIMEOUT_MS


@pytest.mark.asyncio
async def test_normal_select_readback_index_mismatch_falls_back_to_llm(monkeypatch: pytest.MonkeyPatch) -> None:
    action = _select_action("United States")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=True,
        action=action,
        options=[
            {"optionIndex": 0, "text": "Canada", "value": "CA"},
            {"optionIndex": 1, "text": "United States", "value": "US"},
        ],
        selected_value="US",
        selected_index=0,
        llm_response={"value": "US", "index": None},
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    normal_select_llm.assert_awaited_once()


@pytest.mark.asyncio
async def test_normal_select_flag_off_uses_llm_path(monkeypatch: pytest.MonkeyPatch) -> None:
    gate = AsyncMock(return_value=False)
    resolver = AsyncMock()
    action = _select_action("United States")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=False,
        action=action,
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_value="US",
        llm_response={"value": "US", "index": None},
        collapse_gate=gate,
        agent_function=SimpleNamespace(resolve_field_option=resolver),
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    gate.assert_awaited_once()
    resolver.assert_not_awaited()
    locator.select_option.assert_awaited_once_with(value="US", timeout=handler.settings.BROWSER_ACTION_TIMEOUT_MS)
    normal_select_llm.assert_awaited_once()


@pytest.mark.asyncio
async def test_normal_select_flag_off_marks_mini_agent_before_already_selected_return(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    action = _select_action("United States")
    result, locator, normal_select_llm = await _run_normal_select(
        monkeypatch,
        feature_enabled=False,
        action=action,
        options=[{"optionIndex": 0, "text": "United States", "value": "US"}],
        selected_attr="United States",
        selected_value="US",
        llm_response={"value": "US", "index": None},
    )

    assert handler._normal_select_successful(result)
    assert action.has_mini_agent is True
    locator.select_option.assert_not_awaited()
    normal_select_llm.assert_not_awaited()