Skip to content

Automate browser based workflows with AI

11.6 KB
Raw Blame
"""Regression tests for MCP workflow list resilience to Fern SDK drift.

Context: the vendored Fern SDK at ``skyvern/client/`` validates workflow
responses through a discriminated pydantic Union that does not know about
block types added to the backend after the last SDK regeneration (currently
``google_sheets_read`` and ``google_sheets_write``). Before the Strategy B fix,
``skyvern_workflow_list`` deserialized through that stale Union and would
crash the entire page for any workflow whose definition referenced an
unknown block_type. These tests lock in the bypass: list calls go through
raw httpx and return plain dicts, so new backend block types pass through
unchanged.
"""

from __future__ import annotations

from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock

import httpx
import pytest

from skyvern.cli.mcp_tools import _workflow_http
from skyvern.cli.mcp_tools import workflow as mcp_workflow
from skyvern.client.core.client_wrapper import AsyncClientWrapper
from tests.unit._mcp_test_helpers import patch_skyvern_client as _patch_skyvern_client


def _make_workflow_dict(workflow_id: str, block_type: str, *, label: str = "step1") -> dict[str, Any]:
    """Minimal workflow payload shape, close enough to what ``v1/workflows`` returns."""
    block: dict[str, Any] = {
        "block_type": block_type,
        "label": label,
    }
    if block_type == "google_sheets_read":
        block["spreadsheet_url"] = "https://docs.google.com/spreadsheets/d/xxx/edit"
        block["sheet_name"] = "Sheet1"
        block["range"] = "A1:D100"
        block["credential_id"] = "{{ google_credential_id }}"
        block["has_header_row"] = True
    elif block_type == "pdf_fill":
        block["file_url"] = "{{ source_pdf }}"
        block["prompt"] = "Fill the PDF using the payload."
        block["payload"] = {"name": "{{ applicant.name }}"}
    elif block_type == "web_search":
        block.update(
            query="site:example.com {{ terms }}",
            provider="auto",
            num_results=100,
            no_results_error_code="NO_SEARCH_RESULTS",
            no_match_error_code="NO_MATCHING_RESULT",
            prompt="Return the relevant links.",
            json_schema={"type": "array", "items": {"type": "string"}},
            parameter_keys=["terms"],
        )
    elif block_type == "navigation":
        block["url"] = "https://example.com"
        block["navigation_goal"] = "do the thing"
    elif block_type == "terminate":
        block["reason"] = "ACCOUNT_NOT_FOUND: {{ account_number }}"
        block["error_code"] = "ACCOUNT_NOT_FOUND"
    return {
        "workflow_permanent_id": workflow_id,
        "workflow_id": f"wf_{workflow_id.split('_', 1)[-1]}",
        "title": f"Workflow {workflow_id}",
        "version": 1,
        "status": "published",
        "description": None,
        "is_saved_task": False,
        "folder_id": None,
        "created_at": "2026-04-20T10:00:00+00:00",
        "modified_at": "2026-04-22T10:00:00+00:00",
        "workflow_definition": {
            "parameters": [],
            "blocks": [block],
        },
    }


def _patch_skyvern_list_response(
    monkeypatch: pytest.MonkeyPatch,
    *,
    payload: list[dict[str, Any]],
    status_code: int = 200,
) -> AsyncMock:
    response = MagicMock()
    response.status_code = status_code
    response.json.return_value = payload
    response.text = ""

    request_mock = AsyncMock(return_value=response)
    httpx_client = SimpleNamespace(request=request_mock)
    client_wrapper = SimpleNamespace(httpx_client=httpx_client)
    fake_skyvern = SimpleNamespace(_client_wrapper=client_wrapper)

    _patch_skyvern_client(monkeypatch, fake_skyvern)
    return request_mock


@pytest.mark.asyncio
async def test_list_succeeds_when_workflow_uses_google_sheets_block(monkeypatch: pytest.MonkeyPatch) -> None:
    """Regression: a google_sheets_read block must not break list deserialization.

    Pre-fix behavior: ``skyvern.get_workflows(...)`` ran ``parse_obj_as(List[Workflow], ...)``
    on the response, and the stale Fern discriminated Union raised ValidationError
    on ``google_sheets_read``, taking the entire page down. The bypass returns raw
    dicts, so any block_type string round-trips through the tool untouched.
    """
    payload = [
        _make_workflow_dict("wpid_ok", "navigation"),
        _make_workflow_dict("wpid_sheets", "google_sheets_read"),
        _make_workflow_dict("wpid_pdf_fill", "pdf_fill"),
        _make_workflow_dict("wpid_search", "web_search"),
        _make_workflow_dict("wpid_stop", "terminate"),
    ]
    request_mock = _patch_skyvern_list_response(monkeypatch, payload=payload)

    result = await mcp_workflow.skyvern_workflow_list(page=1, page_size=10)

    assert result["ok"] is True, result
    data = result["data"]
    assert data["count"] == 5
    assert data["page"] == 1
    ids = {wf["workflow_permanent_id"] for wf in data["workflows"]}
    assert ids == {"wpid_ok", "wpid_sheets", "wpid_pdf_fill", "wpid_search", "wpid_stop"}

    request_mock.assert_awaited_once()
    call = request_mock.await_args
    assert call.args[0] == "v1/workflows"
    assert call.kwargs["method"] == "GET"
    params = call.kwargs["params"]
    assert "search_key" not in params
    assert params["page"] == 1
    assert params["page_size"] == 10
    assert params["only_workflows"] is False


@pytest.mark.parametrize("block_type", ["web_search", "terminate"])
def test_block_fields_survive_mcp_definition_normalization(block_type: str) -> None:
    workflow = _make_workflow_dict("wpid_block", block_type)
    if block_type == "web_search":
        workflow["workflow_definition"]["blocks"][0]["error_code_mapping"] = {"NO_RESULT": "No result fits."}
    workflow["workflow_definition"]["parameters"] = [
        {"parameter_type": "workflow", "key": "terms", "workflow_parameter_type": "string", "default_value": "example"}
    ]
    normalized = mcp_workflow._normalize_json_definition(workflow)
    block = normalized["workflow_definition"]["blocks"][0]
    expected = workflow["workflow_definition"]["blocks"][0]
    assert {key: block[key] for key in expected} == expected


@pytest.mark.asyncio
async def test_list_includes_search_key_only_when_search_is_present(monkeypatch: pytest.MonkeyPatch) -> None:
    """Avoid leaking a literal search_key=None query parameter."""
    request_mock = _patch_skyvern_list_response(monkeypatch, payload=[])

    result = await mcp_workflow.skyvern_workflow_list(search="invoice", page=2, page_size=5)

    assert result["ok"] is True, result
    params = request_mock.await_args.kwargs["params"]
    assert params == {
        "search_key": "invoice",
        "page": 2,
        "page_size": 5,
        "only_workflows": False,
    }


@pytest.mark.asyncio
async def test_list_accepts_query_alias_for_search(monkeypatch: pytest.MonkeyPatch) -> None:
    request_mock = _patch_skyvern_list_response(monkeypatch, payload=[])

    result = await mcp_workflow.skyvern_workflow_list(query="invoice", page=2, page_size=5)

    assert result["ok"] is True, result
    params = request_mock.await_args.kwargs["params"]
    assert params == {
        "search_key": "invoice",
        "page": 2,
        "page_size": 5,
        "only_workflows": False,
    }


@pytest.mark.asyncio
async def test_list_rejects_conflicting_search_and_query(monkeypatch: pytest.MonkeyPatch) -> None:
    request_mock = _patch_skyvern_list_response(monkeypatch, payload=[])

    result = await mcp_workflow.skyvern_workflow_list(search="invoice", query="receipt")

    assert result["ok"] is False
    assert result["error"]["code"] == "INVALID_INPUT"
    request_mock.assert_not_awaited()


@pytest.mark.asyncio
async def test_raw_list_uses_fern_http_wrapper_auth_headers(monkeypatch: pytest.MonkeyPatch) -> None:
    """The raw bypass still uses Fern's HTTP wrapper, including auth and query encoding."""
    seen_requests: list[httpx.Request] = []

    async def handler(request: httpx.Request) -> httpx.Response:
        seen_requests.append(request)
        return httpx.Response(200, json=[])

    async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as base_client:
        client_wrapper = AsyncClientWrapper(
            api_key="sk_test",
            headers={"x-test-header": "present"},
            base_url="https://api.example.test/api",
            timeout=60,
            httpx_client=base_client,
        )
        fake_skyvern = SimpleNamespace(_client_wrapper=client_wrapper)
        _patch_skyvern_client(monkeypatch, fake_skyvern)

        result = await mcp_workflow.skyvern_workflow_list(only_workflows=True)

    assert result["ok"] is True, result
    assert len(seen_requests) == 1
    request = seen_requests[0]
    assert request.url.path == "/api/v1/workflows"
    assert request.headers["x-api-key"] == "sk_test"
    assert request.headers["x-test-header"] == "present"
    assert request.headers["x-fern-sdk-name"] == "skyvern"
    assert request.url.params["only_workflows"] == "true"


def test_extract_error_detail_truncates_non_json_response_text() -> None:
    """Avoid returning an entire proxy/load-balancer HTML page in MCP errors."""
    response = MagicMock()
    response.json.side_effect = ValueError("not json")
    response.text = "x" * 600

    detail = _workflow_http.extract_error_detail(response)

    assert detail == "x" * _workflow_http._ERROR_DETAIL_LIMIT


def test_extract_error_detail_truncates_long_json_detail() -> None:
    """A verbose ``detail`` field (e.g., a backend stack trace) is truncated too."""
    response = MagicMock()
    response.json.return_value = {"detail": "x" * 600}

    detail = _workflow_http.extract_error_detail(response)

    assert detail == "x" * _workflow_http._ERROR_DETAIL_LIMIT


def test_decode_success_payload_wraps_non_json_response() -> None:
    """A non-JSON 2xx response should carry operation context, not a bare decoder error."""
    response = MagicMock()
    response.json.side_effect = ValueError("not json")
    response.text = "<html>proxy response</html>"

    with pytest.raises(RuntimeError, match="Unexpected workflows list response"):
        _workflow_http._decode_success_payload(response, operation="workflows list")


@pytest.mark.asyncio
async def test_list_surfaces_http_error(monkeypatch: pytest.MonkeyPatch) -> None:
    """Non-2xx responses return an API_ERROR result, not a crash."""
    response = MagicMock()
    response.status_code = 500
    response.json.return_value = {"detail": "boom"}
    response.text = '{"detail": "boom"}'

    request_mock = AsyncMock(return_value=response)
    fake_skyvern = SimpleNamespace(_client_wrapper=SimpleNamespace(httpx_client=SimpleNamespace(request=request_mock)))
    _patch_skyvern_client(monkeypatch, fake_skyvern)

    result = await mcp_workflow.skyvern_workflow_list()

    assert result["ok"] is False
    assert "500" in result["error"]["message"]


@pytest.mark.asyncio
async def test_list_preserves_serialization_shape(monkeypatch: pytest.MonkeyPatch) -> None:
    """Response payload keys stay stable so MCP clients see no contract change."""
    payload = [_make_workflow_dict("wpid_ok", "navigation")]
    _patch_skyvern_list_response(monkeypatch, payload=payload)

    result = await mcp_workflow.skyvern_workflow_list(page_size=5)

    assert result["ok"] is True
    data = result["data"]
    assert set(data.keys()) == {"workflows", "page", "page_size", "count", "has_more", "sdk_equivalent"}
    wf = data["workflows"][0]
    expected_keys = {
        "workflow_permanent_id",
        "workflow_id",
        "title",
        "version",
        "status",
        "description",
        "is_saved_task",
        "folder_id",
        "created_at",
        "modified_at",
    }
    assert expected_keys <= set(wf.keys())
    assert wf["created_at"] == "2026-04-20T10:00:00+00:00"