Skip to content

Automate browser based workflows with AI

10.3 KB
Raw Blame
"""Tests for the concise MCP response mode in make_result()."""

from __future__ import annotations

import json
from collections.abc import Iterator

import pytest

import skyvern.cli.mcp_tools.workflow as workflow_tools
from skyvern.cli.core.result import Artifact, BrowserContext, make_result, set_concise_responses


@pytest.fixture(autouse=True)
def _enable_concise() -> Iterator[None]:
    """Enable concise mode for every test; restore after."""
    set_concise_responses(True)
    yield
    set_concise_responses(False)


# -- Helpers ------------------------------------------------------------------

_CTX = BrowserContext(mode="cdp", session_id="pbs_1", cdp_url="wss://example.com/devtools")

_CLICK_DATA = {
    "selector": "#btn",
    "intent": "the Submit button",
    "ai_mode": "proactive",
    "resolved_selector": "xpath=/*[name()='html'][1]/*[name()='body'][1]/*[name()='button'][1]",
    "sdk_equivalent": 'await page.click("xpath=...", prompt="the Submit button")',
}


# -- Stripped fields ----------------------------------------------------------


def test_concise_strips_action_and_browser_context() -> None:
    result = make_result("skyvern_click", browser_context=_CTX, data=_CLICK_DATA)
    assert "action" not in result
    assert "browser_context" not in result


def test_concise_strips_timing() -> None:
    result = make_result("skyvern_click", data=_CLICK_DATA, timing_ms={"sdk": 500, "total": 500})
    assert "timing_ms" not in result


@pytest.mark.parametrize("key", ["sdk_equivalent", "ai_mode", "selector", "intent"])
def test_concise_strips_debug_data_keys(key: str) -> None:
    data = {key: "some_value", "url": "https://example.com"}
    result = make_result("skyvern_navigate", data=data)
    assert key not in result.get("data", {})


def test_concise_strips_none_values_from_data() -> None:
    data = {"url": "https://example.com", "title": None}
    result = make_result("skyvern_navigate", data=data)
    assert "title" not in result.get("data", {})


def test_concise_omits_data_when_all_keys_stripped() -> None:
    """When every key in data is strippable, the data key should be omitted entirely."""
    data = {"sdk_equivalent": "await page.click(...)", "ai_mode": "proactive", "selector": "#x", "intent": "foo"}
    result = make_result("skyvern_click", data=data)
    assert "data" not in result


# -- Minimal response --------------------------------------------------------


def test_concise_minimal_response() -> None:
    """No data, no error, no artifacts — should return just {"ok": True}."""
    result = make_result("skyvern_click")
    assert result == {"ok": True}


# -- Omitted empty collections -----------------------------------------------


def test_concise_omits_empty_artifacts() -> None:
    result = make_result("skyvern_click", data=_CLICK_DATA, artifacts=[])
    assert "artifacts" not in result


def test_concise_omits_empty_warnings() -> None:
    result = make_result("skyvern_click", data=_CLICK_DATA, warnings=[])
    assert "warnings" not in result


def test_concise_omits_null_error() -> None:
    result = make_result("skyvern_click", data=_CLICK_DATA, error=None)
    assert "error" not in result


# -- Preserved fields ---------------------------------------------------------


def test_concise_click_preserves_resolved_selector() -> None:
    """resolved_selector is actionable feedback — shows what the AI resolver matched."""
    result = make_result("skyvern_click", data=_CLICK_DATA)
    assert result["data"]["resolved_selector"] == _CLICK_DATA["resolved_selector"]


def test_concise_click_strips_other_echoed_fields() -> None:
    result = make_result("skyvern_click", data=_CLICK_DATA)
    data = result.get("data", {})
    assert "selector" not in data
    assert "intent" not in data
    assert "ai_mode" not in data
    assert "sdk_equivalent" not in data


def test_concise_preserves_meaningful_data() -> None:
    data = {"extracted": {"price": 42.0}, "sdk_equivalent": "await page.extract(...)"}
    result = make_result("skyvern_extract", data=data)
    assert result["data"] == {"extracted": {"price": 42.0}}


def test_concise_preserves_trajectory_capture_status() -> None:
    result = make_result(
        "skyvern_trajectory_get",
        data={"trajectory_json": "[]", "entry_count": 0, "truncated": False, "capture_status": "not_found"},
    )

    assert result["data"]["capture_status"] == "not_found"


def test_concise_preserves_error() -> None:
    err = {"code": "SELECTOR_NOT_FOUND", "message": "Not found", "hint": "Try another selector"}
    result = make_result("skyvern_click", ok=False, error=err)
    assert result["ok"] is False
    assert result["error"] == err


def test_concise_preserves_nonempty_warnings() -> None:
    result = make_result("skyvern_click", ok=False, warnings=["Element hidden"])
    assert result["warnings"] == ["Element hidden"]


def test_concise_preserves_nonempty_artifacts() -> None:
    artifact = Artifact(kind="screenshot", path="/tmp/shot.png", mime="image/png", bytes=1024)
    result = make_result("skyvern_screenshot", artifacts=[artifact])
    assert len(result["artifacts"]) == 1
    assert result["artifacts"][0]["path"] == "/tmp/shot.png"


# -- Partial failure with data ------------------------------------------------


def test_concise_preserves_data_on_failure() -> None:
    err = {"code": "TIMEOUT", "message": "Timed out", "hint": "Increase timeout"}
    data = {"partial_result": {"items": 3}, "sdk_equivalent": "await page.extract(...)"}
    result = make_result("skyvern_extract", ok=False, error=err, data=data)
    assert result["ok"] is False
    assert result["error"] == err
    assert result["data"] == {"partial_result": {"items": 3}}


# -- None-preserving keys (result, extracted) ---------------------------------


def test_concise_preserves_none_result_for_evaluate() -> None:
    """JS returning null is a meaningful answer — must not be stripped."""
    data = {"result": None, "sdk_equivalent": "await page.evaluate(...)"}
    result = make_result("skyvern_evaluate", data=data)
    assert "data" in result
    assert result["data"]["result"] is None


def test_concise_preserves_none_extracted() -> None:
    """Extraction returning None means 'found nothing' — must not be stripped."""
    data = {"extracted": None, "sdk_equivalent": "await page.extract(...)"}
    result = make_result("skyvern_extract", data=data)
    assert "data" in result
    assert result["data"]["extracted"] is None


# -- Verbose mode (flag off) --------------------------------------------------


def test_verbose_returns_all_fields() -> None:
    set_concise_responses(False)
    result = make_result(
        "skyvern_click",
        browser_context=_CTX,
        data=_CLICK_DATA,
        timing_ms={"sdk": 500, "total": 500},
    )
    assert result["action"] == "skyvern_click"
    assert "browser_context" in result
    assert result["timing_ms"] == {"sdk": 500, "total": 500}
    assert result["data"]["sdk_equivalent"] is not None
    assert result["data"]["resolved_selector"] is not None
    assert result["artifacts"] == []
    assert result["warnings"] == []


def test_workflow_status_summary_stays_bounded_for_heavy_payload() -> None:
    long_url = "https://artifacts.skyvern.example/" + ("x" * 1450)
    heavy_run = {
        "workflow_run_id": "wr_heavy",
        "status": "terminated",
        "failure_reason": "Execution terminated after repeated navigation failures",
        "workflow_title": "Heavy workflow",
        "recording_url": long_url,
        "screenshot_urls": [f"{long_url}-{idx}" for idx in range(6)],
        "downloaded_files": [{"url": f"{long_url}-download", "filename": "case-export.csv"}],
        "outputs": {
            "collect_customer_data": {
                "task_screenshot_artifact_ids": [f"art_task_{idx}" for idx in range(12)],
                "workflow_screenshot_artifact_ids": [f"art_workflow_{idx}" for idx in range(12)],
                "task_screenshots": [f"{long_url}-task-{idx}" for idx in range(4)],
                "workflow_screenshots": [f"{long_url}-workflow-{idx}" for idx in range(4)],
                "extracted_information": [{"account_id": "acct_123", "status": "terminated"}],
            },
            "submit_case": [
                {
                    "task_screenshot_artifact_ids": [f"art_nested_{idx}" for idx in range(6)],
                    "workflow_screenshot_artifact_ids": [f"art_followup_{idx}" for idx in range(6)],
                    "task_screenshots": [f"{long_url}-nested-task-{idx}" for idx in range(3)],
                    "workflow_screenshots": [f"{long_url}-nested-workflow-{idx}" for idx in range(4)],
                }
            ],
            "extracted_information": [{"duplicated_rollup": True}],
        },
        "run_with": "code",
    }

    full_payload = workflow_tools._serialize_run_full(heavy_run)
    summary_payload = workflow_tools._serialize_run_summary(heavy_run)

    assert len(json.dumps(full_payload)) > 20_000
    # The download URL is the one URL a summary keeps verbatim — it is what makes
    # the file retrievable — so the bound covers it on top of the URL-free summary.
    assert len(json.dumps(summary_payload)) < 2_000 + len(f"{long_url}-download")

    result = make_result("skyvern_workflow_status", data=summary_payload)
    assert len(json.dumps(result)) < 2_200 + len(f"{long_url}-download")
    assert "recording_url" not in result["data"]
    assert "output" not in result["data"]
    assert result["data"]["artifact_summary"]["artifact_id_count"] == 36
    assert result["data"]["artifact_summary"]["downloaded_files"] == [
        {"url": f"{long_url}-download", "filename": "case-export.csv"}
    ]


def test_workflow_status_summary_shrinks_simple_payload() -> None:
    long_url = "https://artifacts.skyvern.example/" + ("y" * 1450)
    simple_run = {
        "workflow_run_id": "wr_simple",
        "status": "completed",
        "workflow_title": "Simple workflow",
        "recording_url": long_url,
        "screenshot_urls": [f"{long_url}-shot"],
        "outputs": {
            "result": "success",
            "order_id": "ord_123",
        },
        "run_with": "code",
    }

    full_payload = workflow_tools._serialize_run_full(simple_run)
    summary_payload = workflow_tools._serialize_run_summary(simple_run)

    assert len(json.dumps(full_payload)) > 1_500
    assert len(json.dumps(summary_payload)) < 800
    assert summary_payload["output_summary"]["scalar_preview"] == {"result": "success", "order_id": "ord_123"}