Skip to content

Automate browser based workflows with AI

6.8 KB
Raw Blame
from __future__ import annotations

import asyncio
from unittest.mock import AsyncMock

import pytest
from typer.testing import CliRunner

from skyvern.cli.config_command import config_app
from skyvern.cli.mcp_tools.org import _UPDATE_FIELDS, skyvern_org_update
from skyvern.forge.sdk.schemas.organizations import OrganizationUpdate


class TestOrganizationUpdateSchema:
    def test_accepts_all_settable_fields(self) -> None:
        update = OrganizationUpdate(
            max_steps_per_run=25,
            max_retries_per_step=3,
            webhook_callback_url="https://example.com/hook",
            artifact_url_expiry_seconds=3600,
            default_llm_key="CUSTOM_LLM_oat_primary",
            default_secondary_llm_key="CUSTOM_LLM_oat_secondary",
        )
        assert update.model_dump(exclude_unset=True) == {
            "max_steps_per_run": 25,
            "max_retries_per_step": 3,
            "webhook_callback_url": "https://example.com/hook",
            "artifact_url_expiry_seconds": 3600,
            "default_llm_key": "CUSTOM_LLM_oat_primary",
            "default_secondary_llm_key": "CUSTOM_LLM_oat_secondary",
        }

    def test_partial_update_excludes_unset(self) -> None:
        update = OrganizationUpdate(max_steps_per_run=10)
        assert update.model_dump(exclude_unset=True) == {"max_steps_per_run": 10}

    def test_clear_artifact_flag_defaults_false(self) -> None:
        assert OrganizationUpdate().clear_artifact_url_expiry_seconds is False

    def test_clear_max_steps_per_workflow_run_defaults_false(self) -> None:
        assert OrganizationUpdate().clear_max_steps_per_workflow_run is False

    def test_clear_default_llm_flags_default_false(self) -> None:
        update = OrganizationUpdate()

        assert update.clear_default_llm_key is False
        assert update.clear_default_secondary_llm_key is False

    def test_accepts_max_steps_per_workflow_run(self) -> None:
        update = OrganizationUpdate(max_steps_per_workflow_run=42)
        assert update.model_dump(exclude_unset=True) == {"max_steps_per_workflow_run": 42}

    def test_rejects_zero_max_steps_per_workflow_run(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_steps_per_workflow_run=0)

    def test_rejects_negative_max_steps_per_workflow_run(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_steps_per_workflow_run=-5)

    def test_clear_max_steps_per_workflow_run_can_be_set(self) -> None:
        update = OrganizationUpdate(clear_max_steps_per_workflow_run=True)
        assert update.clear_max_steps_per_workflow_run is True

    def test_rejects_non_int_max_steps(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_steps_per_run="not a number")  # type: ignore[arg-type]

    def test_zero_max_retries_round_trips(self) -> None:
        # 0 means "disable retries" — see ForgeAgent.execute_step.
        update = OrganizationUpdate(max_retries_per_step=0)
        assert update.model_dump(exclude_unset=True) == {"max_retries_per_step": 0}

    def test_rejects_zero_max_steps_per_run(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_steps_per_run=0)

    def test_rejects_negative_max_steps_per_run(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_steps_per_run=-1)

    def test_accepts_max_steps_per_run_above_150(self) -> None:
        # This schema is shared with the self-hosted CLI/MCP tool, so it must
        # not carry Skyvern Cloud's frontend-only cap.
        update = OrganizationUpdate(max_steps_per_run=500)
        assert update.model_dump(exclude_unset=True) == {"max_steps_per_run": 500}

    def test_rejects_negative_max_retries_per_step(self) -> None:
        with pytest.raises(ValueError):
            OrganizationUpdate(max_retries_per_step=-1)

    def test_empty_webhook_url_round_trips(self) -> None:
        # "" clears the webhook via the repository's ``is not None`` guard.
        update = OrganizationUpdate(webhook_callback_url="")
        assert update.model_dump(exclude_unset=True) == {"webhook_callback_url": ""}

    def test_raw_aws_load_balancer_webhook_url_is_deferred_to_the_update_route(self) -> None:
        url = "https://service-123.us-east-1.elb.amazonaws.com/webhook"

        update = OrganizationUpdate(webhook_callback_url=url)

        assert update.webhook_callback_url == url


class TestMcpUpdateFieldsDerivedFromSchema:
    def test_update_fields_match_schema(self) -> None:
        assert _UPDATE_FIELDS == frozenset(OrganizationUpdate.model_fields)


class TestMcpUpdateRejectsNoneValues:
    def test_explicit_none_rejected(self) -> None:
        result = asyncio.run(skyvern_org_update(updates={"max_steps_per_run": None}))
        assert not result["ok"]
        assert "None" in result["error"]["message"]

    def test_raw_load_balancer_webhook_server_rejection_returns_structured_error(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        raw_http_put = AsyncMock(
            side_effect=RuntimeError(
                "HTTP 400: Webhook URL must use a stable custom hostname instead of an AWS load balancer DNS name."
            )
        )
        monkeypatch.setattr("skyvern.cli.mcp_tools.org.raw_http_put", raw_http_put)

        result = asyncio.run(
            skyvern_org_update(updates={"webhook_callback_url": "https://service-123.elb.us-east-1.amazonaws.com/hook"})
        )

        assert not result["ok"]
        assert "stable custom hostname" in result["error"]["message"]
        raw_http_put.assert_awaited_once()


class TestConfigCli:
    def test_set_rejects_unknown_key(self) -> None:
        runner = CliRunner()
        result = runner.invoke(config_app, ["set", "totally_made_up_key", "5"])
        assert result.exit_code != 0
        assert "Unknown key" in result.output or "Unknown key" in (result.stderr or "")

    def test_get_rejects_unknown_key(self) -> None:
        runner = CliRunner()
        result = runner.invoke(config_app, ["get", "totally_made_up_key"])
        assert result.exit_code != 0

    def test_get_rejects_write_only_clear_flag(self) -> None:
        # clear_artifact_url_expiry_seconds is a verb — readable settings exclude it.
        runner = CliRunner()
        result = runner.invoke(config_app, ["get", "clear_artifact_url_expiry_seconds"])
        assert result.exit_code != 0

    def test_set_rejects_non_int_for_int_key(self) -> None:
        runner = CliRunner()
        result = runner.invoke(config_app, ["set", "max_steps_per_run", "twenty"])
        assert result.exit_code != 0

    def test_help_lists_subcommands(self) -> None:
        runner = CliRunner()
        result = runner.invoke(config_app, ["--help"])
        assert result.exit_code == 0
        assert "show" in result.output
        assert "get" in result.output
        assert "set" in result.output