Skip to content

Automate browser based workflows with AI

70.1 KB
Raw Blame
"""Tests for CLI commands infrastructure: _state.py and _output.py."""

from __future__ import annotations

import asyncio
import importlib
import json
import os
import shlex
import subprocess
import sys
from collections.abc import Callable
from contextlib import suppress
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import httpx
import pytest
import typer
from playwright.async_api import Error as PlaywrightError
from typer.testing import CliRunner

from skyvern import _cli_bootstrap, config
from skyvern.cli.commands import _output as cli_output
from skyvern.cli.commands import browser as cli_browser
from skyvern.cli.commands import cli_app
from skyvern.cli.commands._state import CLIState, clear_state, load_state, save_state
from skyvern.cli.core import client as cli_client
from skyvern.cli.core.guards import STALE_FRAME_HINT
from skyvern.cli.status import _status_data
from skyvern.exceptions import StaleFrameSelectionError
from skyvern.library.skyvern import Skyvern
from tests.unit._mcp_browser_fakes import StaleScopePage

# ---------------------------------------------------------------------------
# _state.py
# ---------------------------------------------------------------------------


def _patch_state_dir(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    monkeypatch.setattr("skyvern.cli.commands._state.STATE_DIR", tmp_path)
    monkeypatch.setattr("skyvern.cli.commands._state.STATE_FILE", tmp_path / "state.json")


class TestCLIState:
    def test_save_load_roundtrip(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        _patch_state_dir(monkeypatch, tmp_path)
        save_state(CLIState(session_id="pbs_123", cdp_url=None, mode="cloud"))
        loaded = load_state()
        assert loaded is not None
        assert loaded.session_id == "pbs_123"
        assert loaded.cdp_url is None
        assert loaded.mode == "cloud"

    def test_save_load_roundtrip_cdp(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        _patch_state_dir(monkeypatch, tmp_path)
        save_state(CLIState(session_id=None, cdp_url="ws://localhost:9222/devtools/browser/abc", mode="cdp"))
        loaded = load_state()
        assert loaded is not None
        assert loaded.session_id is None
        assert loaded.cdp_url == "ws://localhost:9222/devtools/browser/abc"
        assert loaded.mode == "cdp"

    def test_load_returns_none_when_missing(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setattr("skyvern.cli.commands._state.STATE_FILE", tmp_path / "nonexistent.json")
        assert load_state() is None

    def test_24h_ttl_expires(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        _patch_state_dir(monkeypatch, tmp_path)
        state_file = tmp_path / "state.json"
        state_file.write_text(
            json.dumps(
                {
                    "session_id": "pbs_old",
                    "mode": "cloud",
                    "created_at": "2020-01-01T00:00:00+00:00",
                }
            )
        )
        assert load_state() is None

    def test_clear_state(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        _patch_state_dir(monkeypatch, tmp_path)
        save_state(CLIState(session_id="pbs_123"))
        clear_state()
        assert not (tmp_path / "state.json").exists()

    def test_load_ignores_corrupt_file(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        state_file = tmp_path / "state.json"
        monkeypatch.setattr("skyvern.cli.commands._state.STATE_FILE", state_file)
        state_file.write_text("not-json")
        assert load_state() is None


# ---------------------------------------------------------------------------
# _output.py
# ---------------------------------------------------------------------------


class TestOutput:
    def test_json_envelope(self, capsys: pytest.CaptureFixture) -> None:
        from skyvern.cli.commands._output import output

        output({"key": "value"}, action="test", json_mode=True)
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["action"] == "test"
        assert parsed["data"]["key"] == "value"

    def test_json_error(self, capsys: pytest.CaptureFixture) -> None:
        from skyvern.cli.commands._output import output_error

        with pytest.raises(SystemExit, match="1"):
            output_error("bad thing", hint="fix it", json_mode=True)
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["error"]["message"] == "bad thing"


# ---------------------------------------------------------------------------
# status.py
# ---------------------------------------------------------------------------


class TestStatusCommands:
    def test_status_start_commands_are_accepted_by_cli_parser(self) -> None:
        runner = CliRunner()

        for row in _status_data():
            command_parts = shlex.split(row["start_command"])
            assert command_parts[0] == "skyvern"

            result = runner.invoke(cli_app, [*command_parts[1:], "--help"])

            assert result.exit_code == 0, result.output


# ---------------------------------------------------------------------------
# Connection resolution
# ---------------------------------------------------------------------------


class TestResolveConnection:
    def test_explicit_session_wins(self) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        result = _resolve_connection("pbs_explicit", None)
        assert result.mode == "cloud"
        assert result.session_id == "pbs_explicit"
        assert result.cdp_url is None

    def test_explicit_cdp_wins(self) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        cdp_url = "wss://browser.test/devtools/browser/pbs_123"
        result = _resolve_connection(None, cdp_url)
        assert result.mode == "cdp"
        assert result.session_id is None
        assert result.cdp_url == cdp_url

    def test_bare_session_id_in_explicit_cdp_routes_to_cloud(self) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        result = _resolve_connection(None, "pbs_explicit")
        assert result.mode == "cloud"
        assert result.session_id == "pbs_explicit"
        assert result.cdp_url is None

    def test_rejects_both_connection_flags(self) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        with pytest.raises(typer.BadParameter, match="Pass only one of --session or --cdp"):
            _resolve_connection("pbs_explicit", "ws://localhost:9222/devtools/browser/abc")

    def test_state_fallback(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        _patch_state_dir(monkeypatch, tmp_path)
        save_state(CLIState(session_id="pbs_from_state", mode="cloud"))
        result = _resolve_connection(None, None)
        assert result.mode == "cloud"
        assert result.session_id == "pbs_from_state"

    @pytest.mark.parametrize("mode", ["cdp", None])
    def test_state_fallback_cdp(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str | None) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        _patch_state_dir(monkeypatch, tmp_path)
        cdp_url = "wss://browser.test/devtools/browser/pbs_123"
        save_state(CLIState(session_id=None, cdp_url=cdp_url, mode=mode))
        result = _resolve_connection(None, None)
        assert result.mode == "cdp"
        assert result.cdp_url == cdp_url

    @pytest.mark.parametrize("mode", ["cdp", None])
    def test_bare_session_id_in_cdp_state_routes_to_cloud(
        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mode: str | None
    ) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        _patch_state_dir(monkeypatch, tmp_path)
        save_state(CLIState(session_id=None, cdp_url="pbs_from_state", mode=mode))
        result = _resolve_connection(None, None)
        assert result.mode == "cloud"
        assert result.session_id == "pbs_from_state"
        assert result.cdp_url is None

    def test_no_session_raises(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.commands.browser import _resolve_connection

        monkeypatch.setattr("skyvern.cli.commands._state.STATE_FILE", tmp_path / "nonexistent.json")
        with pytest.raises(typer.BadParameter, match="No active browser connection"):
            _resolve_connection(None, None)


# ---------------------------------------------------------------------------
# Browser command helpers and command behavior
# ---------------------------------------------------------------------------


class TestBrowserCommandGuards:
    def test_resolve_ai_target_requires_selector_or_intent(self) -> None:
        from skyvern.cli.commands.browser import _resolve_ai_target
        from skyvern.cli.core.guards import GuardError

        with pytest.raises(GuardError, match="Must provide intent, selector, or both"):
            _resolve_ai_target(None, None, operation="click")

    def test_validate_wait_state_rejects_invalid(self) -> None:
        from skyvern.cli.commands.browser import _validate_wait_state
        from skyvern.cli.core.guards import GuardError

        with pytest.raises(GuardError, match="Invalid state"):
            _validate_wait_state("bad-state")

    @pytest.mark.parametrize("clear", [True, False], ids=["fill", "append"])
    def test_type_with_intent_refuses_top_document_password_match(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, clear: bool
    ) -> None:
        top_match = MagicMock()
        top_match.first = top_match
        top_match.evaluate = AsyncMock(return_value=True)
        top_match.type = AsyncMock()
        raw_page = MagicMock()
        raw_page.locator = MagicMock(return_value=top_match)
        frame_match = MagicMock()
        frame_match.first = frame_match
        frame_match.evaluate = AsyncMock(return_value=False)
        active_frame = MagicMock()
        active_frame.locator = MagicMock(return_value=frame_match)
        page = MagicMock()
        page.page = raw_page
        page.locator_scope = active_frame
        page.locator = MagicMock(return_value=top_match)
        browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
        monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
        monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
        monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
        monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())

        with pytest.raises(SystemExit):
            cli_browser.type_text(
                text="opaque-value",
                intent="the promo code box",
                selector=".shared-field",
                session=None,
                cdp=None,
                timeout=1000,
                clear=clear,
                delay=None,
                json_output=True,
            )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "password" in parsed["error"]["message"].lower()
        top_match.type.assert_not_awaited()
        page.fill.assert_not_called()


class TestBrowserCommands:
    def test_session_get_outputs_session_details(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        session_obj = SimpleNamespace(
            browser_session_id="pbs_123",
            status="active",
            started_at=datetime(2026, 2, 17, 12, 0, tzinfo=timezone.utc),
            completed_at=None,
            timeout=60,
            runnable_id=None,
            app_url="https://app.skyvern.com/browser-session/pbs_123",
            recordings=[],
            downloaded_files=[],
        )
        skyvern = SimpleNamespace(get_browser_session=AsyncMock(return_value=session_obj))
        monkeypatch.setattr(browser_cmd, "get_skyvern", lambda: skyvern)
        monkeypatch.setattr(browser_cmd, "load_state", lambda: CLIState(session_id="pbs_123", mode="cloud"))

        browser_cmd.session_get(session="pbs_123", json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["action"] == "session_get"
        assert parsed["data"]["session_id"] == "pbs_123"
        assert parsed["data"]["is_current"] is True

    def test_evaluate_blocks_password_js_before_connection(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: (_ for _ in ()).throw(AssertionError("should not resolve connection")),
        )

        with pytest.raises(SystemExit, match="1"):
            browser_cmd.evaluate(
                expression='document.querySelector("input[type=password]").value = ""', json_output=True
            )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "Cannot set password field values" in parsed["error"]["message"]

    def test_click_requires_target_before_connection(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        capture_mock = MagicMock()
        monkeypatch.setattr(browser_cmd, "capture_cli_tool_call", capture_mock)
        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: (_ for _ in ()).throw(AssertionError("should not resolve connection")),
        )

        with pytest.raises(SystemExit, match="1"):
            browser_cmd.click(
                intent=None,
                selector=None,
                session=None,
                cdp=None,
                timeout=30000,
                button=None,
                click_count=None,
                json_output=True,
            )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "Must provide intent, selector, or both" in parsed["error"]["message"]
        capture_mock.assert_not_called()

    def test_click_with_intent_uses_proactive_ai_mode(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        page = MagicMock()
        page.click = AsyncMock(return_value="xpath=//button[@id='submit']")
        browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))

        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: browser_cmd.ConnectionTarget(mode="cloud", session_id="pbs_123"),
        )
        monkeypatch.setattr(browser_cmd, "_connect_browser", AsyncMock(return_value=browser))
        capture_mock = MagicMock()
        monkeypatch.setattr(browser_cmd, "capture_cli_tool_call", capture_mock)

        browser_cmd.click(
            intent="the Submit button",
            selector=None,
            session="pbs_123",
            cdp=None,
            timeout=30000,
            button=None,
            click_count=None,
            json_output=True,
        )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["action"] == "click"
        assert parsed["data"]["ai_mode"] == "proactive"
        assert parsed["data"]["resolved_selector"] == "xpath=//button[@id='submit']"
        capture_mock.assert_called_once_with("skyvern_click", ok=True)

    def test_wait_rejects_invalid_state_before_connection(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: (_ for _ in ()).throw(AssertionError("should not resolve connection")),
        )

        with pytest.raises(SystemExit, match="1"):
            browser_cmd.wait(state="bad-state", time_ms=1000, json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "Invalid state" in parsed["error"]["message"]


# ---------------------------------------------------------------------------
# Workflow command behavior
# ---------------------------------------------------------------------------


class TestWorkflowCommands:
    def test_workflow_list_query_alias_maps_to_search(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli import workflow as workflow_cmd

        monkeypatch.setattr(workflow_cmd, "prepare_cli_runtime", lambda **_: None)
        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_list",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"workflows": [], "page": 1, "page_size": 10, "count": 0, "has_more": False},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_list", tool)

        result = CliRunner().invoke(workflow_cmd.workflow_app, ["list", "--query", "invoice", "--json"])

        assert result.exit_code == 0, result.output
        assert tool.await_args.kwargs == {
            "search": "invoice",
            "page": 1,
            "page_size": 10,
            "only_workflows": False,
        }
        parsed = json.loads(result.output)
        assert parsed["ok"] is True

    def test_workflow_get_outputs_mcp_envelope_in_json_mode(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        expected = {
            "ok": True,
            "action": "skyvern_workflow_get",
            "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
            "data": {"workflow_permanent_id": "wpid_123"},
            "artifacts": [],
            "timing_ms": {},
            "warnings": [],
            "error": None,
        }
        tool = AsyncMock(return_value=expected)
        monkeypatch.setattr(workflow_cmd, "tool_workflow_get", tool)

        workflow_cmd.workflow_get(workflow_id="wpid_123", version=2, definition_file=None, json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        # emit_tool_result adds schema_version and warnings defaults to the output copy
        assert parsed["ok"] is True
        assert parsed["data"] == expected["data"]
        assert parsed["schema_version"] == "1.0"
        assert tool.await_args.kwargs == {"workflow_id": "wpid_123", "version": 2}

    def test_workflow_get_definition_file_feeds_directly_into_update(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        import asyncio

        from skyvern.cli import workflow as workflow_cmd
        from skyvern.cli.mcp_tools import workflow as workflow_tools

        preserved_settings = {
            "description": "Existing description",
            "webhook_callback_url": "https://example.com/webhook",
            "persist_browser_session": True,
            "browser_profile_id": "bp_existing",
            "run_with": "code",
            "cache_key": None,
            "adaptive_caching": True,
            "ai_fallback": False,
        }
        existing_workflow = {
            "title": "Status workflow",
            "proxy_location": "RESIDENTIAL",
            "workflow_definition": {
                "parameters": [],
                "blocks": [{"block_type": "code", "label": "report_status", "code": "return {'status': 'ready'}"}],
            },
            **preserved_settings,
        }
        exported = {
            "title": existing_workflow["title"],
            "proxy_location": existing_workflow["proxy_location"],
            "workflow_definition": existing_workflow["workflow_definition"],
        }
        monkeypatch.setattr(workflow_cmd, "prepare_cli_runtime", lambda **_: None)
        monkeypatch.setattr(
            workflow_cmd,
            "tool_workflow_get",
            AsyncMock(return_value={"ok": True, "data": existing_workflow}),
        )
        definition_file = tmp_path / "wf.json"

        cli_result = CliRunner().invoke(
            workflow_cmd.workflow_app,
            ["get", "--id", "wpid_123", "--definition-file", str(definition_file)],
        )

        assert cli_result.exit_code == 0, cli_result.output
        assert "Wrote workflow definition to" in cli_result.output
        assert json.loads(definition_file.read_text()) == exported

        monkeypatch.setattr(workflow_tools, "get_workflow_by_id", AsyncMock(return_value=existing_workflow))
        update_raw = AsyncMock(
            return_value={"workflow_permanent_id": "wpid_123", "title": "Status workflow", "version": 2}
        )
        monkeypatch.setattr(workflow_tools, "update_workflow_raw", update_raw)

        update_result = asyncio.run(
            workflow_cmd.tool_workflow_update(
                workflow_id="wpid_123",
                definition=definition_file.read_text(),
                format="json",
            )
        )

        assert update_result["ok"] is True
        update_raw.assert_awaited_once()
        sent_definition = update_raw.await_args.kwargs["json_definition"]
        assert {field: sent_definition[field] for field in preserved_settings} == preserved_settings

    def test_workflow_create_reads_definition_from_file(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture,
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        definition_file = tmp_path / "workflow.json"
        definition_text = '{"title": "Example", "workflow_definition": {"blocks": []}}'
        definition_file.write_text(definition_text)

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_create",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"workflow_permanent_id": "wpid_new"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_create", tool)

        workflow_cmd.workflow_create(
            definition=f"@{definition_file}",
            definition_format="json",
            folder_id="fld_123",
            json_output=True,
        )

        assert tool.await_args.kwargs == {
            "definition": definition_text,
            "format": "json",
            "folder_id": "fld_123",
        }
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["workflow_permanent_id"] == "wpid_new"

    def test_workflow_create_preserves_empty_definition_file(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture,
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        definition_file = tmp_path / "workflow.json"
        definition_file.write_text("")

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_create",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"workflow_permanent_id": "wpid_empty"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_create", tool)

        workflow_cmd.workflow_create(
            definition=f"@{definition_file}",
            definition_format="json",
            folder_id=None,
            json_output=True,
        )

        assert tool.await_args.kwargs["definition"] == ""
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True

    def test_workflow_run_reads_params_file_and_maps_options(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture,
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        params_file = tmp_path / "params.json"
        params_file.write_text('{"company": "Acme"}')

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_run",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"run_id": "wr_123", "status": "queued"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_run", tool)

        workflow_cmd.workflow_run(
            workflow_id="wpid_123",
            params=f"@{params_file}",
            session="pbs_456",
            webhook="https://example.com/webhook",
            proxy="RESIDENTIAL",
            wait=True,
            timeout=450,
            run_with=None,
            json_output=True,
        )

        assert tool.await_args.kwargs == {
            "workflow_id": "wpid_123",
            "parameters": '{"company": "Acme"}',
            "browser_session_id": "pbs_456",
            "webhook_url": "https://example.com/webhook",
            "proxy_location": "RESIDENTIAL",
            "wait": True,
            "timeout_seconds": 450,
            "run_with": None,
        }
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["run_id"] == "wr_123"

    def test_workflow_status_json_error_exits_nonzero(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        tool = AsyncMock(
            return_value={
                "ok": False,
                "action": "skyvern_workflow_status",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": None,
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": {
                    "code": "RUN_NOT_FOUND",
                    "message": "Run 'wr_missing' not found",
                    "hint": "Verify the run ID",
                    "details": {},
                },
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_status", tool)

        with pytest.raises(SystemExit, match="1"):
            workflow_cmd.workflow_status(run_id="wr_missing", json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["error"]["code"] == "RUN_NOT_FOUND"

    def test_workflow_status_cli_preserves_full_detail_behavior(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_status",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"run_id": "wr_123", "recording_url": "https://example.com/recording"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_status", tool)

        workflow_cmd.workflow_status(run_id="wr_123", json_output=True)

        assert tool.await_args.kwargs == {"run_id": "wr_123", "verbosity": "full"}
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["recording_url"] == "https://example.com/recording"

    def test_workflow_update_missing_definition_file_raises_bad_parameter(
        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        tool = AsyncMock()
        monkeypatch.setattr(workflow_cmd, "tool_workflow_update", tool)
        missing_file = tmp_path / "missing-definition.json"

        with pytest.raises(typer.BadParameter, match="Unable to read definition file"):
            workflow_cmd.workflow_update(
                workflow_id="wpid_123",
                definition=f"@{missing_file}",
                definition_format="json",
                json_output=False,
            )

        tool.assert_not_called()

    def test_workflow_update_preserves_empty_definition_file(
        self,
        tmp_path: Path,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture,
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        definition_file = tmp_path / "workflow.json"
        definition_file.write_text("")

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_workflow_update",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"workflow_permanent_id": "wpid_123"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(workflow_cmd, "tool_workflow_update", tool)

        workflow_cmd.workflow_update(
            workflow_id="wpid_123",
            definition=f"@{definition_file}",
            definition_format="json",
            json_output=True,
        )

        assert tool.await_args.kwargs["definition"] == ""
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True


# ---------------------------------------------------------------------------
# PR C parity command behavior
# ---------------------------------------------------------------------------


class TestCredentialParityCommands:
    def test_credential_list_maps_options(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture) -> None:
        from skyvern.cli import credential as credential_cmd

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_credential_list",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"credentials": [], "page": 2, "page_size": 25, "count": 0, "has_more": False},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(credential_cmd, "tool_credential_list", tool)

        credential_cmd.credential_list(page=2, page_size=25, json_output=True)

        assert tool.await_args.kwargs == {"page": 2, "page_size": 25}
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["action"] == "skyvern_credential_list"

    def test_credential_get_json_error_exits_nonzero(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import credential as credential_cmd

        tool = AsyncMock(
            return_value={
                "ok": False,
                "action": "skyvern_credential_get",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": None,
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": {
                    "code": "INVALID_INPUT",
                    "message": "Invalid credential_id format: 'bad'",
                    "hint": "Credential IDs start with cred_",
                    "details": {},
                },
            }
        )
        monkeypatch.setattr(credential_cmd, "tool_credential_get", tool)

        with pytest.raises(SystemExit, match="1"):
            credential_cmd.credential_get(credential_id="bad", json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "Invalid credential_id format" in parsed["error"]["message"]

    def test_credential_delete_maps_options(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import credential as credential_cmd

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_credential_delete",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"credential_id": "cred_123", "deleted": True},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(credential_cmd, "tool_credential_delete", tool)

        credential_cmd.credential_delete(credential_id="cred_123", json_output=True)

        assert tool.await_args.kwargs == {"credential_id": "cred_123"}
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["deleted"] is True


class TestBlockParityCommands:
    def test_block_schema_passes_block_type(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import block as block_cmd

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_block_schema",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"block_type": "navigation", "schema": {"type": "object"}},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(block_cmd, "tool_block_schema", tool)

        block_cmd.block_schema(block_type="navigation", json_output=True)

        assert tool.await_args.kwargs == {"block_type": "navigation"}
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["block_type"] == "navigation"

    def test_block_validate_reads_json_from_file(
        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import block as block_cmd

        block_file = tmp_path / "block.json"
        block_file.write_text('{"block_type":"navigation","label":"step1","navigation_goal":"Go to page"}')

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_block_validate",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"valid": True, "block_type": "navigation", "label": "step1"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(block_cmd, "tool_block_validate", tool)

        block_cmd.block_validate(block_json=f"@{block_file}", json_output=True)

        assert tool.await_args.kwargs == {
            "block_json": '{"block_type":"navigation","label":"step1","navigation_goal":"Go to page"}'
        }
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["valid"] is True

    def test_block_validate_preserves_empty_file(
        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import block as block_cmd

        block_file = tmp_path / "block.json"
        block_file.write_text("")

        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_block_validate",
                "browser_context": {"mode": "none", "session_id": None, "cdp_url": None},
                "data": {"valid": False},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(block_cmd, "tool_block_validate", tool)

        block_cmd.block_validate(block_json=f"@{block_file}", json_output=True)

        assert tool.await_args.kwargs["block_json"] == ""
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True


class TestTasksCommands:
    def test_tasks_list_json_error_includes_action(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import tasks as tasks_cmd

        monkeypatch.setattr(tasks_cmd, "_get_client", lambda _api_key=None, **_: object())
        monkeypatch.setattr(
            tasks_cmd, "_list_workflow_tasks", lambda _client, _run_id: (_ for _ in ()).throw(RuntimeError("boom"))
        )

        with pytest.raises(SystemExit, match="1"):
            tasks_cmd.list_tasks(
                ctx=SimpleNamespace(obj={}),
                workflow_run_id="wr_123",
                json_output=True,
            )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["action"] == "tasks.list"


@pytest.mark.parametrize("module_name", ["skyvern.cli.credentials", "skyvern.cli.tasks"])
class TestCredentialClientBaseUrlGuard:
    def test_rejects_credential_when_base_url_uses_untouched_default(
        self, module_name: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern import config

        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        cli_module = importlib.import_module(module_name)
        monkeypatch.setattr(cli_module, "prepare_cli_runtime", lambda **_: None)
        monkeypatch.setattr(config, "settings", config.Settings(_env_file=None))

        with pytest.raises(SystemExit, match="1"):
            cli_module._get_client("test-api-key")

        output = " ".join(capsys.readouterr().out.split())
        assert "Set SKYVERN_BASE_URL" in output
        # The remediation must not hand the user the production URL this guard exists to protect.
        assert "https://api.skyvern.com" not in output

    def test_rejection_uses_json_envelope_when_requested(
        self, module_name: str, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern import config

        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        cli_module = importlib.import_module(module_name)
        monkeypatch.setattr(cli_module, "prepare_cli_runtime", lambda **_: None)
        monkeypatch.setattr(config, "settings", config.Settings(_env_file=None))

        with pytest.raises(SystemExit, match="1"):
            cli_module._get_client("test-api-key", action="credentials.list", json_mode=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["action"] == "credentials.list"
        assert "Set SKYVERN_BASE_URL" in parsed["error"]["message"]
        assert "https://api.skyvern.com" not in parsed["error"]["message"]

    @pytest.mark.parametrize("base_url", ["https://api.skyvern.com", "https://staging.skyvern.com"])
    def test_allows_credential_when_base_url_is_explicitly_configured(
        self, module_name: str, base_url: str, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        from skyvern import config

        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.setenv("SKYVERN_BASE_URL", base_url)
        cli_module = importlib.import_module(module_name)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(cli_module, "prepare_cli_runtime", lambda **_: None)
        monkeypatch.setattr(cli_module, "Skyvern", client_constructor)
        monkeypatch.setattr(config, "settings", config.Settings(_env_file=None))

        cli_module._get_client("test-api-key")

        client_constructor.assert_called_once_with(base_url=base_url, api_key="test-api-key")

    def test_allows_untouched_default_without_a_credential(
        self, module_name: str, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        from skyvern import config

        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        cli_module = importlib.import_module(module_name)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(cli_module, "prepare_cli_runtime", lambda **_: None)
        monkeypatch.setattr(cli_module, "Skyvern", client_constructor)
        monkeypatch.setattr(config, "settings", config.Settings(_env_file=None))

        cli_module._get_client()

        client_constructor.assert_called_once_with(base_url="https://api.skyvern.com", api_key="PLACEHOLDER")


class TestSharedFactoryBaseUrlGuard:
    def _client_module_with_fresh_settings(self, monkeypatch: pytest.MonkeyPatch) -> object:
        from skyvern import config
        from skyvern.cli.core import client as client_module

        # Stateless HTTP mode is process-global; pin it so tests hold under xdist
        # regardless of what an earlier test on the same worker left behind.
        monkeypatch.setattr("skyvern.cli.core.session_manager.is_stateless_http_mode", lambda: False)
        monkeypatch.setattr(client_module, "settings", config.Settings(_env_file=None))
        return client_module

    def test_cli_process_rejects_credential_on_untouched_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        client_module = self._client_module_with_fresh_settings(monkeypatch)
        monkeypatch.setattr(client_module, "is_cli_runtime", lambda: True)

        with pytest.raises(RuntimeError, match="Set SKYVERN_BASE_URL"):
            client_module._build_cloud_client("test-api-key")

    def test_non_cli_process_builds_client_on_untouched_default(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """Prod temporal workers run with SKYVERN_BASE_URL unset and reach this factory via
        copilot self-heal; an unconditional guard here is a production outage."""
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        client_module = self._client_module_with_fresh_settings(monkeypatch)
        monkeypatch.setattr(client_module, "is_cli_runtime", lambda: False)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(client_module, "Skyvern", client_constructor)

        client_module._build_cloud_client("test-api-key")

        assert client_constructor.call_args.kwargs["base_url"] == "https://api.skyvern.com"

    @pytest.mark.parametrize("base_url", ["https://api.skyvern.com", "http://localhost:8000"])
    def test_cli_process_builds_client_when_base_url_is_explicit(
        self, base_url: str, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        monkeypatch.setenv("SKYVERN_BASE_URL", base_url)
        client_module = self._client_module_with_fresh_settings(monkeypatch)
        monkeypatch.setattr(client_module, "is_cli_runtime", lambda: True)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(client_module, "Skyvern", client_constructor)

        client_module._build_cloud_client("test-api-key")

        assert client_constructor.call_args.kwargs["base_url"] == base_url

    def test_cli_process_builds_client_with_placeholder_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        client_module = self._client_module_with_fresh_settings(monkeypatch)
        monkeypatch.setattr(client_module, "is_cli_runtime", lambda: True)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(client_module, "Skyvern", client_constructor)

        client_module._build_cloud_client("PLACEHOLDER")

        assert client_constructor.call_args.kwargs["base_url"] == "https://api.skyvern.com"

    def test_stateless_http_mode_targets_self_and_skips_guard(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        client_module = self._client_module_with_fresh_settings(monkeypatch)
        monkeypatch.setattr(client_module, "is_cli_runtime", lambda: True)
        monkeypatch.setattr("skyvern.cli.core.session_manager.is_stateless_http_mode", lambda: True)
        client_constructor = MagicMock(return_value=object())
        monkeypatch.setattr(client_module, "_LoopbackSkyvern", client_constructor)

        client_module._build_cloud_client("test-api-key")

        assert client_constructor.call_args.kwargs["base_url"] == f"http://127.0.0.1:{client_module.settings.PORT}"


class TestBrowserGroupBaseUrlGuard:
    def test_browser_session_create_refuses_credential_on_untouched_default(
        self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
    ) -> None:
        """The browser group callback must mark the CLI runtime so the shared factory guard
        fires for nested subcommands; without it a real key silently reaches production."""
        from skyvern import _cli_bootstrap, config
        from skyvern.cli.commands import browser as browser_cmd
        from skyvern.cli.core import client as client_module

        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        monkeypatch.setattr(_cli_bootstrap, "_CLI_RUNTIME_PREPARED", False)
        monkeypatch.setattr("skyvern.utils.env_paths.load_backend_env_files", lambda intent: tmp_path / ".env")
        monkeypatch.setattr(_cli_bootstrap, "configure_cli_runtime_logging", lambda: None)
        monkeypatch.setattr("skyvern.cli.core.session_manager.is_stateless_http_mode", lambda: False)
        monkeypatch.setattr(client_module, "settings", config.Settings(_env_file=None))
        monkeypatch.setattr(client_module, "_resolve_api_key", lambda: "test-api-key")
        monkeypatch.setattr(client_module, "_global_skyvern_instance", None)
        monkeypatch.setattr(browser_cmd, "capture_cli_tool_call", lambda *_args, **_kwargs: None)

        token = client_module._skyvern_instance.set(None)
        try:
            result = CliRunner().invoke(browser_cmd.browser_app, ["session", "create", "--json"])
        finally:
            client_module._skyvern_instance.reset(token)

        assert result.exit_code == 1, result.output
        parsed = json.loads(result.output)
        assert parsed["ok"] is False
        assert "Set SKYVERN_BASE_URL" in parsed["error"]["message"]


class TestCliUserAgent:
    @pytest.mark.parametrize(
        "args",
        [
            pytest.param(["workflow", "list", "--json"], id="mcp_parity_command"),
            pytest.param(["browser", "session", "create", "--json"], id="browser_command"),
        ],
    )
    def test_cli_command_labels_its_api_calls_as_the_cli(
        self, args: list[str], monkeypatch: pytest.MonkeyPatch, tmp_path: Path
    ) -> None:
        """The CLI drives the MCP's own client and tools; only the header lets the server tell the two apart."""
        requests: list[httpx.Request] = []

        def handler(request: httpx.Request) -> httpx.Response:
            requests.append(request)
            return httpx.Response(200, json=[])

        real_build_cloud_client = cli_client._build_cloud_client

        def build_cloud_client_offline(api_key: str) -> Skyvern:
            instance = real_build_cloud_client(api_key)
            instance._client_wrapper.httpx_client.httpx_client._transport = httpx.MockTransport(handler)
            return instance

        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.setattr(_cli_bootstrap, "_CLI_RUNTIME_PREPARED", False)
        monkeypatch.setattr(_cli_bootstrap, "_CLI_COMMAND_RUNNING", False)
        monkeypatch.setattr("skyvern.utils.env_paths.load_backend_env_files", lambda intent: tmp_path / ".env")
        monkeypatch.setattr(_cli_bootstrap, "configure_cli_runtime_logging", lambda: None)
        monkeypatch.setattr("skyvern.cli.core.session_manager.is_stateless_http_mode", lambda: False)
        monkeypatch.setattr(
            cli_client, "settings", config.Settings(_env_file=None, SKYVERN_BASE_URL="http://skyvern.test")
        )
        monkeypatch.setattr(cli_client, "_resolve_api_key", lambda: "test-api-key")
        monkeypatch.setattr(cli_client, "_global_skyvern_instance", None)
        monkeypatch.setattr(cli_client, "_build_cloud_client", build_cloud_client_offline)
        monkeypatch.setattr(cli_output, "capture_cli_tool_call", lambda *_args, **_kwargs: None)
        monkeypatch.setattr(cli_browser, "capture_cli_tool_call", lambda *_args, **_kwargs: None)

        token = cli_client._skyvern_instance.set(None)
        try:
            CliRunner().invoke(cli_app, args)
        finally:
            cli_client._skyvern_instance.reset(token)

        assert requests
        assert {request.headers.get("x-user-agent") for request in requests} == {"skyvern-cli"}


class TestCliRuntimeFlag:
    def test_prepare_cli_runtime_marks_cli_process(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
        from skyvern import _cli_bootstrap

        monkeypatch.setattr(_cli_bootstrap, "_CLI_RUNTIME_PREPARED", False)
        monkeypatch.setattr("skyvern.utils.env_paths.load_backend_env_files", lambda intent: tmp_path / ".env")
        monkeypatch.setattr(_cli_bootstrap, "configure_cli_runtime_logging", lambda: None)

        assert not _cli_bootstrap.is_cli_runtime()
        _cli_bootstrap.prepare_cli_runtime(intent="cloud")
        assert _cli_bootstrap.is_cli_runtime()

    def test_cli_import_defers_settings_construction(self) -> None:
        """Both base-url guards read a settings singleton that must be constructed only after
        prepare_cli_runtime has loaded env files; an eager skyvern.config import breaks that."""
        code = "import sys; import skyvern.cli; raise SystemExit(1 if 'skyvern.config' in sys.modules else 0)"
        completed = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=120)
        assert completed.returncode == 0, completed.stderr


class TestCliRuntimeResetFixture:
    def test_reset_cli_runtime_entry_restores_process_and_env_state(self, monkeypatch: pytest.MonkeyPatch) -> None:
        """The autouse conftest fixture must undo everything a CLI entry leaves in the process.

        prepare_cli_runtime flips _CLI_RUNTIME_PREPARED, and load_backend_env_files writes
        SKYVERN_ENV_INTENT unconditionally plus SKYVERN_API_KEY / SKYVERN_BASE_URL when a .env
        supplies them. config._settings_env_intent reads SKYVERN_ENV_INTENT for env precedence,
        so leaking it past teardown steers a later cloud-client test in the same shard.
        """
        from skyvern import _cli_bootstrap
        from tests.unit.conftest import reset_cli_runtime_entry

        run_reset = reset_cli_runtime_entry.__wrapped__

        monkeypatch.setattr(_cli_bootstrap, "_CLI_RUNTIME_PREPARED", False)
        monkeypatch.setattr(_cli_bootstrap, "_CLI_COMMAND_RUNNING", False)
        monkeypatch.delenv("SKYVERN_API_KEY", raising=False)
        monkeypatch.delenv("SKYVERN_BASE_URL", raising=False)
        monkeypatch.setenv("SKYVERN_ENV_INTENT", "auto")

        teardown = run_reset()
        next(teardown)

        _cli_bootstrap._CLI_RUNTIME_PREPARED = True
        _cli_bootstrap.mark_cli_command()
        os.environ["SKYVERN_API_KEY"] = "leaked-key"
        os.environ["SKYVERN_BASE_URL"] = "https://leaked.example"
        os.environ["SKYVERN_ENV_INTENT"] = "cloud"

        with pytest.raises(StopIteration):
            next(teardown)

        assert _cli_bootstrap._CLI_RUNTIME_PREPARED is False
        assert _cli_bootstrap.is_cli_command() is False
        assert "SKYVERN_API_KEY" not in os.environ
        assert "SKYVERN_BASE_URL" not in os.environ
        assert os.environ["SKYVERN_ENV_INTENT"] == "auto"


class TestCredentialsCommands:
    def test_add_credential_invalid_type_json_error_includes_action(self, capsys: pytest.CaptureFixture) -> None:
        from skyvern.cli import credentials as credentials_cmd

        ctx = SimpleNamespace(obj={})

        with pytest.raises(SystemExit, match="1"):
            credentials_cmd.add_credential(
                ctx=ctx,
                name="Bad",
                credential_type="not-real",
                username=None,
                password=None,
                totp=None,
                card_number=None,
                cvv=None,
                exp_month=None,
                exp_year=None,
                card_brand=None,
                holder_name=None,
                secret_value=None,
                secret_label=None,
                json_output=True,
            )

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["action"] == "credentials.add"


class TestBrowserPRCCommands:
    def test_run_task_uses_resolved_connection(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import browser as browser_cmd

        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: browser_cmd.ConnectionTarget(mode="cloud", session_id="pbs_123"),
        )
        tool = AsyncMock(
            return_value={
                "ok": True,
                "action": "skyvern_run_task",
                "browser_context": {"mode": "cloud", "session_id": "pbs_123", "cdp_url": None},
                "data": {"run_id": "run_123", "status": "completed"},
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": None,
            }
        )
        monkeypatch.setattr(browser_cmd, "tool_run_task", tool)

        browser_cmd.run_task(
            prompt="Find the latest headline",
            session=None,
            cdp=None,
            url="https://news.ycombinator.com",
            data_extraction_schema='{"type":"object"}',
            max_steps=5,
            timeout_seconds=240,
            json_output=True,
        )

        assert tool.await_args.kwargs == {
            "prompt": "Find the latest headline",
            "session_id": "pbs_123",
            "cdp_url": None,
            "url": "https://news.ycombinator.com",
            "data_extraction_schema": '{"type":"object"}',
            "max_steps": 5,
            "timeout_seconds": 240,
        }
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is True
        assert parsed["data"]["run_id"] == "run_123"

    def test_login_json_error_exits_nonzero(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli.commands import _output as output_mod
        from skyvern.cli.commands import browser as browser_cmd

        monkeypatch.setattr(
            browser_cmd,
            "_resolve_connection",
            lambda _session, _cdp: browser_cmd.ConnectionTarget(mode="cloud", session_id="pbs_abc"),
        )
        tool = AsyncMock(
            return_value={
                "ok": False,
                "action": "skyvern_login",
                "browser_context": {"mode": "cloud", "session_id": "pbs_abc", "cdp_url": None},
                "data": None,
                "artifacts": [],
                "timing_ms": {},
                "warnings": [],
                "error": {
                    "code": "INVALID_INPUT",
                    "message": "Missing required fields for credential_type='skyvern': credential_id",
                    "hint": "Provide: credential_id",
                    "details": {},
                },
            }
        )
        monkeypatch.setattr(browser_cmd, "tool_login", tool)
        capture_mock = MagicMock()
        monkeypatch.setattr(output_mod, "capture_cli_tool_call", capture_mock)

        with pytest.raises(SystemExit, match="1"):
            browser_cmd.login(
                credential_type="skyvern",
                session=None,
                cdp=None,
                credential_id=None,
                json_output=True,
            )

        kwargs = tool.await_args.kwargs
        assert kwargs["session_id"] == "pbs_abc"
        assert kwargs["credential_type"] == "skyvern"
        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert "Missing required fields" in parsed["error"]["message"]
        capture_mock.assert_called_once_with("skyvern_login", ok=False)

    def test_workflow_create_missing_definition_file_does_not_emit_cli_telemetry(
        self,
        monkeypatch: pytest.MonkeyPatch,
    ) -> None:
        from skyvern.cli import workflow as workflow_cmd

        capture_mock = MagicMock()
        monkeypatch.setattr("skyvern.cli.commands._output.capture_cli_tool_call", capture_mock)

        with pytest.raises(typer.BadParameter):
            workflow_cmd.workflow_create(
                definition="@/does/not/exist.json",
                definition_format="json",
                folder_id=None,
                json_output=False,
            )

        capture_mock.assert_not_called()


class TestParityErrorFormatting:
    def test_credential_emit_tool_result_handles_none_message_and_hint(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.commands import _output as output_mod

        captured: dict[str, str | bool] = {}

        def _fake_output_error(
            message: str,
            *,
            hint: str = "",
            action: str = "",
            json_mode: bool = False,
            exit_code: int = 1,
        ) -> None:
            captured["message"] = message
            captured["hint"] = hint
            captured["json_mode"] = json_mode
            raise SystemExit(exit_code)

        monkeypatch.setattr(output_mod, "output_error", _fake_output_error)

        with pytest.raises(SystemExit, match="1"):
            output_mod.emit_tool_result(
                {"ok": False, "error": {"message": None, "hint": None}},
                json_output=False,
            )

        assert captured == {"message": "Unknown error", "hint": "", "json_mode": False}

    def test_block_emit_tool_result_handles_none_message_and_hint(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.commands import _output as output_mod

        captured: dict[str, str | bool] = {}

        def _fake_output_error(
            message: str,
            *,
            hint: str = "",
            action: str = "",
            json_mode: bool = False,
            exit_code: int = 1,
        ) -> None:
            captured["message"] = message
            captured["hint"] = hint
            captured["json_mode"] = json_mode
            raise SystemExit(exit_code)

        monkeypatch.setattr(output_mod, "output_error", _fake_output_error)

        with pytest.raises(SystemExit, match="1"):
            output_mod.emit_tool_result(
                {"ok": False, "error": {"message": None, "hint": None}},
                json_output=False,
            )

        assert captured == {"message": "Unknown error", "hint": "", "json_mode": False}

    def test_browser_emit_tool_result_handles_none_message_and_hint(self, monkeypatch: pytest.MonkeyPatch) -> None:
        from skyvern.cli.commands import _output as output_mod
        from skyvern.cli.commands import browser as browser_cmd

        captured: dict[str, str | bool] = {}

        def _fake_output_error(
            message: str,
            *,
            hint: str = "",
            action: str = "",
            json_mode: bool = False,
            exit_code: int = 1,
        ) -> None:
            captured["message"] = message
            captured["hint"] = hint
            captured["json_mode"] = json_mode
            raise SystemExit(exit_code)

        monkeypatch.setattr(output_mod, "output_error", _fake_output_error)

        with pytest.raises(SystemExit, match="1"):
            browser_cmd._emit_tool_result(
                {"ok": False, "error": {"message": None, "hint": None}},
                json_output=False,
                action="login",
            )

        assert captured == {"message": "Unknown error", "hint": "", "json_mode": False}


class TestStopCommands:
    def test_stop_all_json_reports_failure_when_nothing_running(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        from skyvern.cli import stop_commands as stop_cmd

        monkeypatch.setattr(stop_cmd, "get_pids_on_port", lambda _port: [])
        monkeypatch.setattr(stop_cmd, "kill_pids", lambda pids, service_name, quiet=False: False)

        with pytest.raises(SystemExit, match="1"):
            stop_cmd.stop_all(json_output=True)

        parsed = json.loads(capsys.readouterr().out)
        assert parsed["ok"] is False
        assert parsed["action"] == "stop.all"
        assert parsed["error"]["message"] == "No Skyvern services found running on ports 8000, 8080, or 9090."


def test_cli_evaluate_bounds_a_never_resolving_expression(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture,
) -> None:
    released = asyncio.Event()
    awaits = 0

    async def hanging_evaluate(_expression: str) -> str:
        nonlocal awaits
        awaits += 1
        with suppress(asyncio.TimeoutError):
            await asyncio.wait_for(released.wait(), timeout=2)
        return "escaped the bound"

    raw = MagicMock()
    page = SimpleNamespace(page=raw, locator_scope=raw, evaluate=hanging_evaluate)
    browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
    monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
    monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
    monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
    monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())
    monkeypatch.setattr(cli_browser, "DEFAULT_ACTION_TIMEOUT_MS", 150)

    with pytest.raises(SystemExit):
        cli_browser.evaluate(expression="await new Promise(() => {})", session=None, cdp=None, json_output=True)

    error = json.loads(capsys.readouterr().out)["error"]
    assert "timed out" in error["message"]
    assert awaits == 1


def test_cli_evaluate_reports_a_deadline_transport_error_as_a_timeout(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture,
) -> None:
    monkeypatch.setattr(cli_browser, "DEFAULT_ACTION_TIMEOUT_MS", 50)

    async def torn_down(expression: str, arg: object | None = None) -> object:
        try:
            await asyncio.sleep(3600)
        except asyncio.CancelledError:
            raise PlaywrightError("Target page, context or browser has been closed") from None

    raw = MagicMock()
    raw.evaluate = torn_down
    page = SimpleNamespace(page=raw, locator_scope=raw, evaluate=torn_down)
    browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
    monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
    monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
    monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
    monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())

    with pytest.raises(SystemExit):
        cli_browser.evaluate(expression="fetch('/slow')", session=None, cdp=None, json_output=True)

    error = json.loads(capsys.readouterr().out)["error"]
    assert "never settled" in error["hint"], error
    assert "Check JavaScript syntax" not in error["hint"], error


def test_cli_evaluate_keeps_caller_js_out_of_the_deadline_helper(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    recorded: dict[str, object] = {}

    async def record(**kwargs: object) -> object:
        recorded.update(kwargs)
        return "ok"

    raw = MagicMock()
    raw.evaluate = AsyncMock(return_value="ok")
    page = SimpleNamespace(page=raw, locator_scope=raw, evaluate=raw.evaluate)
    browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
    monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
    monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
    monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
    monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())
    monkeypatch.setattr(cli_browser.SkyvernFrame, "_evaluate_expression", record)

    cli_browser.evaluate(expression="fetch('/orders?token=tok-do-not-log')", session=None, cdp=None, json_output=True)

    assert "tok-do-not-log" not in str(recorded["expression"])


def test_cli_evaluate_dispatches_caller_js_once_when_the_context_dies(
    monkeypatch: pytest.MonkeyPatch,
    capsys: pytest.CaptureFixture,
) -> None:
    expressions: list[str] = []

    async def failing_evaluate(expression: str, arg: object | None = None) -> object:
        expressions.append(expression)
        if expression == "postOrder()":
            raise PlaywrightError("Execution context was destroyed, most likely because of a navigation")
        return None

    raw = MagicMock()
    raw.evaluate = failing_evaluate
    raw.wait_for_load_state = AsyncMock()
    page = SimpleNamespace(page=raw, locator_scope=raw, evaluate=failing_evaluate)
    browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
    monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
    monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
    monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
    monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())

    with pytest.raises(SystemExit):
        cli_browser.evaluate(expression="postOrder()", session=None, cdp=None, json_output=True)

    error = json.loads(capsys.readouterr().out)["error"]
    assert "Execution context was destroyed" in error["message"]
    assert expressions == ["postOrder()"]
    raw.wait_for_load_state.assert_not_awaited()


class TestCLIStaleFrameSurface:
    @staticmethod
    def _patch_page(monkeypatch: pytest.MonkeyPatch, page: object) -> None:
        browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
        monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
        monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
        monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
        monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())

    @pytest.mark.parametrize(
        "invoke",
        [
            lambda: cli_browser.evaluate(expression="1 + 1", session=None, cdp=None, json_output=True),
            lambda: cli_browser.scroll(
                direction="down", session=None, cdp=None, amount=None, intent=None, selector=None, json_output=True
            ),
            lambda: cli_browser.press_key(
                key="Enter", session=None, cdp=None, intent=None, selector=None, json_output=True
            ),
            lambda: cli_browser.select(
                value="Ground",
                intent=None,
                selector="#ship",
                session=None,
                cdp=None,
                timeout=30000,
                by_label=True,
                json_output=True,
            ),
        ],
        ids=["evaluate", "scroll", "press_key", "select_by_label"],
    )
    def test_page_space_commands_refuse_an_unowned_frame_selection(
        self,
        monkeypatch: pytest.MonkeyPatch,
        capsys: pytest.CaptureFixture,
        invoke: Callable[[], None],
    ) -> None:
        raw = MagicMock()
        raw.evaluate = AsyncMock(return_value=2)
        raw.keyboard = SimpleNamespace(press=AsyncMock())
        self._patch_page(monkeypatch, StaleScopePage(raw))

        with pytest.raises(SystemExit):
            invoke()

        error = json.loads(capsys.readouterr().out)["error"]
        assert "Stale frame selection" in error["message"]
        assert error["hint"] == STALE_FRAME_HINT
        raw.locator.assert_not_called()

    def test_select_by_label_resolves_in_page_space_not_the_selected_frame(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        raw = MagicMock()
        raw.locator = MagicMock(return_value=SimpleNamespace(select_option=AsyncMock()))
        frame = MagicMock()
        self._patch_page(monkeypatch, SimpleNamespace(page=raw, locator_scope=frame))

        cli_browser.select(
            value="Ground",
            intent=None,
            selector="#ship",
            session=None,
            cdp=None,
            timeout=30000,
            by_label=True,
            json_output=True,
        )

        raw.locator.assert_called_once_with("#ship")
        frame.locator.assert_not_called()

    def test_wait_intent_envelope_keeps_the_stale_selection_hint(
        self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture
    ) -> None:
        page = SimpleNamespace(
            wait_for_timeout=AsyncMock(),
            validate=AsyncMock(side_effect=StaleFrameSelectionError("pay", "https://pay.example.com/")),
        )
        browser = SimpleNamespace(get_working_page=AsyncMock(return_value=page))
        monkeypatch.setattr(cli_browser, "_resolve_connection", MagicMock())
        monkeypatch.setattr(cli_browser, "_connect_browser", AsyncMock(return_value=browser))
        monkeypatch.setattr(cli_browser, "_apply_cli_frame_state", AsyncMock())
        monkeypatch.setattr(cli_browser, "capture_cli_tool_call", MagicMock())

        with pytest.raises(SystemExit):
            cli_browser.wait(
                session=None,
                cdp=None,
                time_ms=None,
                intent="the card form is ready",
                selector=None,
                state="visible",
                timeout=0,
                poll_interval=0,
                json_output=True,
            )

        error = json.loads(capsys.readouterr().out)["error"]
        assert "Stale frame selection" in error["message"]
        assert error["hint"] == STALE_FRAME_HINT