Skip to content

Automate browser based workflows with AI

14.5 KB
Raw Blame
"""Tests for WorkflowService.should_run_script() and script reviewer gating.

Verifies the priority chain:
  run-level run_with > workflow-level run_with > code_version fallback > default (agent).

run_with is always "agent" or "code" (never null). When run_with="agent" and
code_version >= 1, the code_version fallback still applies (workflow intended code mode).

Also covers the app-level code-mode gate: when intended mode is "code",
WorkflowService asks app.AGENT_FUNCTION whether the run should keep code mode.
"""

from datetime import datetime, timezone

import pytest

from skyvern.forge import app
from skyvern.forge.agent_functions import AgentFunction
from skyvern.forge.sdk.workflow.models.workflow import (
    Workflow,
    WorkflowRun,
    WorkflowRunStatus,
    is_adaptive_caching,
)


def _make_workflow(
    run_with: str = "agent",
    adaptive_caching: bool = False,
    code_version: int | None = None,
) -> Workflow:
    return Workflow(
        workflow_id="wf_test",
        organization_id="org_test",
        workflow_permanent_id="wpid_test",
        title="test",
        version=1,
        is_saved_task=False,
        workflow_definition={"parameters": [], "blocks": []},
        run_with=run_with,
        adaptive_caching=adaptive_caching,
        code_version=code_version,
        created_at=datetime.now(timezone.utc),
        modified_at=datetime.now(timezone.utc),
    )


def _make_run(
    run_with: str | None = "agent",
    workflow_run_id: str = "wr_test",
    retried_from_workflow_run_id: str | None = None,
) -> WorkflowRun:
    return WorkflowRun(
        workflow_run_id=workflow_run_id,
        workflow_id="wf_test",
        workflow_permanent_id="wpid_test",
        organization_id="org_test",
        status=WorkflowRunStatus.running,
        run_with=run_with,
        retried_from_workflow_run_id=retried_from_workflow_run_id,
        created_at=datetime.now(timezone.utc),
        modified_at=datetime.now(timezone.utc),
    )


class _RecordingAgentFunction(AgentFunction):
    def __init__(self, enabled: bool) -> None:
        self.enabled = enabled
        self.calls: list[tuple[str, str, str]] = []

    async def should_keep_code_mode_for_workflow_run(
        self,
        *,
        workflow: Workflow,
        workflow_run: WorkflowRun,
    ) -> bool:
        self.calls.append(
            (
                workflow_run.workflow_run_id,
                workflow_run.organization_id,
                workflow.workflow_permanent_id,
            )
        )
        return self.enabled


@pytest.fixture
def service(monkeypatch):
    monkeypatch.setattr(app, "AGENT_FUNCTION", AgentFunction())

    from skyvern.forge.sdk.workflow.service import WorkflowService

    return WorkflowService()


class TestShouldRunScript:
    """Run-level run_with takes priority, then workflow-level, then code_version fallback, then agent."""

    @pytest.mark.asyncio
    async def test_run_code_overrides_workflow_agent(self, service):
        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="code")
        assert await service.should_run_script(wf, wr) is True

    @pytest.mark.asyncio
    async def test_run_agent_overrides_workflow_code(self, service):
        wf = _make_workflow(run_with="code")
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_run_agent_overrides_code_version(self, service):
        """Explicit run-level agent overrides code_version fallback."""
        wf = _make_workflow(run_with="agent", code_version=2)
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_workflow_code_with_agent_run(self, service):
        """Workflow says code, run says agent → run wins."""
        wf = _make_workflow(run_with="code")
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_workflow_agent_with_agent_run(self, service):
        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_workflow_agent_overrides_code_version(self, service):
        """Explicit workflow-level agent takes priority over code_version."""
        wf = _make_workflow(run_with="agent", code_version=2)
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_both_agent_defaults_to_agent(self, service):
        """When both workflow and run are agent, default to agent."""
        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_code_version_with_agent_run_with(self, service):
        """code_version >= 1 with run_with=agent should still check code_version fallback."""
        wf = _make_workflow(run_with="agent", code_version=2)
        wr = _make_run(run_with="agent")
        # run_with=agent on both sides → agent wins, code_version ignored
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_code_version_with_code_run_with(self, service):
        """code_version >= 1 with explicit run_with=code should run code."""
        wf = _make_workflow(run_with="code", code_version=2)
        wr = _make_run(run_with="agent")
        # run-level agent overrides workflow code
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_workflow_code_run_code(self, service):
        """Both workflow and run say code → code."""
        wf = _make_workflow(run_with="code", code_version=2)
        wr = _make_run(run_with="code")
        assert await service.should_run_script(wf, wr) is True

    @pytest.mark.asyncio
    async def test_workflow_code_no_code_version(self, service):
        """Workflow says code, no code_version → should still run code."""
        wf = _make_workflow(run_with="code")
        wr = _make_run(run_with="code")
        assert await service.should_run_script(wf, wr) is True

    @pytest.mark.asyncio
    async def test_legacy_adaptive_caching_does_not_override_agent(self, service):
        """Legacy adaptive_caching=True does NOT override explicit run_with=agent."""
        wf = _make_workflow(run_with="agent", adaptive_caching=True)
        wr = _make_run(run_with="agent")
        # Explicit agent wins — adaptive_caching is a legacy fallback
        # that only applied when run_with was null (no longer possible).
        assert await service.should_run_script(wf, wr) is False


class TestCodeModeGate:
    """WorkflowService asks the app-level gate only for intended-code runs."""

    @pytest.mark.asyncio
    async def test_gate_enabled_keeps_code(self, monkeypatch):
        agent_function = _RecordingAgentFunction(enabled=True)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        service = WorkflowService()
        wf = _make_workflow(run_with="code", code_version=2)
        wr = _make_run(run_with="code")
        assert await service.should_run_script(wf, wr) is True
        assert agent_function.calls == [("wr_test", "org_test", "wpid_test")]

    @pytest.mark.asyncio
    async def test_gate_disabled_diverts(self, monkeypatch):
        agent_function = _RecordingAgentFunction(enabled=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        service = WorkflowService()
        wf = _make_workflow(run_with="code", code_version=2)
        wr = _make_run(run_with="code")
        assert await service.should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_gate_not_consulted_when_intended_agent(self, monkeypatch):
        agent_function = _RecordingAgentFunction(enabled=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        service = WorkflowService()
        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await service.should_run_script(wf, wr) is False
        assert agent_function.calls == []

    @pytest.mark.asyncio
    async def test_inherits_workflow_run_with_when_run_run_with_is_none(self, monkeypatch):
        """workflow_run.run_with=None means inherit-from-workflow. The app gate
        must still apply when the workflow itself says 'code'."""
        agent_function = _RecordingAgentFunction(enabled=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        service = WorkflowService()
        wf = _make_workflow(run_with="code", code_version=2)
        wr = _make_run(run_with=None)
        assert await service.should_run_script(wf, wr) is False
        assert len(agent_function.calls) == 1


class _UpgradingAgentFunction(AgentFunction):
    def __init__(self, upgrade: bool, keep_code: bool = True) -> None:
        self.upgrade = upgrade
        self.keep_code = keep_code
        self.upgrade_calls: list[tuple[str, str, str]] = []

    async def should_upgrade_to_code_mode(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
        self.upgrade_calls.append(
            (workflow_run.workflow_run_id, workflow_run.organization_id, workflow.workflow_permanent_id)
        )
        return self.upgrade

    async def should_keep_code_mode_for_workflow_run(self, *, workflow: Workflow, workflow_run: WorkflowRun) -> bool:
        return self.keep_code


class TestCodeModeUpgrade:
    """A flag-gated rollout can upgrade an agent-intended run to code."""

    @pytest.mark.asyncio
    async def test_upgrade_flips_agent_to_code(self, monkeypatch):
        agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await WorkflowService().should_run_script(wf, wr) is True
        assert agent_function.upgrade_calls == [("wr_test", "org_test", "wpid_test")]

    @pytest.mark.asyncio
    async def test_upgrade_disabled_stays_agent(self, monkeypatch):
        agent_function = _UpgradingAgentFunction(upgrade=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await WorkflowService().should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_upgraded_run_still_passes_keep_code_gate(self, monkeypatch):
        """Upgrade True but keep-code gate False → still agent (gate can veto)."""
        agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent")
        assert await WorkflowService().should_run_script(wf, wr) is False

    @pytest.mark.asyncio
    async def test_upgrade_not_consulted_when_already_code(self, monkeypatch):
        agent_function = _UpgradingAgentFunction(upgrade=False)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        wf = _make_workflow(run_with="code")
        wr = _make_run(run_with="code")
        assert await WorkflowService().should_run_script(wf, wr) is True
        assert agent_function.upgrade_calls == []

    @pytest.mark.asyncio
    async def test_fallback_retry_never_re_upgraded(self, monkeypatch):
        # A code->agent fallback retry (run_with=agent, retried_from set) must never be re-upgraded
        # to code, or the rollout would re-run the very script that just failed and the fallback
        # would be a no-op for exactly the rollout population.
        agent_function = _UpgradingAgentFunction(upgrade=True, keep_code=True)
        monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
        from skyvern.forge.sdk.workflow.service import WorkflowService

        wf = _make_workflow(run_with="agent")
        wr = _make_run(run_with="agent", retried_from_workflow_run_id="wr_orig")
        assert await WorkflowService().should_run_script(wf, wr) is False
        assert agent_function.upgrade_calls == []


class TestScriptReviewerGate:
    """The script reviewer should only fire when the script was actually executed.

    The gate requires BOTH is_adaptive_caching()=True AND should_run_script()=True.
    This prevents wasting LLM tokens reviewing scripts based on agent-only runs.
    """

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        "wf_run_with,run_run_with,code_version,expect_reviewer",
        [
            ("code", "code", 2, True),  # code + code_version=2 → adaptive caching on
            ("code", "code", 1, False),  # code + code_version=1 → adaptive caching off
            ("agent", "agent", 2, False),  # agent overrides everything
            ("agent", "agent", None, False),  # agent, no code_version
            ("code", "agent", 2, False),  # run-level agent overrides
            ("agent", "code", None, False),  # code without code_version → not adaptive
            ("agent", "code", 2, True),  # run-level code + code_version=2 → adaptive
        ],
    )
    async def test_reviewer_gate(self, service, wf_run_with, run_run_with, code_version, expect_reviewer):
        wf = _make_workflow(run_with=wf_run_with, code_version=code_version)
        wr = _make_run(run_with=run_run_with)
        should_review = is_adaptive_caching(wf, wr) and await service.should_run_script(wf, wr)
        assert should_review is expect_reviewer

    @pytest.mark.asyncio
    async def test_legacy_adaptive_caching_backward_compat(self, service):
        """Legacy adaptive_caching=True with code_version=None still enables reviewer."""
        wf = _make_workflow(run_with="agent", adaptive_caching=True, code_version=None)
        wr = _make_run(run_with="code")
        should_review = is_adaptive_caching(wf, wr) and await service.should_run_script(wf, wr)
        assert should_review is True