Skip to content

Automate browser based workflows with AI

Gate script reviewer on actual script execution (#5183)

pedrohsdb74a2f6187887
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index ef60f5c6c..5e2cee47e 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1111,12 +1111,20 @@ class WorkflowService:
# Trigger AI Script Reviewer for adaptive caching workflows
# Include terminated and failed runs (triage will filter non-code-fixable failures)
# Skip canceled (user stopped) and timed_out (infrastructure issue)
+ # Only trigger if the script was actually executed this run — reviewing based on
+ # agent-only runs provides no signal about script quality and wastes LLM tokens.
# Only trigger if this run used the latest script version — stale runs produce
# episodes that may already be fixed in newer versions, and reviewing them creates
# redundant/regressive versions.
- if is_adaptive_caching(workflow, workflow_run) and pre_finally_status not in (
- WorkflowRunStatus.canceled,
- WorkflowRunStatus.timed_out,
+ is_script_execution = self.should_run_script(workflow, workflow_run)
+ if (
+ is_adaptive_caching(workflow, workflow_run)
+ and is_script_execution
+ and pre_finally_status
+ not in (
+ WorkflowRunStatus.canceled,
+ WorkflowRunStatus.timed_out,
+ )
):
should_trigger_reviewer = True
current_ctx = skyvern_context.current()
@@ -1139,6 +1147,12 @@ class WorkflowService:
self._trigger_script_reviewer(workflow, workflow_run, pre_finally_status=pre_finally_status),
name=f"script_reviewer_{workflow_run.workflow_run_id}",
)
+ elif is_adaptive_caching(workflow, workflow_run):
+ LOG.info(
+ "Skipping script reviewer - script was not executed this run",
+ workflow_run_id=workflow_run.workflow_run_id,
+ run_with=workflow_run.run_with,
+ )
# Execute finally block if configured. Skip for: canceled (user explicitly stopped)
should_run_finally = finally_block_label and pre_finally_status != WorkflowRunStatus.canceled
diff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.py
index c0874d6b1..ce5364e5d 100644
--- a/tests/unit/test_should_run_script.py
+++ b/tests/unit/test_should_run_script.py
@@ -1,15 +1,23 @@
-"""Tests for WorkflowService.should_run_script().
+"""Tests for WorkflowService.should_run_script() and script reviewer gating.
Verifies the priority chain:
run-level run_with > workflow-level run_with > default (agent).
adaptive_caching alone does NOT force code mode.
+
+Also verifies that the script reviewer only fires when the script was actually
+executed (should_run_script=True), not merely when adaptive caching is enabled.
"""
from datetime import datetime, timezone
import pytest
-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
+from skyvern.forge.sdk.workflow.models.workflow import (
+ Workflow,
+ WorkflowRun,
+ WorkflowRunStatus,
+ is_adaptive_caching,
+)
def _make_workflow(run_with: str | None = None, adaptive_caching: bool = False) -> Workflow:
@@ -105,3 +113,30 @@ class TestShouldRunScript:
wf = _make_workflow(run_with="code_v2")
wr = _make_run(run_with=None)
assert service.should_run_script(wf, wr) is True
+
+
+class TestScriptReviewerGate:
+ """The script reviewer should only fire when the script was actually executed.
+
+ The gate requires BOTH is_adaptive_caching()=True AND should_run_script()=True.
+ This prevents wasting LLM tokens reviewing scripts based on agent-only runs.
+ """
+
+ @pytest.mark.parametrize(
+ "run_with,adaptive_caching,expect_reviewer",
+ [
+ ("code_v2", True, True),
+ ("code_v2", False, True), # is_adaptive_caching=True for code_v2 regardless
+ (None, True, False), # the key case: adaptive caching on but script not executed
+ (None, False, False),
+ ("agent", True, False),
+ ("agent", False, False),
+ ("code", True, False), # code without code_v2 → is_adaptive_caching=False
+ ("code", False, False),
+ ],
+ )
+ def test_reviewer_gate(self, service, run_with, adaptive_caching, expect_reviewer):
+ wf = _make_workflow(run_with=None, adaptive_caching=adaptive_caching)
+ wr = _make_run(run_with=run_with)
+ should_review = is_adaptive_caching(wf, wr) and service.should_run_script(wf, wr)
+ assert should_review is expect_reviewer