Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Gate script reviewer on actual script execution (#5183)
pedrohsdb74a2f6187887
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex ef60f5c6c..5e2cee47e 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -1111,12 +1111,20 @@ class WorkflowService:# Trigger AI Script Reviewer for adaptive caching workflows# Include terminated and failed runs (triage will filter non-code-fixable failures)# Skip canceled (user stopped) and timed_out (infrastructure issue)+ # Only trigger if the script was actually executed this run — reviewing based on+ # agent-only runs provides no signal about script quality and wastes LLM tokens.# Only trigger if this run used the latest script version — stale runs produce# episodes that may already be fixed in newer versions, and reviewing them creates# redundant/regressive versions.- if is_adaptive_caching(workflow, workflow_run) and pre_finally_status not in (- WorkflowRunStatus.canceled,- WorkflowRunStatus.timed_out,+ is_script_execution = self.should_run_script(workflow, workflow_run)+ if (+ is_adaptive_caching(workflow, workflow_run)+ and is_script_execution+ and pre_finally_status+ not in (+ WorkflowRunStatus.canceled,+ WorkflowRunStatus.timed_out,+ )):should_trigger_reviewer = Truecurrent_ctx = skyvern_context.current()@@ -1139,6 +1147,12 @@ class WorkflowService:self._trigger_script_reviewer(workflow, workflow_run, pre_finally_status=pre_finally_status),name=f"script_reviewer_{workflow_run.workflow_run_id}",)+ elif is_adaptive_caching(workflow, workflow_run):+ LOG.info(+ "Skipping script reviewer - script was not executed this run",+ workflow_run_id=workflow_run.workflow_run_id,+ run_with=workflow_run.run_with,+ )# Execute finally block if configured. Skip for: canceled (user explicitly stopped)should_run_finally = finally_block_label and pre_finally_status != WorkflowRunStatus.canceleddiff --git a/tests/unit/test_should_run_script.py b/tests/unit/test_should_run_script.pyindex c0874d6b1..ce5364e5d 100644--- a/tests/unit/test_should_run_script.py+++ b/tests/unit/test_should_run_script.py@@ -1,15 +1,23 @@-"""Tests for WorkflowService.should_run_script().+"""Tests for WorkflowService.should_run_script() and script reviewer gating.Verifies the priority chain:run-level run_with > workflow-level run_with > default (agent).adaptive_caching alone does NOT force code mode.++Also verifies that the script reviewer only fires when the script was actually+executed (should_run_script=True), not merely when adaptive caching is enabled."""from datetime import datetime, timezoneimport pytest-from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus+from skyvern.forge.sdk.workflow.models.workflow import (+ Workflow,+ WorkflowRun,+ WorkflowRunStatus,+ is_adaptive_caching,+)def _make_workflow(run_with: str | None = None, adaptive_caching: bool = False) -> Workflow:@@ -105,3 +113,30 @@ class TestShouldRunScript:wf = _make_workflow(run_with="code_v2")wr = _make_run(run_with=None)assert service.should_run_script(wf, wr) is True+++class TestScriptReviewerGate:+ """The script reviewer should only fire when the script was actually executed.++ The gate requires BOTH is_adaptive_caching()=True AND should_run_script()=True.+ This prevents wasting LLM tokens reviewing scripts based on agent-only runs.+ """++ @pytest.mark.parametrize(+ "run_with,adaptive_caching,expect_reviewer",+ [+ ("code_v2", True, True),+ ("code_v2", False, True), # is_adaptive_caching=True for code_v2 regardless+ (None, True, False), # the key case: adaptive caching on but script not executed+ (None, False, False),+ ("agent", True, False),+ ("agent", False, False),+ ("code", True, False), # code without code_v2 → is_adaptive_caching=False+ ("code", False, False),+ ],+ )+ def test_reviewer_gate(self, service, run_with, adaptive_caching, expect_reviewer):+ wf = _make_workflow(run_with=None, adaptive_caching=adaptive_caching)+ wr = _make_run(run_with=run_with)+ should_review = is_adaptive_caching(wf, wr) and service.should_run_script(wf, wr)+ assert should_review is expect_reviewer