Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Apply script review daily cap to ALL review types (#5273)
pedrohsdb48db694cac1d
diff --git a/skyvern/config.py b/skyvern/config.pyindex c4699f17e..25ceb69d8 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -30,7 +30,7 @@ class Settings(BaseSettings):ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE: bool = False# Script reviewer settings- FAILURE_REVIEW_DAILY_CAP: int = 3 # Max failure-triggered script reviews per wpid per day+ SCRIPT_REVIEW_DAILY_CAP: int = 5 # Max script reviews per wpid per day (all review types)ADDITIONAL_MODULES: list[str] = []diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 4aa906a43..fb61f244e 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -5109,22 +5109,19 @@ class WorkflowService:)return- # Determine if this is a failure-triggered review (script crashed, run failed)- # vs a fallback-triggered review (script failed but agent succeeded).- # Failure reviews are capped per wpid per day to prevent spam.- is_failure_review = pre_finally_status == WorkflowRunStatus.failed-- if is_failure_review:- cap_exceeded = await self._check_failure_review_cap(+ # Cap ALL script reviews (fallback + failure) per wpid per day to prevent+ # runaway revision churn when the same issue repeats every run.+ cap_exceeded = await self._check_script_review_cap(+ workflow_permanent_id=workflow.workflow_permanent_id,+ )+ if cap_exceeded:+ LOG.info(+ "Skipping script review — daily cap exceeded for wpid",workflow_permanent_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run.workflow_run_id,+ pre_finally_status=pre_finally_status,)- if cap_exceeded:- LOG.info(- "Skipping failure-triggered script review — daily cap exceeded for wpid",- workflow_permanent_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run.workflow_run_id,- )- return+ return# Non-blocking lock per script familycache = CacheFactory.get_cache()@@ -5153,10 +5150,10 @@ class WorkflowService:await self._run_reviewer_locked(workflow, workflow_run, script_revision_id, script_id)review_ran = True- # Increment the failure review counter ONLY after a review actually ran.+ # Increment the review counter ONLY after a review actually ran.# Skipped reviews (e.g., LockError) should not consume cap budget.- if is_failure_review and review_ran:- await self._increment_failure_review_counter(+ if review_ran:+ await self._increment_script_review_counter(workflow_permanent_id=workflow.workflow_permanent_id,)except Exception:@@ -5167,13 +5164,13 @@ class WorkflowService:)@staticmethod- def _failure_review_cap_key(workflow_permanent_id: str) -> str:- """Build the Redis key for the daily failure-review counter."""+ def _script_review_cap_key(workflow_permanent_id: str) -> str:+ """Build the Redis key for the daily script-review counter."""today = datetime.now(UTC).strftime("%Y-%m-%d")- return f"script_reviewer:failure_cap:{workflow_permanent_id}:{today}"+ return f"script_reviewer:daily_cap:{workflow_permanent_id}:{today}"- async def _check_failure_review_cap(self, workflow_permanent_id: str) -> bool:- """Check if the daily failure-review cap has been reached for this wpid.+ async def _check_script_review_cap(self, workflow_permanent_id: str) -> bool:+ """Check if the daily script-review cap has been reached for this wpid.Returns True if the cap is exceeded and the review should be skipped.Uses Redis get/set to maintain a per-wpid daily counter.@@ -5182,18 +5179,18 @@ class WorkflowService:cache = CacheFactory.get_cache()if cache is None:return False- cap_key = self._failure_review_cap_key(workflow_permanent_id)+ cap_key = self._script_review_cap_key(workflow_permanent_id)raw_count = await cache.get(cap_key)if raw_count is not None:count = int(raw_count)- if count >= settings.FAILURE_REVIEW_DAILY_CAP:+ if count >= settings.SCRIPT_REVIEW_DAILY_CAP:return Trueexcept Exception:- LOG.debug("Failed to check failure review cap, allowing review", exc_info=True)+ LOG.debug("Failed to check script review cap, allowing review", exc_info=True)return False- async def _increment_failure_review_counter(self, workflow_permanent_id: str) -> None:- """Increment the daily failure-review counter for this wpid.+ async def _increment_script_review_counter(self, workflow_permanent_id: str) -> None:+ """Increment the daily script-review counter for this wpid.Uses Redis get+set with a 48-hour TTL (covers timezone edge cases).Note: get+set is not atomic, so concurrent reviews for the same wpid@@ -5206,12 +5203,12 @@ class WorkflowService:cache = CacheFactory.get_cache()if cache is None:return- cap_key = self._failure_review_cap_key(workflow_permanent_id)+ cap_key = self._script_review_cap_key(workflow_permanent_id)raw_count = await cache.get(cap_key)new_count = (int(raw_count) + 1) if raw_count is not None else 1await cache.set(cap_key, str(new_count), ex=timedelta(hours=48))except Exception:- LOG.debug("Failed to increment failure review counter", exc_info=True)+ LOG.debug("Failed to increment script review counter", exc_info=True)async def _run_reviewer_locked(self,diff --git a/tests/unit/test_failure_review_cap.py b/tests/unit/test_script_review_cap.pysimilarity index 75%rename from tests/unit/test_failure_review_cap.pyrename to tests/unit/test_script_review_cap.pyindex a6332d261..9e45cdf32 100644--- a/tests/unit/test_failure_review_cap.py+++ b/tests/unit/test_script_review_cap.py@@ -1,10 +1,10 @@-"""Tests for failure-triggered script review daily cap (SKY-8334).+"""Tests for script review daily cap.Validates that:-1. Failure reviews are capped at 3 per wpid per day-2. AI fallback reviews (non-failure) are NOT capped-3. Cap is keyed on wpid, not script_id (new script revision doesn't reset)-4. Cap resets on a new day+1. ALL script reviews (fallback + failure) are capped at SCRIPT_REVIEW_DAILY_CAP per wpid per day+2. Cap is keyed on wpid, not script_id (new script revision doesn't reset)+3. Cap resets on a new day+4. Counter only increments after a review actually runs (not on LockError)"""from datetime import UTC, datetime, timedelta@@ -38,8 +38,8 @@ def _make_workflow_run(run_id: str = "wr_test1") -> MagicMock:return wr-class TestCheckFailureReviewCap:- """Tests for _check_failure_review_cap."""+class TestCheckScriptReviewCap:+ """Tests for _check_script_review_cap."""@pytest.fixture(autouse=True)def setup(self) -> None:@@ -52,63 +52,63 @@ class TestCheckFailureReviewCap:@pytest.mark.asyncioasync def test_cap_not_reached_returns_false(self) -> None:- """When counter is below 3, review should proceed."""- self.mock_cache.get = AsyncMock(return_value="2")- result = await self.service._check_failure_review_cap("wpid_abc")+ """When counter is below cap, review should proceed."""+ self.mock_cache.get = AsyncMock(return_value="4")+ result = await self.service._check_script_review_cap("wpid_abc")assert result is False@pytest.mark.asyncioasync def test_cap_reached_returns_true(self) -> None:- """When counter is at 3 or above, review should be skipped."""- self.mock_cache.get = AsyncMock(return_value="3")- result = await self.service._check_failure_review_cap("wpid_abc")+ """When counter is at cap (5), review should be skipped."""+ self.mock_cache.get = AsyncMock(return_value="5")+ result = await self.service._check_script_review_cap("wpid_abc")assert result is True@pytest.mark.asyncioasync def test_cap_exceeded_returns_true(self) -> None:- """When counter is above 3, review should be skipped."""- self.mock_cache.get = AsyncMock(return_value="5")- result = await self.service._check_failure_review_cap("wpid_abc")+ """When counter is above cap, review should be skipped."""+ self.mock_cache.get = AsyncMock(return_value="10")+ result = await self.service._check_script_review_cap("wpid_abc")assert result is True@pytest.mark.asyncioasync def test_no_counter_returns_false(self) -> None:"""When no counter exists (first review of the day), review should proceed."""self.mock_cache.get = AsyncMock(return_value=None)- result = await self.service._check_failure_review_cap("wpid_abc")+ result = await self.service._check_script_review_cap("wpid_abc")assert result is False@pytest.mark.asyncioasync def test_cache_error_allows_review(self) -> None:"""When cache raises an error, review should still proceed (fail open)."""self.mock_cache.get = AsyncMock(side_effect=Exception("Redis down"))- result = await self.service._check_failure_review_cap("wpid_abc")+ result = await self.service._check_script_review_cap("wpid_abc")assert result is False@pytest.mark.asyncioasync def test_key_uses_wpid_not_script_id(self) -> None:"""The cap key must use workflow_permanent_id, not script_id."""- self.mock_cache.get = AsyncMock(return_value="3")+ self.mock_cache.get = AsyncMock(return_value="5")wpid = "wpid_specific_workflow"- await self.service._check_failure_review_cap(wpid)+ await self.service._check_script_review_cap(wpid)call_args = self.mock_cache.get.call_args[0][0]assert wpid in call_args- assert "script_reviewer:failure_cap:" in call_args+ assert "script_reviewer:daily_cap:" in call_args@pytest.mark.asyncioasync def test_key_includes_date(self) -> None:"""The cap key must include the current date for daily reset."""self.mock_cache.get = AsyncMock(return_value=None)today = datetime.now(UTC).strftime("%Y-%m-%d")- await self.service._check_failure_review_cap("wpid_abc")+ await self.service._check_script_review_cap("wpid_abc")call_args = self.mock_cache.get.call_args[0][0]assert today in call_args-class TestIncrementFailureReviewCounter:- """Tests for _increment_failure_review_counter."""+class TestIncrementScriptReviewCounter:+ """Tests for _increment_script_review_counter."""@pytest.fixture(autouse=True)def setup(self) -> None:@@ -123,7 +123,7 @@ class TestIncrementFailureReviewCounter:async def test_first_increment(self) -> None:"""First review of the day should set counter to 1."""self.mock_cache.get = AsyncMock(return_value=None)- await self.service._increment_failure_review_counter("wpid_abc")+ await self.service._increment_script_review_counter("wpid_abc")self.mock_cache.set.assert_called_once()args, kwargs = self.mock_cache.set.call_args@@ -134,7 +134,7 @@ class TestIncrementFailureReviewCounter:async def test_subsequent_increment(self) -> None:"""Second review should increment counter from 1 to 2."""self.mock_cache.get = AsyncMock(return_value="1")- await self.service._increment_failure_review_counter("wpid_abc")+ await self.service._increment_script_review_counter("wpid_abc")args, kwargs = self.mock_cache.set.call_argsassert args[1] == "2"@@ -144,11 +144,11 @@ class TestIncrementFailureReviewCounter:"""Cache errors should not propagate."""self.mock_cache.get = AsyncMock(side_effect=Exception("Redis down"))# Should not raise- await self.service._increment_failure_review_counter("wpid_abc")+ await self.service._increment_script_review_counter("wpid_abc")-class TestTriggerScriptReviewerFailureCap:- """Integration tests for _trigger_script_reviewer failure cap logic."""+class TestTriggerScriptReviewerCap:+ """Integration tests for _trigger_script_reviewer daily cap logic."""@pytest.fixture(autouse=True)def setup(self) -> None:@@ -178,8 +178,7 @@ class TestTriggerScriptReviewerFailureCap:workflow = _make_workflow()workflow_run = _make_workflow_run()- # Cap already at 3- self.mock_cache.get = AsyncMock(return_value="3")+ self.mock_cache.get = AsyncMock(return_value="5")with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:ctx = MagicMock()@@ -187,14 +186,12 @@ class TestTriggerScriptReviewerFailureCap:ctx.script_id = "script_1"mock_ctx.return_value = ctx- # Mock _run_reviewer_locked to track if it gets calledself.service._run_reviewer_locked = AsyncMock()await self.service._trigger_script_reviewer(workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Should NOT have called _run_reviewer_lockedself.service._run_reviewer_locked.assert_not_called()@pytest.mark.asyncio@@ -203,7 +200,6 @@ class TestTriggerScriptReviewerFailureCap:workflow = _make_workflow()workflow_run = _make_workflow_run()- # Counter at 1, under capself.mock_cache.get = AsyncMock(return_value="1")with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:@@ -218,17 +214,15 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Should have called _run_reviewer_lockedself.service._run_reviewer_locked.assert_called_once()@pytest.mark.asyncio- async def test_fallback_review_not_capped(self) -> None:- """AI fallback reviews (completed runs) should NEVER be capped."""+ async def test_fallback_review_proceeds_when_under_cap(self) -> None:+ """AI fallback review (completed run) should proceed when under the daily cap."""workflow = _make_workflow()workflow_run = _make_workflow_run()- # Even with counter at 100, fallback reviews should proceed- self.mock_cache.get = AsyncMock(return_value="100")+ self.mock_cache.get = AsyncMock(return_value="1")with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:ctx = MagicMock()@@ -238,21 +232,42 @@ class TestTriggerScriptReviewerFailureCap:self.service._run_reviewer_locked = AsyncMock()- # completed status = fallback review, NOT failure reviewawait self.service._trigger_script_reviewer(workflow, workflow_run, pre_finally_status=WorkflowRunStatus.completed)- # Should have called _run_reviewer_locked despite high counterself.service._run_reviewer_locked.assert_called_once()@pytest.mark.asyncio- async def test_terminated_review_not_capped(self) -> None:- """Terminated run reviews should NOT be capped (only failed runs are capped)."""+ async def test_fallback_review_also_capped(self) -> None:+ """AI fallback reviews (completed runs) should also be capped."""workflow = _make_workflow()workflow_run = _make_workflow_run()- self.mock_cache.get = AsyncMock(return_value="100")+ self.mock_cache.get = AsyncMock(return_value="5")++ with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:+ ctx = MagicMock()+ ctx.script_revision_id = "rev_1"+ ctx.script_id = "script_1"+ mock_ctx.return_value = ctx++ self.service._run_reviewer_locked = AsyncMock()++ await self.service._trigger_script_reviewer(+ workflow, workflow_run, pre_finally_status=WorkflowRunStatus.completed+ )++ # Should be skipped — fallback reviews are now capped too+ self.service._run_reviewer_locked.assert_not_called()++ @pytest.mark.asyncio+ async def test_terminated_review_also_capped(self) -> None:+ """Terminated run reviews should also be capped."""+ workflow = _make_workflow()+ workflow_run = _make_workflow_run()++ self.mock_cache.get = AsyncMock(return_value="5")with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:ctx = MagicMock()@@ -266,7 +281,7 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.terminated)- self.service._run_reviewer_locked.assert_called_once()+ self.service._run_reviewer_locked.assert_not_called()@pytest.mark.asyncioasync def test_cap_keyed_on_wpid_not_script_id(self) -> None:@@ -275,19 +290,17 @@ class TestTriggerScriptReviewerFailureCap:workflow = _make_workflow(wpid=wpid)workflow_run = _make_workflow_run()- # Simulate cap at 3 for this wpidtoday = datetime.now(UTC).strftime("%Y-%m-%d")- expected_key = f"script_reviewer:failure_cap:{wpid}:{today}"+ expected_key = f"script_reviewer:daily_cap:{wpid}:{today}"async def mock_get(key: str) -> str | None:if key == expected_key:- return "3"+ return "5"return Noneself.mock_cache.get = AsyncMock(side_effect=mock_get)with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:- # Even with a brand new script_id/revision, the cap should still applyctx = MagicMock()ctx.script_revision_id = "rev_brand_new"ctx.script_id = "script_brand_new"@@ -299,7 +312,6 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Should be skipped because the wpid cap is at 3self.service._run_reviewer_locked.assert_not_called()@pytest.mark.asyncio@@ -310,12 +322,11 @@ class TestTriggerScriptReviewerFailureCap:workflow_run = _make_workflow_run()today = datetime.now(UTC).strftime("%Y-%m-%d")- expected_key = f"script_reviewer:failure_cap:{wpid}:{today}"+ expected_key = f"script_reviewer:daily_cap:{wpid}:{today}"- # Counter is None for today (even though yesterday may have been 3+)async def mock_get(key: str) -> str | None:if key == expected_key:- return None # No counter for today+ return Nonereturn Noneself.mock_cache.get = AsyncMock(side_effect=mock_get)@@ -332,16 +343,14 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Should proceed because today's counter is 0self.service._run_reviewer_locked.assert_called_once()@pytest.mark.asyncio- async def test_failure_review_increments_counter_after_success(self) -> None:- """After a successful failure review, the counter should be incremented."""+ async def test_review_increments_counter_after_success(self) -> None:+ """After a successful review, the counter should be incremented."""workflow = _make_workflow()workflow_run = _make_workflow_run()- # Start at 0self.mock_cache.get = AsyncMock(return_value=None)with patch("skyvern.forge.sdk.core.skyvern_context.current") as mock_ctx:@@ -356,14 +365,13 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Counter should have been set to "1"- set_calls = [c for c in self.mock_cache.set.call_args_list if "failure_cap" in c[0][0]]+ set_calls = [c for c in self.mock_cache.set.call_args_list if "daily_cap" in c[0][0]]assert len(set_calls) == 1assert set_calls[0][0][1] == "1"@pytest.mark.asyncio- async def test_fallback_review_does_not_increment_counter(self) -> None:- """AI fallback reviews should NOT increment the failure counter."""+ async def test_fallback_review_also_increments_counter(self) -> None:+ """AI fallback reviews should also increment the counter."""workflow = _make_workflow()workflow_run = _make_workflow_run()@@ -381,9 +389,9 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.completed)- # No failure_cap set calls- set_calls = [c for c in self.mock_cache.set.call_args_list if "failure_cap" in str(c)]- assert len(set_calls) == 0+ set_calls = [c for c in self.mock_cache.set.call_args_list if "daily_cap" in c[0][0]]+ assert len(set_calls) == 1+ assert set_calls[0][0][1] == "1"@pytest.mark.asyncioasync def test_lock_error_does_not_increment_counter(self) -> None:@@ -391,10 +399,8 @@ class TestTriggerScriptReviewerFailureCap:workflow = _make_workflow()workflow_run = _make_workflow_run()- # Counter at 0, under capself.mock_cache.get = AsyncMock(return_value=None)- # Make the lock raise LockError to simulate contentionmock_lock = AsyncMock()mock_lock.__aenter__ = AsyncMock(side_effect=LockError("Could not acquire lock"))mock_lock.__aexit__ = AsyncMock(return_value=False)@@ -412,9 +418,7 @@ class TestTriggerScriptReviewerFailureCap:workflow, workflow_run, pre_finally_status=WorkflowRunStatus.failed)- # Review should NOT have run (lock contention)self.service._run_reviewer_locked.assert_not_called()- # Counter should NOT have been incremented- set_calls = [c for c in self.mock_cache.set.call_args_list if "failure_cap" in str(c)]+ set_calls = [c for c in self.mock_cache.set.call_args_list if "daily_cap" in str(c)]assert len(set_calls) == 0