Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Add flag-gated skip of the check-user-goal LLM call for cached-script runs (#7977)
Aaron Perez35bc570c77f5
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.pyindex cae32d741..f762e14e8 100644--- a/skyvern/core/script_generations/script_skyvern_page.py+++ b/skyvern/core/script_generations/script_skyvern_page.py@@ -77,6 +77,30 @@ LOG = structlog.get_logger()action_wrap = SkyvernPage.action_wrap+async def _skip_cached_script_complete_verification(context: skyvern_context.SkyvernContext) -> bool:+ """A cached script that replayed every recorded action has already determined+ completion; with the flag on, complete() skips the check-user-goal LLM call."""+ try:+ enabled = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION",+ context.workflow_run_id or context.task_id or "",+ properties={"organization_id": context.organization_id},+ )+ except Exception:+ LOG.warning(+ "Failed to check SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION; keeping LLM verification",+ exc_info=True,+ )+ return False+ if enabled:+ LOG.info(+ "Skipping cached-script complete verification (SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION enabled)",+ workflow_run_id=context.workflow_run_id,+ task_id=context.task_id,+ )+ return bool(enabled)++class ScriptSkyvernPage(SkyvernPage):"""A minimal adapter around the chosen driver that:@@ -1141,17 +1165,22 @@ class ScriptSkyvernPage(SkyvernPage):if not step:return+ # Static (pinned) scripts verify page state themselves —+ # skip LLM verification which can reject valid completions+ # (e.g. sign-in page with pending email verification).+ # AI-generated cached scripts still get LLM verification unless+ # SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION opts the run out.+ verified = bool(context.is_static_script)+ if not verified and context.script_mode:+ verified = await _skip_cached_script_complete_verification(context)+action = CompleteAction(organization_id=context.organization_id,task_id=context.task_id,step_id=context.step_id,step_order=step.order,action_order=context.action_order,- # Static (pinned) scripts verify page state themselves —- # skip LLM verification which can reject valid completions- # (e.g. sign-in page with pending email verification).- # AI-generated cached scripts still get LLM verification.- verified=bool(context.is_static_script),+ verified=verified,)# result = await ActionHandler.handle_action(self.scraped_page, task, step, self.page, action)result = await handle_complete_action(action, self.page, self.scraped_page, task, step)diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.pyindex 96de26853..8fd178cc6 100644--- a/tests/unit/test_script_skyvern_page.py+++ b/tests/unit/test_script_skyvern_page.py@@ -18,6 +18,7 @@ import pytestfrom playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom skyvern.config import settings+from skyvern.core.script_generations import script_skyvern_page as script_page_modulefrom skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAifrom skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPagefrom skyvern.core.script_generations.skyvern_page import ResolvedSensitiveValue, SkyvernPage@@ -685,6 +686,92 @@ async def test_complete_raises_illegit_complete_subclass_when_handler_rejects(moawait script_page.complete()+ "script_mode,flag_result,expected_verified,expect_flag_consulted",+ [+ (True, True, True, True),+ (True, False, False, True),+ (True, Exception("posthog down"), False, True),+ (False, True, False, False),+ ],+)+async def test_complete_skip_cached_script_verification_flag(+ mock_scraped_page,+ mock_ai,+ monkeypatch: pytest.MonkeyPatch,+ script_mode,+ flag_result,+ expected_verified,+ expect_flag_consulted,+):+ """+ SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION (default off) marks the script-mode+ CompleteAction as verified so handle_complete_action skips the check-user-goal+ LLM call. Flag errors fail safe (LLM verification runs); the flag is only+ consulted in script mode.+ """+ script_page = _make_script_page(mock_scraped_page, mock_ai)++ mock_context = MagicMock()+ mock_context.organization_id = "org_123"+ mock_context.workflow_run_id = "wr_456"+ mock_context.task_id = "tsk_789"+ mock_context.step_id = "stp_012"+ mock_context.action_order = 0+ mock_context.skip_complete_verification = False+ mock_context.code_version = 1+ mock_context.is_static_script = False+ mock_context.script_mode = script_mode++ mock_step = MagicMock()+ mock_step.order = 0++ provider = MagicMock()+ if isinstance(flag_result, Exception):+ provider.is_feature_enabled_cached = AsyncMock(side_effect=flag_result)+ else:+ provider.is_feature_enabled_cached = AsyncMock(return_value=flag_result)+ handle_complete = AsyncMock(return_value=[MagicMock(success=True)])++ monkeypatch.setattr(script_page_module.app, "EXPERIMENTATION_PROVIDER", provider)++ with (+ patch(+ "skyvern.core.script_generations.script_skyvern_page.skyvern_context.current",+ return_value=mock_context,+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_task",+ new_callable=AsyncMock,+ return_value=MagicMock(),+ ),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_step",+ new_callable=AsyncMock,+ return_value=mock_step,+ ),+ patch.object(ScriptSkyvernPage, "_update_step_output_before_complete", new_callable=AsyncMock),+ patch.object(ScriptSkyvernPage, "_create_final_screenshot", new_callable=AsyncMock),+ patch(+ "skyvern.core.script_generations.script_skyvern_page.handle_complete_action",+ handle_complete,+ ),+ ):+ await script_page.complete()++ action = handle_complete.call_args.args[0]+ assert action.verified is expected_verified+ if expect_flag_consulted:+ provider.is_feature_enabled_cached.assert_awaited_once_with(+ "SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION",+ "wr_456",+ properties={"organization_id": "org_123"},+ )+ else:+ provider.is_feature_enabled_cached.assert_not_awaited()++# =============================================================================# Tests for fill() proactive upgrade when value=None + prompt# =============================================================================