Skip to content

Automate browser based workflows with AI

Add flag-gated skip of the check-user-goal LLM call for cached-script runs (#7977)

Aaron Perez35bc570c77f5
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index cae32d741..f762e14e8 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -77,6 +77,30 @@ LOG = structlog.get_logger()
action_wrap = SkyvernPage.action_wrap
+async def _skip_cached_script_complete_verification(context: skyvern_context.SkyvernContext) -> bool:
+ """A cached script that replayed every recorded action has already determined
+ completion; with the flag on, complete() skips the check-user-goal LLM call."""
+ try:
+ enabled = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION",
+ context.workflow_run_id or context.task_id or "",
+ properties={"organization_id": context.organization_id},
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to check SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION; keeping LLM verification",
+ exc_info=True,
+ )
+ return False
+ if enabled:
+ LOG.info(
+ "Skipping cached-script complete verification (SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION enabled)",
+ workflow_run_id=context.workflow_run_id,
+ task_id=context.task_id,
+ )
+ return bool(enabled)
+
+
class ScriptSkyvernPage(SkyvernPage):
"""
A minimal adapter around the chosen driver that:
@@ -1141,17 +1165,22 @@ class ScriptSkyvernPage(SkyvernPage):
if not step:
return
+ # Static (pinned) scripts verify page state themselves —
+ # skip LLM verification which can reject valid completions
+ # (e.g. sign-in page with pending email verification).
+ # AI-generated cached scripts still get LLM verification unless
+ # SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION opts the run out.
+ verified = bool(context.is_static_script)
+ if not verified and context.script_mode:
+ verified = await _skip_cached_script_complete_verification(context)
+
action = CompleteAction(
organization_id=context.organization_id,
task_id=context.task_id,
step_id=context.step_id,
step_order=step.order,
action_order=context.action_order,
- # Static (pinned) scripts verify page state themselves —
- # skip LLM verification which can reject valid completions
- # (e.g. sign-in page with pending email verification).
- # AI-generated cached scripts still get LLM verification.
- verified=bool(context.is_static_script),
+ verified=verified,
)
# result = await ActionHandler.handle_action(self.scraped_page, task, step, self.page, action)
result = await handle_complete_action(action, self.page, self.scraped_page, task, step)
diff --git a/tests/unit/test_script_skyvern_page.py b/tests/unit/test_script_skyvern_page.py
index 96de26853..8fd178cc6 100644
--- a/tests/unit/test_script_skyvern_page.py
+++ b/tests/unit/test_script_skyvern_page.py
@@ -18,6 +18,7 @@ import pytest
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from skyvern.config import settings
+from skyvern.core.script_generations import script_skyvern_page as script_page_module
from skyvern.core.script_generations.real_skyvern_page_ai import RealSkyvernPageAi
from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
from skyvern.core.script_generations.skyvern_page import ResolvedSensitiveValue, SkyvernPage
@@ -685,6 +686,92 @@ async def test_complete_raises_illegit_complete_subclass_when_handler_rejects(mo
await script_page.complete()
+ "script_mode,flag_result,expected_verified,expect_flag_consulted",
+ [
+ (True, True, True, True),
+ (True, False, False, True),
+ (True, Exception("posthog down"), False, True),
+ (False, True, False, False),
+ ],
+)
+async def test_complete_skip_cached_script_verification_flag(
+ mock_scraped_page,
+ mock_ai,
+ monkeypatch: pytest.MonkeyPatch,
+ script_mode,
+ flag_result,
+ expected_verified,
+ expect_flag_consulted,
+):
+ """
+ SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION (default off) marks the script-mode
+ CompleteAction as verified so handle_complete_action skips the check-user-goal
+ LLM call. Flag errors fail safe (LLM verification runs); the flag is only
+ consulted in script mode.
+ """
+ script_page = _make_script_page(mock_scraped_page, mock_ai)
+
+ mock_context = MagicMock()
+ mock_context.organization_id = "org_123"
+ mock_context.workflow_run_id = "wr_456"
+ mock_context.task_id = "tsk_789"
+ mock_context.step_id = "stp_012"
+ mock_context.action_order = 0
+ mock_context.skip_complete_verification = False
+ mock_context.code_version = 1
+ mock_context.is_static_script = False
+ mock_context.script_mode = script_mode
+
+ mock_step = MagicMock()
+ mock_step.order = 0
+
+ provider = MagicMock()
+ if isinstance(flag_result, Exception):
+ provider.is_feature_enabled_cached = AsyncMock(side_effect=flag_result)
+ else:
+ provider.is_feature_enabled_cached = AsyncMock(return_value=flag_result)
+ handle_complete = AsyncMock(return_value=[MagicMock(success=True)])
+
+ monkeypatch.setattr(script_page_module.app, "EXPERIMENTATION_PROVIDER", provider)
+
+ with (
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.skyvern_context.current",
+ return_value=mock_context,
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_task",
+ new_callable=AsyncMock,
+ return_value=MagicMock(),
+ ),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.app.DATABASE.tasks.get_step",
+ new_callable=AsyncMock,
+ return_value=mock_step,
+ ),
+ patch.object(ScriptSkyvernPage, "_update_step_output_before_complete", new_callable=AsyncMock),
+ patch.object(ScriptSkyvernPage, "_create_final_screenshot", new_callable=AsyncMock),
+ patch(
+ "skyvern.core.script_generations.script_skyvern_page.handle_complete_action",
+ handle_complete,
+ ),
+ ):
+ await script_page.complete()
+
+ action = handle_complete.call_args.args[0]
+ assert action.verified is expected_verified
+ if expect_flag_consulted:
+ provider.is_feature_enabled_cached.assert_awaited_once_with(
+ "SKIP_CACHED_SCRIPT_COMPLETE_VERIFICATION",
+ "wr_456",
+ properties={"organization_id": "org_123"},
+ )
+ else:
+ provider.is_feature_enabled_cached.assert_not_awaited()
+
+
# =============================================================================
# Tests for fill() proactive upgrade when value=None + prompt
# =============================================================================