Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Gate AI-fallback completion for the Workday submit block (SKY-12992) (#7718)
Cindy Li86187714ffa1
diff --git a/skyvern/__init__.py b/skyvern/__init__.pyindex da736e0b0..8dfdedf48 100644--- a/skyvern/__init__.py+++ b/skyvern/__init__.py@@ -5,6 +5,12 @@ from skyvern._version import __version__from skyvern.utils import setup_windows_event_loop_policyif typing.TYPE_CHECKING:+ # Expose the server lazy-imports (runtime-gated via __getattr__ below) to type checkers so+ # `skyvern.SkyvernPage` / `skyvern.cached` resolve to the real symbols. Without this mypy sees+ # them only as __getattr__'s Any and flakily reports "not valid as a type" / "Any? not+ # callable" in the generated scripts, depending on incremental-cache state.+ from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage # noqa: E402,F401+ from skyvern.core.script_generations.workflow_wrappers import cached, workflow # noqa: E402,F401from skyvern.library import Skyvern # noqa: E402,F401diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex f6b297103..f38e83c71 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -5390,6 +5390,20 @@ class ForgeAgent:)complete_action = None+ if isinstance(complete_action, CompleteAction) and not await app.AGENT_FUNCTION.gate_step_completion(+ task=task,+ step=step,+ task_block=task_block,+ page=page,+ browser_state=browser_state,+ ):+ LOG.info(+ "Step completion vetoed by completion gate; continuing with next step",+ task_id=task.task_id,+ step_id=step.step_id,+ )+ complete_action = None+if complete_action is not None:asyncio.create_task(self._persist_speculative_metadata_for_discarded_plan(@@ -6187,7 +6201,28 @@ class ForgeAgent:task_block=task_block,)- if step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal)):+ goal_achieved = step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal))+ # A decisive COMPLETE action skips parallel verification, so this is the other place a+ # completion is accepted — gate it too, or the veto is bypassed whenever the agent emits+ # complete directly.+ if (+ goal_achieved+ and browser_state is not None+ and not await app.AGENT_FUNCTION.gate_step_completion(+ task=task,+ step=step,+ task_block=task_block,+ page=page,+ browser_state=browser_state,+ )+ ):+ LOG.info(+ "Decisive step completion vetoed by completion gate; continuing with next step",+ task_id=task.task_id,+ step_id=step.step_id,+ )+ goal_achieved = False+ if goal_achieved:LOG.info("Step completed and goal achieved, marking task as completed",step_order=step.order,diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex 93d4d17d0..d7cd6d9d5 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -51,7 +51,7 @@ from skyvern.forge.sdk.services import ()from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResultfrom skyvern.forge.sdk.trace import traced-from skyvern.forge.sdk.workflow.models.block import BlockTypeVar+from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVarfrom skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestinationfrom skyvern.services.otp_gmail import GmailOTPVerificationContextfrom skyvern.webeye.actions.actions import Action@@ -1126,6 +1126,26 @@ class AgentFunction:if step.status == StepStatus.completed:await self._maybe_close_magic_link_page(task)+ async def gate_step_completion(+ self,+ *,+ task: Task,+ step: Step,+ task_block: BaseTaskBlock | None,+ page: Page | None,+ browser_state: BrowserState,+ ) -> bool:+ """Gate whether an agent completion verdict for this step is accepted.++ Called once the parallel verifier has produced a CompleteAction, before the step+ and task are marked completed. Return True to accept; False to reject, which makes+ the agent keep going and fail safe at max steps rather than falsely completing.+ OSS accepts everything; a deployment may override to hold specific blocks (e.g. a+ submit block whose AI fallback would otherwise complete without a deterministic+ confirmation check) to a stricter gate.+ """+ return True+async def _maybe_close_magic_link_page(self, task: Task) -> None:"""Close a magic-link confirmation page if it shows close/return signals.diff --git a/tests/unit/test_completion_gate.py b/tests/unit/test_completion_gate.pynew file mode 100644index 000000000..f26a87914--- /dev/null+++ b/tests/unit/test_completion_gate.py@@ -0,0 +1,163 @@+"""The completion gate (AGENT_FUNCTION.gate_step_completion) can veto an agent completion.++A vetoed CompleteAction must not mark the task completed; the agent continues (and fails safe+at max steps) instead of falsely completing. See SKY-12992.+"""++from datetime import UTC, datetime+from unittest.mock import AsyncMock++import pytest++from skyvern.forge import app+from skyvern.forge.agent import ForgeAgent+from skyvern.forge.sdk.models import StepStatus+from skyvern.forge.sdk.schemas.tasks import TaskStatus+from skyvern.schemas.runs import RunEngine+from skyvern.schemas.steps import AgentStepOutput+from skyvern.webeye.actions.actions import CompleteAction+from skyvern.webeye.actions.responses import ActionSuccess+from tests.unit.helpers import (+ make_browser_state,+ make_organization,+ make_step,+ make_task,+ setup_parallel_verification_mocks,+)+++async def test_completion_gate_veto_does_not_complete_task(monkeypatch: pytest.MonkeyPatch) -> None:+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization)++ step = make_step(+ now,+ task,+ step_id="step-123",+ status=StepStatus.completed,+ order=0,+ output=AgentStepOutput(action_results=[], actions_and_results=[]),+ )+ next_step = make_step(now, task, step_id="step-next", status=StepStatus.created, order=1, output=None)++ mocks = setup_parallel_verification_mocks(+ agent,+ step=step,+ task=task,+ monkeypatch=monkeypatch,+ next_step=next_step,+ complete_action=CompleteAction(reasoning="done", verified=True),+ handle_action_responses=[[ActionSuccess()]],+ )++ # Veto the completion.+ gate = AsyncMock(return_value=False)+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)++ browser_state, scraped_page, page = make_browser_state()+ completed, _last_step, next_created_step = await agent._handle_completed_step_with_parallel_verification(+ organization=organization,+ task=task,+ step=step,+ page=page,+ browser_state=browser_state,+ scraped_page=scraped_page,+ engine=RunEngine.skyvern_v1,+ )++ assert gate.await_count == 1+ assert completed is not True+ assert next_created_step is not None # loop continues with another step+ completed_calls = [c for c in mocks.update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]+ assert completed_calls == []+++async def test_completion_gate_accept_completes_task(monkeypatch: pytest.MonkeyPatch) -> None:+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization, navigation_goal=None) # skip data-extraction branch++ step = make_step(+ now,+ task,+ step_id="step-123",+ status=StepStatus.completed,+ order=0,+ output=AgentStepOutput(action_results=[], actions_and_results=[]),+ )++ mocks = setup_parallel_verification_mocks(+ agent,+ step=step,+ task=task,+ monkeypatch=monkeypatch,+ next_step=step,+ complete_action=CompleteAction(reasoning="done", verified=True),+ handle_action_responses=[[ActionSuccess()]],+ )++ gate = AsyncMock(return_value=True)+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)++ browser_state, scraped_page, page = make_browser_state()+ completed, _last_step, _next = await agent._handle_completed_step_with_parallel_verification(+ organization=organization,+ task=task,+ step=step,+ page=page,+ browser_state=browser_state,+ scraped_page=scraped_page,+ engine=RunEngine.skyvern_v1,+ )++ assert gate.await_count == 1+ assert completed is True+ completed_calls = [c for c in mocks.update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]+ assert completed_calls != []+++async def test_decisive_completion_gate_veto_creates_next_step(monkeypatch: pytest.MonkeyPatch) -> None:+ # A decisive COMPLETE action bypasses parallel verification and completes via+ # handle_completed_step's is_goal_achieved branch — the gate must fire here too.+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ task = make_task(now, organization)++ complete = CompleteAction(reasoning="done", verified=True)+ output = AgentStepOutput(action_results=[ActionSuccess()], actions_and_results=[(complete, [ActionSuccess()])])+ step = make_step(now, task, step_id="step-123", status=StepStatus.completed, order=0, output=output)+ assert step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal)) # sanity++ update_task = AsyncMock()+ monkeypatch.setattr(agent, "update_task", update_task)+ monkeypatch.setattr(agent, "update_step", AsyncMock(side_effect=lambda s, **k: s))+ monkeypatch.setattr(agent, "_check_workflow_run_step_budget", AsyncMock(return_value=None))+ next_step = make_step(now, task, step_id="step-next", status=StepStatus.created, order=1, output=None)+ monkeypatch.setattr(app.DATABASE.tasks, "create_step", AsyncMock(return_value=next_step))++ gate = AsyncMock(return_value=False)+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)++ browser_state, scraped_page, page = make_browser_state()+ completed, _last_step, created_next = await agent.handle_completed_step(+ organization=organization,+ task=task,+ step=step,+ page=page,+ browser_state=browser_state,+ scraped_page=scraped_page,+ engine=RunEngine.skyvern_v1,+ )++ assert gate.await_count == 1+ assert completed is None # not completed; loop continues+ assert created_next is next_step+ completed_calls = [c for c in update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]+ assert completed_calls == []