Skip to content

Automate browser based workflows with AI

Gate AI-fallback completion for the Workday submit block (SKY-12992) (#7718)

Cindy Li86187714ffa1
diff --git a/skyvern/__init__.py b/skyvern/__init__.py
index da736e0b0..8dfdedf48 100644
--- a/skyvern/__init__.py
+++ b/skyvern/__init__.py
@@ -5,6 +5,12 @@ from skyvern._version import __version__
from skyvern.utils import setup_windows_event_loop_policy
if typing.TYPE_CHECKING:
+ # Expose the server lazy-imports (runtime-gated via __getattr__ below) to type checkers so
+ # `skyvern.SkyvernPage` / `skyvern.cached` resolve to the real symbols. Without this mypy sees
+ # them only as __getattr__'s Any and flakily reports "not valid as a type" / "Any? not
+ # callable" in the generated scripts, depending on incremental-cache state.
+ from skyvern.core.script_generations.skyvern_page import RunContext, SkyvernPage # noqa: E402,F401
+ from skyvern.core.script_generations.workflow_wrappers import cached, workflow # noqa: E402,F401
from skyvern.library import Skyvern # noqa: E402,F401
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index f6b297103..f38e83c71 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -5390,6 +5390,20 @@ class ForgeAgent:
)
complete_action = None
+ if isinstance(complete_action, CompleteAction) and not await app.AGENT_FUNCTION.gate_step_completion(
+ task=task,
+ step=step,
+ task_block=task_block,
+ page=page,
+ browser_state=browser_state,
+ ):
+ LOG.info(
+ "Step completion vetoed by completion gate; continuing with next step",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ )
+ complete_action = None
+
if complete_action is not None:
asyncio.create_task(
self._persist_speculative_metadata_for_discarded_plan(
@@ -6187,7 +6201,28 @@ class ForgeAgent:
task_block=task_block,
)
- if step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal)):
+ goal_achieved = step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal))
+ # A decisive COMPLETE action skips parallel verification, so this is the other place a
+ # completion is accepted — gate it too, or the veto is bypassed whenever the agent emits
+ # complete directly.
+ if (
+ goal_achieved
+ and browser_state is not None
+ and not await app.AGENT_FUNCTION.gate_step_completion(
+ task=task,
+ step=step,
+ task_block=task_block,
+ page=page,
+ browser_state=browser_state,
+ )
+ ):
+ LOG.info(
+ "Decisive step completion vetoed by completion gate; continuing with next step",
+ task_id=task.task_id,
+ step_id=step.step_id,
+ )
+ goal_achieved = False
+ if goal_achieved:
LOG.info(
"Step completed and goal achieved, marking task as completed",
step_order=step.order,
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 93d4d17d0..d7cd6d9d5 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -51,7 +51,7 @@ from skyvern.forge.sdk.services import (
)
from skyvern.forge.sdk.services.credentials import AuthenticatorTotpParseResult
from skyvern.forge.sdk.trace import traced
-from skyvern.forge.sdk.workflow.models.block import BlockTypeVar
+from skyvern.forge.sdk.workflow.models.block import BaseTaskBlock, BlockTypeVar
from skyvern.schemas.workflows import BlockResult, FileStorageType, FileUploadDestination
from skyvern.services.otp_gmail import GmailOTPVerificationContext
from skyvern.webeye.actions.actions import Action
@@ -1126,6 +1126,26 @@ class AgentFunction:
if step.status == StepStatus.completed:
await self._maybe_close_magic_link_page(task)
+ async def gate_step_completion(
+ self,
+ *,
+ task: Task,
+ step: Step,
+ task_block: BaseTaskBlock | None,
+ page: Page | None,
+ browser_state: BrowserState,
+ ) -> bool:
+ """Gate whether an agent completion verdict for this step is accepted.
+
+ Called once the parallel verifier has produced a CompleteAction, before the step
+ and task are marked completed. Return True to accept; False to reject, which makes
+ the agent keep going and fail safe at max steps rather than falsely completing.
+ OSS accepts everything; a deployment may override to hold specific blocks (e.g. a
+ submit block whose AI fallback would otherwise complete without a deterministic
+ confirmation check) to a stricter gate.
+ """
+ return True
+
async def _maybe_close_magic_link_page(self, task: Task) -> None:
"""Close a magic-link confirmation page if it shows close/return signals.
diff --git a/tests/unit/test_completion_gate.py b/tests/unit/test_completion_gate.py
new file mode 100644
index 000000000..f26a87914
--- /dev/null
+++ b/tests/unit/test_completion_gate.py
@@ -0,0 +1,163 @@
+"""The completion gate (AGENT_FUNCTION.gate_step_completion) can veto an agent completion.
+
+A vetoed CompleteAction must not mark the task completed; the agent continues (and fails safe
+at max steps) instead of falsely completing. See SKY-12992.
+"""
+
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.schemas.runs import RunEngine
+from skyvern.schemas.steps import AgentStepOutput
+from skyvern.webeye.actions.actions import CompleteAction
+from skyvern.webeye.actions.responses import ActionSuccess
+from tests.unit.helpers import (
+ make_browser_state,
+ make_organization,
+ make_step,
+ make_task,
+ setup_parallel_verification_mocks,
+)
+
+
+async def test_completion_gate_veto_does_not_complete_task(monkeypatch: pytest.MonkeyPatch) -> None:
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization)
+
+ step = make_step(
+ now,
+ task,
+ step_id="step-123",
+ status=StepStatus.completed,
+ order=0,
+ output=AgentStepOutput(action_results=[], actions_and_results=[]),
+ )
+ next_step = make_step(now, task, step_id="step-next", status=StepStatus.created, order=1, output=None)
+
+ mocks = setup_parallel_verification_mocks(
+ agent,
+ step=step,
+ task=task,
+ monkeypatch=monkeypatch,
+ next_step=next_step,
+ complete_action=CompleteAction(reasoning="done", verified=True),
+ handle_action_responses=[[ActionSuccess()]],
+ )
+
+ # Veto the completion.
+ gate = AsyncMock(return_value=False)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)
+
+ browser_state, scraped_page, page = make_browser_state()
+ completed, _last_step, next_created_step = await agent._handle_completed_step_with_parallel_verification(
+ organization=organization,
+ task=task,
+ step=step,
+ page=page,
+ browser_state=browser_state,
+ scraped_page=scraped_page,
+ engine=RunEngine.skyvern_v1,
+ )
+
+ assert gate.await_count == 1
+ assert completed is not True
+ assert next_created_step is not None # loop continues with another step
+ completed_calls = [c for c in mocks.update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]
+ assert completed_calls == []
+
+
+async def test_completion_gate_accept_completes_task(monkeypatch: pytest.MonkeyPatch) -> None:
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization, navigation_goal=None) # skip data-extraction branch
+
+ step = make_step(
+ now,
+ task,
+ step_id="step-123",
+ status=StepStatus.completed,
+ order=0,
+ output=AgentStepOutput(action_results=[], actions_and_results=[]),
+ )
+
+ mocks = setup_parallel_verification_mocks(
+ agent,
+ step=step,
+ task=task,
+ monkeypatch=monkeypatch,
+ next_step=step,
+ complete_action=CompleteAction(reasoning="done", verified=True),
+ handle_action_responses=[[ActionSuccess()]],
+ )
+
+ gate = AsyncMock(return_value=True)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)
+
+ browser_state, scraped_page, page = make_browser_state()
+ completed, _last_step, _next = await agent._handle_completed_step_with_parallel_verification(
+ organization=organization,
+ task=task,
+ step=step,
+ page=page,
+ browser_state=browser_state,
+ scraped_page=scraped_page,
+ engine=RunEngine.skyvern_v1,
+ )
+
+ assert gate.await_count == 1
+ assert completed is True
+ completed_calls = [c for c in mocks.update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]
+ assert completed_calls != []
+
+
+async def test_decisive_completion_gate_veto_creates_next_step(monkeypatch: pytest.MonkeyPatch) -> None:
+ # A decisive COMPLETE action bypasses parallel verification and completes via
+ # handle_completed_step's is_goal_achieved branch — the gate must fire here too.
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ task = make_task(now, organization)
+
+ complete = CompleteAction(reasoning="done", verified=True)
+ output = AgentStepOutput(action_results=[ActionSuccess()], actions_and_results=[(complete, [ActionSuccess()])])
+ step = make_step(now, task, step_id="step-123", status=StepStatus.completed, order=0, output=output)
+ assert step.is_goal_achieved(has_navigation_goal=bool(task.navigation_goal)) # sanity
+
+ update_task = AsyncMock()
+ monkeypatch.setattr(agent, "update_task", update_task)
+ monkeypatch.setattr(agent, "update_step", AsyncMock(side_effect=lambda s, **k: s))
+ monkeypatch.setattr(agent, "_check_workflow_run_step_budget", AsyncMock(return_value=None))
+ next_step = make_step(now, task, step_id="step-next", status=StepStatus.created, order=1, output=None)
+ monkeypatch.setattr(app.DATABASE.tasks, "create_step", AsyncMock(return_value=next_step))
+
+ gate = AsyncMock(return_value=False)
+ monkeypatch.setattr(app.AGENT_FUNCTION, "gate_step_completion", gate)
+
+ browser_state, scraped_page, page = make_browser_state()
+ completed, _last_step, created_next = await agent.handle_completed_step(
+ organization=organization,
+ task=task,
+ step=step,
+ page=page,
+ browser_state=browser_state,
+ scraped_page=scraped_page,
+ engine=RunEngine.skyvern_v1,
+ )
+
+ assert gate.await_count == 1
+ assert completed is None # not completed; loop continues
+ assert created_next is next_step
+ completed_calls = [c for c in update_task.await_args_list if c.kwargs.get("status") == TaskStatus.completed]
+ assert completed_calls == []