Skip to content

Automate browser based workflows with AI

test(SKY-11696): characterization tests for agent_step (pin 12 uncovered behaviors) (#7352)

Aaron Perez0a03e62ad575
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 0161be959..068e2b575 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -3590,144 +3590,13 @@ class WorkflowService:
)
if block.block_type == BlockType.LOGIN:
- await self._apply_login_block_credential_proxy_pin(
+ workflow_run = await self._prepare_login_block_browser_profile(
block=block,
workflow_run=workflow_run,
workflow_run_id=workflow_run_id,
organization_id=organization_id,
+ browser_session_id=browser_session_id,
)
- resolved_browser_profile_id = await self._resolve_login_block_browser_profile_id(
- block=block,
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- workflow_permanent_id=workflow_run.workflow_permanent_id,
- )
- # Save the original navigation goal before any mutation so
- # retries don't stack the browser-session prefix repeatedly.
- original_navigation_goal = block.navigation_goal
- if resolved_browser_profile_id:
- decision = await self._evaluate_debug_session_profile_decision(
- workflow_run=workflow_run,
- browser_session_id=browser_session_id,
- resolved_browser_profile_id=resolved_browser_profile_id,
- organization_id=organization_id,
- )
-
- if decision.incompatible_reason is not None:
- # Debug session whose visible PBS is profile-incompatible.
- # Stream fidelity wins: attach the visible PBS so the user
- # can watch the action, but DO NOT write
- # workflow_run.browser_profile_id, DO NOT rewrite the
- # navigation_goal, and DO NOT emit "skipping login agent".
- # Fall through to ordinary LoginBlock execution below.
- LOG.warning(
- "Debug session profile incompatible with LoginBlock credential",
- code=DEBUG_SESSION_PROFILE_INCOMPATIBLE_CODE,
- reason=decision.incompatible_reason,
- workflow_run_id=workflow_run_id,
- block_label=block.label,
- debug_session_id=workflow_run.debug_session_id,
- browser_session_id=browser_session_id,
- credential_browser_profile_id=resolved_browser_profile_id,
- )
- if decision.attach_browser_session_id and block.url:
- try:
- await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
- workflow_run=workflow_run,
- url=block.url,
- browser_session_id=decision.attach_browser_session_id,
- )
- except Exception:
- LOG.warning(
- "PBS attach failed for debug incompatible profile; continuing",
- workflow_run_id=workflow_run_id,
- block_label=block.label,
- browser_session_id=decision.attach_browser_session_id,
- exc_info=True,
- )
- else:
- LOG.info(
- "LoginBlock has credential with browser profile — skipping login agent",
- workflow_run_id=workflow_run_id,
- block_label=block.label,
- browser_profile_id=resolved_browser_profile_id,
- url=block.url,
- )
- # Persist the browser_profile_id on the workflow_run so
- # subsequent blocks create / reuse a browser with the
- # saved profile (cookies, localStorage, etc.).
- await app.DATABASE.workflow_runs.update_workflow_run(
- workflow_run_id=workflow_run_id,
- browser_profile_id=resolved_browser_profile_id,
- )
- workflow_run = (
- await app.DATABASE.workflow_runs.get_workflow_run(
- workflow_run_id=workflow_run_id,
- organization_id=organization_id,
- )
- or workflow_run
- )
-
- # Create the browser with the saved profile and navigate
- # to the login block's URL. When a saved-profile credential
- # is selected, the user is guided to enter the post-login
- # target URL (e.g. homepage/dashboard) rather than the
- # login page. The saved cookies will authenticate the
- # session once the page loads.
- profile_loaded = bool(block.url)
- if block.url:
- try:
- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
- workflow_run=workflow_run,
- url=block.url,
- browser_profile_id=resolved_browser_profile_id,
- browser_session_id=decision.attach_browser_session_id,
- )
- working_page = await browser_state.get_working_page()
- if working_page and working_page.url == "about:blank":
- await browser_state.navigate_to_url(page=working_page, url=block.url)
- # Wait for the page to settle so cookies/redirects complete
- if working_page:
- try:
- await working_page.wait_for_load_state("networkidle", timeout=10000)
- except Exception:
- LOG.debug(
- "networkidle timeout after browser profile navigation (non-fatal)",
- workflow_run_id=workflow_run_id,
- )
- except Exception:
- LOG.warning(
- "Saved browser profile failed to load, falling back to normal login",
- workflow_run_id=workflow_run_id,
- block_label=block.label,
- browser_profile_id=resolved_browser_profile_id,
- exc_info=True,
- )
- profile_loaded = False
- # Clear the profile so the normal login path doesn't reuse it
- await app.DATABASE.workflow_runs.update_workflow_run(
- workflow_run_id=workflow_run_id,
- browser_profile_id=None,
- )
-
- if not profile_loaded:
- # Fall through to normal block execution below
- pass
- else:
- # Browser profile loaded — the session may still be
- # valid or may have expired (common with bank sites).
- # Instead of skipping the login block, modify the
- # navigation goal so the AI checks whether the user is
- # already logged in and only performs login if needed.
- if original_navigation_goal:
- block.navigation_goal = (
- "A saved browser session has been loaded. "
- "Check if the user is already logged in. "
- "If already logged in, complete this task immediately without taking any action. "
- "If not logged in (e.g. the session expired), "
- "proceed to log in with the provided credentials.\n\n"
- f"Original goal: {original_navigation_goal}"
- )
valid_to_run_code = (
is_script_run and block.label and block.label in script_blocks_by_label and not block.disable_cache
@@ -4259,6 +4128,155 @@ class WorkflowService:
)
return workflow_run, blocks_to_update, workflow_run_block_result, True, branch_metadata
+ async def _prepare_login_block_browser_profile(
+ self,
+ *,
+ block: BlockTypeVar,
+ workflow_run: WorkflowRun,
+ workflow_run_id: str,
+ organization_id: str,
+ browser_session_id: str | None,
+ ) -> WorkflowRun:
+ await self._apply_login_block_credential_proxy_pin(
+ block=block,
+ workflow_run=workflow_run,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ resolved_browser_profile_id = await self._resolve_login_block_browser_profile_id(
+ block=block,
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ workflow_permanent_id=workflow_run.workflow_permanent_id,
+ )
+ # Save the original navigation goal before any mutation so
+ # retries don't stack the browser-session prefix repeatedly.
+ original_navigation_goal = block.navigation_goal
+ if resolved_browser_profile_id:
+ decision = await self._evaluate_debug_session_profile_decision(
+ workflow_run=workflow_run,
+ browser_session_id=browser_session_id,
+ resolved_browser_profile_id=resolved_browser_profile_id,
+ organization_id=organization_id,
+ )
+
+ if decision.incompatible_reason is not None:
+ # Debug session whose visible PBS is profile-incompatible.
+ # Stream fidelity wins: attach the visible PBS so the user
+ # can watch the action, but DO NOT write
+ # workflow_run.browser_profile_id, DO NOT rewrite the
+ # navigation_goal, and DO NOT emit "skipping login agent".
+ # Fall through to ordinary LoginBlock execution below.
+ LOG.warning(
+ "Debug session profile incompatible with LoginBlock credential",
+ code=DEBUG_SESSION_PROFILE_INCOMPATIBLE_CODE,
+ reason=decision.incompatible_reason,
+ workflow_run_id=workflow_run_id,
+ block_label=block.label,
+ debug_session_id=workflow_run.debug_session_id,
+ browser_session_id=browser_session_id,
+ credential_browser_profile_id=resolved_browser_profile_id,
+ )
+ if decision.attach_browser_session_id and block.url:
+ try:
+ await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
+ workflow_run=workflow_run,
+ url=block.url,
+ browser_session_id=decision.attach_browser_session_id,
+ )
+ except Exception:
+ LOG.warning(
+ "PBS attach failed for debug incompatible profile; continuing",
+ workflow_run_id=workflow_run_id,
+ block_label=block.label,
+ browser_session_id=decision.attach_browser_session_id,
+ exc_info=True,
+ )
+ else:
+ LOG.info(
+ "LoginBlock has credential with browser profile — skipping login agent",
+ workflow_run_id=workflow_run_id,
+ block_label=block.label,
+ browser_profile_id=resolved_browser_profile_id,
+ url=block.url,
+ )
+ # Persist the browser_profile_id on the workflow_run so
+ # subsequent blocks create / reuse a browser with the
+ # saved profile (cookies, localStorage, etc.).
+ await app.DATABASE.workflow_runs.update_workflow_run(
+ workflow_run_id=workflow_run_id,
+ browser_profile_id=resolved_browser_profile_id,
+ )
+ workflow_run = (
+ await app.DATABASE.workflow_runs.get_workflow_run(
+ workflow_run_id=workflow_run_id,
+ organization_id=organization_id,
+ )
+ or workflow_run
+ )
+
+ # Create the browser with the saved profile and navigate
+ # to the login block's URL. When a saved-profile credential
+ # is selected, the user is guided to enter the post-login
+ # target URL (e.g. homepage/dashboard) rather than the
+ # login page. The saved cookies will authenticate the
+ # session once the page loads.
+ profile_loaded = bool(block.url)
+ if block.url:
+ try:
+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(
+ workflow_run=workflow_run,
+ url=block.url,
+ browser_profile_id=resolved_browser_profile_id,
+ browser_session_id=decision.attach_browser_session_id,
+ )
+ working_page = await browser_state.get_working_page()
+ if working_page and working_page.url == "about:blank":
+ await browser_state.navigate_to_url(page=working_page, url=block.url)
+ # Wait for the page to settle so cookies/redirects complete
+ if working_page:
+ try:
+ await working_page.wait_for_load_state("networkidle", timeout=10000)
+ except Exception:
+ LOG.debug(
+ "networkidle timeout after browser profile navigation (non-fatal)",
+ workflow_run_id=workflow_run_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Saved browser profile failed to load, falling back to normal login",
+ workflow_run_id=workflow_run_id,
+ block_label=block.label,
+ browser_profile_id=resolved_browser_profile_id,
+ exc_info=True,
+ )
+ profile_loaded = False
+ # Clear the profile so the normal login path doesn't reuse it
+ await app.DATABASE.workflow_runs.update_workflow_run(
+ workflow_run_id=workflow_run_id,
+ browser_profile_id=None,
+ )
+
+ if not profile_loaded:
+ # Fall through to normal block execution below
+ pass
+ else:
+ # Browser profile loaded — the session may still be
+ # valid or may have expired (common with bank sites).
+ # Instead of skipping the login block, modify the
+ # navigation goal so the AI checks whether the user is
+ # already logged in and only performs login if needed.
+ if original_navigation_goal:
+ block.navigation_goal = (
+ "A saved browser session has been loaded. "
+ "Check if the user is already logged in. "
+ "If already logged in, complete this task immediately without taking any action. "
+ "If not logged in (e.g. the session expired), "
+ "proceed to log in with the provided credentials.\n\n"
+ f"Original goal: {original_navigation_goal}"
+ )
+ return workflow_run
+
async def resolve_login_block_browser_profile_id_pre_run(
self,
*,
diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.py
new file mode 100644
index 000000000..cad0ad6d7
--- /dev/null
+++ b/tests/unit/test_agent_step_characterization.py
@@ -0,0 +1,373 @@
+from __future__ import annotations
+
+import base64
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock
+from zoneinfo import ZoneInfo
+
+import pytest
+
+from skyvern.exceptions import NoTOTPVerificationCodeFound
+from skyvern.forge.agent import ForgeAgent, StepPromptResult
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.forge.sdk.models import Step, StepStatus
+from skyvern.forge.sdk.schemas.organizations import Organization
+from skyvern.forge.sdk.schemas.tasks import Task
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import (
+ Action,
+ ClickAction,
+ CompleteAction,
+ DownloadFileAction,
+ ExtractAction,
+ WaitAction,
+)
+from skyvern.webeye.actions.models import DetailedAgentStepOutput
+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
+from skyvern.webeye.scraper.scraped_page import ScrapedPage
+from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task
+
+
+def _click(element_id: str = "node-1") -> ClickAction:
+ return ClickAction(
+ element_id=element_id,
+ organization_id="org-123",
+ workflow_run_id="workflow-1",
+ task_id="task-123",
+ step_id="step-char",
+ step_order=0,
+ action_order=0,
+ )
+
+
+@dataclass
+class AgentStepRig:
+ agent: ForgeAgent
+ organization: Organization
+ task: Task
+ step: Step
+ browser_state: MagicMock
+ scraped_page: ScrapedPage
+ context: SkyvernContext
+ llm_handler: AsyncMock
+ action_handler: AsyncMock
+ update_statuses: list[StepStatus | None] = field(default_factory=list)
+
+ async def run(self) -> tuple[Step, DetailedAgentStepOutput]:
+ skyvern_context.set(self.context)
+ try:
+ return await self.agent.agent_step(
+ task=self.task,
+ step=self.step,
+ browser_state=self.browser_state,
+ organization=self.organization,
+ )
+ finally:
+ skyvern_context.reset()
+
+
+def make_agent_step_rig(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ parsed_actions: list[Action] | None = None,
+ action_handler: AsyncMock | None = None,
+ injected_actions: list[Action] | None = None,
+ task_overrides: dict[str, Any] | None = None,
+ disable_user_goal_check: bool = True,
+) -> AgentStepRig:
+ agent = ForgeAgent()
+ now = datetime.now(UTC)
+ organization = make_organization(now)
+ overrides: dict[str, Any] = {"navigation_goal": "Reach confirmation page", "workflow_run_id": "workflow-1"}
+ overrides.update(task_overrides or {})
+ task = make_task(now, organization, **overrides)
+ step = make_step(now, task, step_id="step-char", status=StepStatus.created, order=0, output=None)
+
+ browser_state, _, page = make_browser_state()
+ browser_state.must_get_working_page = AsyncMock(return_value=page)
+ browser_state.get_working_page = AsyncMock(return_value=page)
+ browser_state.reload_page = AsyncMock()
+
+ async def _dummy_cleanup(*_args, **_kwargs) -> list[dict]:
+ return []
+
+ scraped_page = ScrapedPage(
+ elements=[],
+ element_tree=[],
+ element_tree_trimmed=[],
+ _browser_state=browser_state,
+ _clean_up_func=_dummy_cleanup,
+ _scrape_exclude=None,
+ )
+ scraped_page.screenshots = [b"image"]
+
+ agent.build_and_record_step_prompt = AsyncMock(
+ return_value=StepPromptResult(
+ scraped_page=scraped_page,
+ extract_action_prompt="prompt",
+ use_caching=False,
+ prompt_name="extract-actions",
+ without_page_information=False,
+ )
+ )
+ json_response: dict[str, object] = {"actions": [{"action_type": "CLICK", "element_id": "node-1"}]}
+ agent.handle_potential_OTP_actions = AsyncMock(return_value=(json_response, []))
+
+ actions = parsed_actions if parsed_actions is not None else [_click()]
+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: actions)
+
+ if action_handler is None:
+ action_handler = AsyncMock(return_value=[ActionSuccess()])
+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", action_handler)
+ agent.record_artifacts_after_action = AsyncMock()
+ agent._is_multi_field_totp_sequence = MagicMock(return_value=False)
+ agent.check_user_goal_complete = AsyncMock()
+
+ llm_handler = AsyncMock(return_value=json_response)
+ monkeypatch.setattr(
+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",
+ lambda *_args, **_kwargs: llm_handler,
+ )
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution",
+ AsyncMock(return_value=injected_actions),
+ )
+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())
+ monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))
+ monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)
+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.workflow_params.create_action", AsyncMock())
+ # Wait-time optimization is a cloud experiment (OSS/killswitch-off returns None).
+ # Pin that here so the rig never routes into the half-mocked experiment provider,
+ # which would cache a malformed WaitConfig in a module-global keyed by task_id and
+ # leak "coroutine never awaited" warnings / cross-test state.
+ monkeypatch.setattr("skyvern.forge.agent.get_or_create_wait_config", AsyncMock(return_value=None))
+
+ async def _flag(flag_name: str, *_args, **_kwargs) -> bool:
+ if flag_name == "DISABLE_USER_GOAL_CHECK":
+ return disable_user_goal_check
+ return False
+
+ monkeypatch.setattr(
+ "skyvern.forge.agent.app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached",
+ AsyncMock(side_effect=_flag),
+ )
+
+ update_statuses: list[StepStatus | None] = []
+
+ async def fake_update_step(
+ step: Step,
+ status: StepStatus | None = None,
+ output=None,
+ is_last: bool | None = None,
+ retry_index: int | None = None,
+ **_kwargs,
+ ) -> Step:
+ update_statuses.append(status)
+ if status is not None:
+ step.status = status
+ if output is not None:
+ step.output = output
+ return step
+
+ agent.update_step = AsyncMock(side_effect=fake_update_step)
+
+ context = SkyvernContext(
+ task_id=task.task_id,
+ step_id=None,
+ organization_id=task.organization_id,
+ workflow_run_id=task.workflow_run_id,
+ tz_info=ZoneInfo("UTC"),
+ )
+ return AgentStepRig(
+ agent=agent,
+ organization=organization,
+ task=task,
+ step=step,
+ browser_state=browser_state,
+ scraped_page=scraped_page,
+ context=context,
+ llm_handler=llm_handler,
+ action_handler=action_handler,
+ update_statuses=update_statuses,
+ )
+
+
+async def test_injected_actions_from_prepare_step_execution_skip_llm(monkeypatch: pytest.MonkeyPatch) -> None:
+ injected = _click()
+ rig = make_agent_step_rig(monkeypatch, injected_actions=[injected])
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert rig.llm_handler.await_count == 0
+ assert rig.action_handler.await_count == 1
+ assert rig.action_handler.await_args.kwargs["action"] is injected
+ assert output.actions == [injected]
+
+
+async def test_no_generated_actions_marks_step_failed(monkeypatch: pytest.MonkeyPatch) -> None:
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[])
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.failed
+ assert rig.action_handler.await_count == 0
+ assert output.actions == []
+
+
+async def test_totp_polling_timeout_produces_terminate_action(monkeypatch: pytest.MonkeyPatch) -> None:
+ rig = make_agent_step_rig(monkeypatch, task_overrides={"totp_identifier": "[email protected]"})
+ rig.agent.handle_potential_OTP_actions = AsyncMock(side_effect=NoTOTPVerificationCodeFound(task_id="task-123"))
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert output.actions is not None
+ assert output.actions[0].action_type == ActionType.TERMINATE
+ assert "[email protected]" in output.actions[0].reasoning
+
+
+async def test_pdf_viewer_embed_generates_download_action(monkeypatch: pytest.MonkeyPatch) -> None:
+ pdf_bytes = b"%PDF-1.4 characterization"
+ pdf_src = "data:application/pdf;base64," + base64.b64encode(pdf_bytes).decode()
+ rig = make_agent_step_rig(monkeypatch)
+ monkeypatch.setattr(ScrapedPage, "check_pdf_viewer_embed", lambda self: pdf_src)
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert output.actions is not None
+ action = output.actions[0]
+ assert isinstance(action, DownloadFileAction)
+ assert action.byte == pdf_bytes
+ assert action.download is True
+ assert len(rig.context.downloaded_pdf_sources) == 1
+
+
+async def test_wait_actions_skipped_when_batched_with_other_actions(monkeypatch: pytest.MonkeyPatch) -> None:
+ wait = WaitAction(seconds=3)
+ click = _click()
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[wait, click])
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert rig.action_handler.await_count == 1
+ assert output.actions_and_results is not None
+ assert [action for action, _ in output.actions_and_results] == [click]
+
+
+async def test_failed_action_marks_step_failed_and_skips_remaining(monkeypatch: pytest.MonkeyPatch) -> None:
+ first, second = _click("node-1"), _click("node-2")
+ # The stop-the-batch decision is driven by the RESULT's stop_execution_on_failure
+ # (default True), not by the action. Set it explicitly to pin the flag-driven path.
+ handler = AsyncMock(return_value=[ActionFailure(Exception("element vanished"), stop_execution_on_failure=True)])
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.failed
+ assert handler.await_count == 1
+ # get_clean_detailed_output strips the (second, []) placeholder: only executed actions survive.
+ assert output.actions_and_results is not None
+ assert len(output.actions_and_results) == 1
+ assert output.actions_and_results[0][0] is first
+ assert output.actions_and_results[0][1][0].success is False
+
+
+async def test_failed_action_with_continue_flag_executes_remaining(monkeypatch: pytest.MonkeyPatch) -> None:
+ # Mirror of the skip case: a failure result that opts out of stopping the batch
+ # (stop_execution_on_failure=False) lets the loop run every action.
+ first, second = _click("node-1"), _click("node-2")
+ handler = AsyncMock(return_value=[ActionFailure(Exception("transient"), stop_execution_on_failure=False)])
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)
+
+ step, output = await rig.run()
+
+ # A tolerated failure (stop_execution_on_failure=False) does not fail the step —
+ # every action runs and the step still completes.
+ assert step.status == StepStatus.completed
+ assert handler.await_count == 2
+ assert output.actions_and_results is not None
+ assert [action for action, _ in output.actions_and_results] == [first, second]
+
+
+async def test_skip_remaining_actions_stops_batch_but_step_completes(monkeypatch: pytest.MonkeyPatch) -> None:
+ first, second = _click("node-1"), _click("node-2")
+ handler = AsyncMock(return_value=[ActionResult(success=True, skip_remaining_actions=True)])
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)
+
+ step, _output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert handler.await_count == 1
+
+
+async def test_refresh_working_page_signal_reloads_and_skips_batch(monkeypatch: pytest.MonkeyPatch) -> None:
+ rig = make_agent_step_rig(monkeypatch)
+ rig.context.refresh_working_page = True
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ rig.browser_state.reload_page.assert_awaited_once()
+ assert rig.action_handler.await_count == 0
+ assert output.actions_and_results is not None
+ assert output.actions_and_results[0][0].action_type == ActionType.RELOAD_PAGE
+ assert rig.context.refresh_working_page is False
+
+
+async def test_unexpected_exception_returns_failed_step_instead_of_raising(monkeypatch: pytest.MonkeyPatch) -> None:
+ handler = AsyncMock(side_effect=RuntimeError("browser exploded"))
+ rig = make_agent_step_rig(monkeypatch, action_handler=handler)
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.failed
+ assert output.step_exception == "RuntimeError"
+
+
+async def test_successful_complete_action_with_extraction_goal_appends_extract(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ complete = CompleteAction(reasoning="goal reached")
+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[complete])
+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(return_value=rig.task))
+ extract = ExtractAction(
+ reasoning="collect",
+ data_extraction_goal=rig.task.data_extraction_goal,
+ data_extraction_schema=None,
+ )
+ rig.agent.create_extract_action = AsyncMock(return_value=extract)
+
+ step, output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ rig.agent.create_extract_action.assert_awaited_once()
+ assert rig.action_handler.await_count == 2
+ assert output.actions_and_results is not None
+ assert output.actions_and_results[-1][0] is extract
+
+
+async def test_parallel_verification_marks_speculative_original_status(monkeypatch: pytest.MonkeyPatch) -> None:
+ rig = make_agent_step_rig(monkeypatch, disable_user_goal_check=False)
+
+ step, _output = await rig.run()
+
+ assert step.status == StepStatus.completed
+ assert step.speculative_original_status == StepStatus.completed