Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
test(SKY-11696): characterization tests for agent_step (pin 12 uncovered behaviors) (#7352)
Aaron Perez0a03e62ad575
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 0161be959..068e2b575 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -3590,144 +3590,13 @@ class WorkflowService:)if block.block_type == BlockType.LOGIN:- await self._apply_login_block_credential_proxy_pin(+ workflow_run = await self._prepare_login_block_browser_profile(block=block,workflow_run=workflow_run,workflow_run_id=workflow_run_id,organization_id=organization_id,+ browser_session_id=browser_session_id,)- resolved_browser_profile_id = await self._resolve_login_block_browser_profile_id(- block=block,- workflow_run_id=workflow_run_id,- organization_id=organization_id,- workflow_permanent_id=workflow_run.workflow_permanent_id,- )- # Save the original navigation goal before any mutation so- # retries don't stack the browser-session prefix repeatedly.- original_navigation_goal = block.navigation_goal- if resolved_browser_profile_id:- decision = await self._evaluate_debug_session_profile_decision(- workflow_run=workflow_run,- browser_session_id=browser_session_id,- resolved_browser_profile_id=resolved_browser_profile_id,- organization_id=organization_id,- )-- if decision.incompatible_reason is not None:- # Debug session whose visible PBS is profile-incompatible.- # Stream fidelity wins: attach the visible PBS so the user- # can watch the action, but DO NOT write- # workflow_run.browser_profile_id, DO NOT rewrite the- # navigation_goal, and DO NOT emit "skipping login agent".- # Fall through to ordinary LoginBlock execution below.- LOG.warning(- "Debug session profile incompatible with LoginBlock credential",- code=DEBUG_SESSION_PROFILE_INCOMPATIBLE_CODE,- reason=decision.incompatible_reason,- workflow_run_id=workflow_run_id,- block_label=block.label,- debug_session_id=workflow_run.debug_session_id,- browser_session_id=browser_session_id,- credential_browser_profile_id=resolved_browser_profile_id,- )- if decision.attach_browser_session_id and block.url:- try:- await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,- url=block.url,- browser_session_id=decision.attach_browser_session_id,- )- except Exception:- LOG.warning(- "PBS attach failed for debug incompatible profile; continuing",- workflow_run_id=workflow_run_id,- block_label=block.label,- browser_session_id=decision.attach_browser_session_id,- exc_info=True,- )- else:- LOG.info(- "LoginBlock has credential with browser profile — skipping login agent",- workflow_run_id=workflow_run_id,- block_label=block.label,- browser_profile_id=resolved_browser_profile_id,- url=block.url,- )- # Persist the browser_profile_id on the workflow_run so- # subsequent blocks create / reuse a browser with the- # saved profile (cookies, localStorage, etc.).- await app.DATABASE.workflow_runs.update_workflow_run(- workflow_run_id=workflow_run_id,- browser_profile_id=resolved_browser_profile_id,- )- workflow_run = (- await app.DATABASE.workflow_runs.get_workflow_run(- workflow_run_id=workflow_run_id,- organization_id=organization_id,- )- or workflow_run- )-- # Create the browser with the saved profile and navigate- # to the login block's URL. When a saved-profile credential- # is selected, the user is guided to enter the post-login- # target URL (e.g. homepage/dashboard) rather than the- # login page. The saved cookies will authenticate the- # session once the page loads.- profile_loaded = bool(block.url)- if block.url:- try:- browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(- workflow_run=workflow_run,- url=block.url,- browser_profile_id=resolved_browser_profile_id,- browser_session_id=decision.attach_browser_session_id,- )- working_page = await browser_state.get_working_page()- if working_page and working_page.url == "about:blank":- await browser_state.navigate_to_url(page=working_page, url=block.url)- # Wait for the page to settle so cookies/redirects complete- if working_page:- try:- await working_page.wait_for_load_state("networkidle", timeout=10000)- except Exception:- LOG.debug(- "networkidle timeout after browser profile navigation (non-fatal)",- workflow_run_id=workflow_run_id,- )- except Exception:- LOG.warning(- "Saved browser profile failed to load, falling back to normal login",- workflow_run_id=workflow_run_id,- block_label=block.label,- browser_profile_id=resolved_browser_profile_id,- exc_info=True,- )- profile_loaded = False- # Clear the profile so the normal login path doesn't reuse it- await app.DATABASE.workflow_runs.update_workflow_run(- workflow_run_id=workflow_run_id,- browser_profile_id=None,- )-- if not profile_loaded:- # Fall through to normal block execution below- pass- else:- # Browser profile loaded — the session may still be- # valid or may have expired (common with bank sites).- # Instead of skipping the login block, modify the- # navigation goal so the AI checks whether the user is- # already logged in and only performs login if needed.- if original_navigation_goal:- block.navigation_goal = (- "A saved browser session has been loaded. "- "Check if the user is already logged in. "- "If already logged in, complete this task immediately without taking any action. "- "If not logged in (e.g. the session expired), "- "proceed to log in with the provided credentials.\n\n"- f"Original goal: {original_navigation_goal}"- )valid_to_run_code = (is_script_run and block.label and block.label in script_blocks_by_label and not block.disable_cache@@ -4259,6 +4128,155 @@ class WorkflowService:)return workflow_run, blocks_to_update, workflow_run_block_result, True, branch_metadata+ async def _prepare_login_block_browser_profile(+ self,+ *,+ block: BlockTypeVar,+ workflow_run: WorkflowRun,+ workflow_run_id: str,+ organization_id: str,+ browser_session_id: str | None,+ ) -> WorkflowRun:+ await self._apply_login_block_credential_proxy_pin(+ block=block,+ workflow_run=workflow_run,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ resolved_browser_profile_id = await self._resolve_login_block_browser_profile_id(+ block=block,+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ workflow_permanent_id=workflow_run.workflow_permanent_id,+ )+ # Save the original navigation goal before any mutation so+ # retries don't stack the browser-session prefix repeatedly.+ original_navigation_goal = block.navigation_goal+ if resolved_browser_profile_id:+ decision = await self._evaluate_debug_session_profile_decision(+ workflow_run=workflow_run,+ browser_session_id=browser_session_id,+ resolved_browser_profile_id=resolved_browser_profile_id,+ organization_id=organization_id,+ )++ if decision.incompatible_reason is not None:+ # Debug session whose visible PBS is profile-incompatible.+ # Stream fidelity wins: attach the visible PBS so the user+ # can watch the action, but DO NOT write+ # workflow_run.browser_profile_id, DO NOT rewrite the+ # navigation_goal, and DO NOT emit "skipping login agent".+ # Fall through to ordinary LoginBlock execution below.+ LOG.warning(+ "Debug session profile incompatible with LoginBlock credential",+ code=DEBUG_SESSION_PROFILE_INCOMPATIBLE_CODE,+ reason=decision.incompatible_reason,+ workflow_run_id=workflow_run_id,+ block_label=block.label,+ debug_session_id=workflow_run.debug_session_id,+ browser_session_id=browser_session_id,+ credential_browser_profile_id=resolved_browser_profile_id,+ )+ if decision.attach_browser_session_id and block.url:+ try:+ await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=block.url,+ browser_session_id=decision.attach_browser_session_id,+ )+ except Exception:+ LOG.warning(+ "PBS attach failed for debug incompatible profile; continuing",+ workflow_run_id=workflow_run_id,+ block_label=block.label,+ browser_session_id=decision.attach_browser_session_id,+ exc_info=True,+ )+ else:+ LOG.info(+ "LoginBlock has credential with browser profile — skipping login agent",+ workflow_run_id=workflow_run_id,+ block_label=block.label,+ browser_profile_id=resolved_browser_profile_id,+ url=block.url,+ )+ # Persist the browser_profile_id on the workflow_run so+ # subsequent blocks create / reuse a browser with the+ # saved profile (cookies, localStorage, etc.).+ await app.DATABASE.workflow_runs.update_workflow_run(+ workflow_run_id=workflow_run_id,+ browser_profile_id=resolved_browser_profile_id,+ )+ workflow_run = (+ await app.DATABASE.workflow_runs.get_workflow_run(+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ )+ or workflow_run+ )++ # Create the browser with the saved profile and navigate+ # to the login block's URL. When a saved-profile credential+ # is selected, the user is guided to enter the post-login+ # target URL (e.g. homepage/dashboard) rather than the+ # login page. The saved cookies will authenticate the+ # session once the page loads.+ profile_loaded = bool(block.url)+ if block.url:+ try:+ browser_state = await app.BROWSER_MANAGER.get_or_create_for_workflow_run(+ workflow_run=workflow_run,+ url=block.url,+ browser_profile_id=resolved_browser_profile_id,+ browser_session_id=decision.attach_browser_session_id,+ )+ working_page = await browser_state.get_working_page()+ if working_page and working_page.url == "about:blank":+ await browser_state.navigate_to_url(page=working_page, url=block.url)+ # Wait for the page to settle so cookies/redirects complete+ if working_page:+ try:+ await working_page.wait_for_load_state("networkidle", timeout=10000)+ except Exception:+ LOG.debug(+ "networkidle timeout after browser profile navigation (non-fatal)",+ workflow_run_id=workflow_run_id,+ )+ except Exception:+ LOG.warning(+ "Saved browser profile failed to load, falling back to normal login",+ workflow_run_id=workflow_run_id,+ block_label=block.label,+ browser_profile_id=resolved_browser_profile_id,+ exc_info=True,+ )+ profile_loaded = False+ # Clear the profile so the normal login path doesn't reuse it+ await app.DATABASE.workflow_runs.update_workflow_run(+ workflow_run_id=workflow_run_id,+ browser_profile_id=None,+ )++ if not profile_loaded:+ # Fall through to normal block execution below+ pass+ else:+ # Browser profile loaded — the session may still be+ # valid or may have expired (common with bank sites).+ # Instead of skipping the login block, modify the+ # navigation goal so the AI checks whether the user is+ # already logged in and only performs login if needed.+ if original_navigation_goal:+ block.navigation_goal = (+ "A saved browser session has been loaded. "+ "Check if the user is already logged in. "+ "If already logged in, complete this task immediately without taking any action. "+ "If not logged in (e.g. the session expired), "+ "proceed to log in with the provided credentials.\n\n"+ f"Original goal: {original_navigation_goal}"+ )+ return workflow_run+async def resolve_login_block_browser_profile_id_pre_run(self,*,diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.pynew file mode 100644index 000000000..cad0ad6d7--- /dev/null+++ b/tests/unit/test_agent_step_characterization.py@@ -0,0 +1,373 @@+from __future__ import annotations++import base64+from dataclasses import dataclass, field+from datetime import UTC, datetime+from typing import Any+from unittest.mock import AsyncMock, MagicMock+from zoneinfo import ZoneInfo++import pytest++from skyvern.exceptions import NoTOTPVerificationCodeFound+from skyvern.forge.agent import ForgeAgent, StepPromptResult+from skyvern.forge.sdk.core import skyvern_context+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext+from skyvern.forge.sdk.models import Step, StepStatus+from skyvern.forge.sdk.schemas.organizations import Organization+from skyvern.forge.sdk.schemas.tasks import Task+from skyvern.webeye.actions.action_types import ActionType+from skyvern.webeye.actions.actions import (+ Action,+ ClickAction,+ CompleteAction,+ DownloadFileAction,+ ExtractAction,+ WaitAction,+)+from skyvern.webeye.actions.models import DetailedAgentStepOutput+from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess+from skyvern.webeye.scraper.scraped_page import ScrapedPage+from tests.unit.helpers import make_browser_state, make_organization, make_step, make_task+++def _click(element_id: str = "node-1") -> ClickAction:+ return ClickAction(+ element_id=element_id,+ organization_id="org-123",+ workflow_run_id="workflow-1",+ task_id="task-123",+ step_id="step-char",+ step_order=0,+ action_order=0,+ )+++@dataclass+class AgentStepRig:+ agent: ForgeAgent+ organization: Organization+ task: Task+ step: Step+ browser_state: MagicMock+ scraped_page: ScrapedPage+ context: SkyvernContext+ llm_handler: AsyncMock+ action_handler: AsyncMock+ update_statuses: list[StepStatus | None] = field(default_factory=list)++ async def run(self) -> tuple[Step, DetailedAgentStepOutput]:+ skyvern_context.set(self.context)+ try:+ return await self.agent.agent_step(+ task=self.task,+ step=self.step,+ browser_state=self.browser_state,+ organization=self.organization,+ )+ finally:+ skyvern_context.reset()+++def make_agent_step_rig(+ monkeypatch: pytest.MonkeyPatch,+ *,+ parsed_actions: list[Action] | None = None,+ action_handler: AsyncMock | None = None,+ injected_actions: list[Action] | None = None,+ task_overrides: dict[str, Any] | None = None,+ disable_user_goal_check: bool = True,+) -> AgentStepRig:+ agent = ForgeAgent()+ now = datetime.now(UTC)+ organization = make_organization(now)+ overrides: dict[str, Any] = {"navigation_goal": "Reach confirmation page", "workflow_run_id": "workflow-1"}+ overrides.update(task_overrides or {})+ task = make_task(now, organization, **overrides)+ step = make_step(now, task, step_id="step-char", status=StepStatus.created, order=0, output=None)++ browser_state, _, page = make_browser_state()+ browser_state.must_get_working_page = AsyncMock(return_value=page)+ browser_state.get_working_page = AsyncMock(return_value=page)+ browser_state.reload_page = AsyncMock()++ async def _dummy_cleanup(*_args, **_kwargs) -> list[dict]:+ return []++ scraped_page = ScrapedPage(+ elements=[],+ element_tree=[],+ element_tree_trimmed=[],+ _browser_state=browser_state,+ _clean_up_func=_dummy_cleanup,+ _scrape_exclude=None,+ )+ scraped_page.screenshots = [b"image"]++ agent.build_and_record_step_prompt = AsyncMock(+ return_value=StepPromptResult(+ scraped_page=scraped_page,+ extract_action_prompt="prompt",+ use_caching=False,+ prompt_name="extract-actions",+ without_page_information=False,+ )+ )+ json_response: dict[str, object] = {"actions": [{"action_type": "CLICK", "element_id": "node-1"}]}+ agent.handle_potential_OTP_actions = AsyncMock(return_value=(json_response, []))++ actions = parsed_actions if parsed_actions is not None else [_click()]+ monkeypatch.setattr("skyvern.forge.agent.parse_actions", lambda *_, **__: actions)++ if action_handler is None:+ action_handler = AsyncMock(return_value=[ActionSuccess()])+ monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", action_handler)+ agent.record_artifacts_after_action = AsyncMock()+ agent._is_multi_field_totp_sequence = MagicMock(return_value=False)+ agent.check_user_goal_complete = AsyncMock()++ llm_handler = AsyncMock(return_value=json_response)+ monkeypatch.setattr(+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",+ lambda *_args, **_kwargs: llm_handler,+ )+ monkeypatch.setattr(+ "skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution",+ AsyncMock(return_value=injected_actions),+ )+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())+ monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))+ monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.workflow_params.create_action", AsyncMock())+ # Wait-time optimization is a cloud experiment (OSS/killswitch-off returns None).+ # Pin that here so the rig never routes into the half-mocked experiment provider,+ # which would cache a malformed WaitConfig in a module-global keyed by task_id and+ # leak "coroutine never awaited" warnings / cross-test state.+ monkeypatch.setattr("skyvern.forge.agent.get_or_create_wait_config", AsyncMock(return_value=None))++ async def _flag(flag_name: str, *_args, **_kwargs) -> bool:+ if flag_name == "DISABLE_USER_GOAL_CHECK":+ return disable_user_goal_check+ return False++ monkeypatch.setattr(+ "skyvern.forge.agent.app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached",+ AsyncMock(side_effect=_flag),+ )++ update_statuses: list[StepStatus | None] = []++ async def fake_update_step(+ step: Step,+ status: StepStatus | None = None,+ output=None,+ is_last: bool | None = None,+ retry_index: int | None = None,+ **_kwargs,+ ) -> Step:+ update_statuses.append(status)+ if status is not None:+ step.status = status+ if output is not None:+ step.output = output+ return step++ agent.update_step = AsyncMock(side_effect=fake_update_step)++ context = SkyvernContext(+ task_id=task.task_id,+ step_id=None,+ organization_id=task.organization_id,+ workflow_run_id=task.workflow_run_id,+ tz_info=ZoneInfo("UTC"),+ )+ return AgentStepRig(+ agent=agent,+ organization=organization,+ task=task,+ step=step,+ browser_state=browser_state,+ scraped_page=scraped_page,+ context=context,+ llm_handler=llm_handler,+ action_handler=action_handler,+ update_statuses=update_statuses,+ )+++async def test_injected_actions_from_prepare_step_execution_skip_llm(monkeypatch: pytest.MonkeyPatch) -> None:+ injected = _click()+ rig = make_agent_step_rig(monkeypatch, injected_actions=[injected])++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ assert rig.llm_handler.await_count == 0+ assert rig.action_handler.await_count == 1+ assert rig.action_handler.await_args.kwargs["action"] is injected+ assert output.actions == [injected]+++async def test_no_generated_actions_marks_step_failed(monkeypatch: pytest.MonkeyPatch) -> None:+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[])++ step, output = await rig.run()++ assert step.status == StepStatus.failed+ assert rig.action_handler.await_count == 0+ assert output.actions == []+++async def test_totp_polling_timeout_produces_terminate_action(monkeypatch: pytest.MonkeyPatch) -> None:+ rig = make_agent_step_rig(monkeypatch, task_overrides={"totp_identifier": "[email protected]"})+ rig.agent.handle_potential_OTP_actions = AsyncMock(side_effect=NoTOTPVerificationCodeFound(task_id="task-123"))++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ assert output.actions is not None+ assert output.actions[0].action_type == ActionType.TERMINATE+ assert "[email protected]" in output.actions[0].reasoning+++async def test_pdf_viewer_embed_generates_download_action(monkeypatch: pytest.MonkeyPatch) -> None:+ pdf_bytes = b"%PDF-1.4 characterization"+ pdf_src = "data:application/pdf;base64," + base64.b64encode(pdf_bytes).decode()+ rig = make_agent_step_rig(monkeypatch)+ monkeypatch.setattr(ScrapedPage, "check_pdf_viewer_embed", lambda self: pdf_src)++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ assert output.actions is not None+ action = output.actions[0]+ assert isinstance(action, DownloadFileAction)+ assert action.byte == pdf_bytes+ assert action.download is True+ assert len(rig.context.downloaded_pdf_sources) == 1+++async def test_wait_actions_skipped_when_batched_with_other_actions(monkeypatch: pytest.MonkeyPatch) -> None:+ wait = WaitAction(seconds=3)+ click = _click()+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[wait, click])++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ assert rig.action_handler.await_count == 1+ assert output.actions_and_results is not None+ assert [action for action, _ in output.actions_and_results] == [click]+++async def test_failed_action_marks_step_failed_and_skips_remaining(monkeypatch: pytest.MonkeyPatch) -> None:+ first, second = _click("node-1"), _click("node-2")+ # The stop-the-batch decision is driven by the RESULT's stop_execution_on_failure+ # (default True), not by the action. Set it explicitly to pin the flag-driven path.+ handler = AsyncMock(return_value=[ActionFailure(Exception("element vanished"), stop_execution_on_failure=True)])+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)++ step, output = await rig.run()++ assert step.status == StepStatus.failed+ assert handler.await_count == 1+ # get_clean_detailed_output strips the (second, []) placeholder: only executed actions survive.+ assert output.actions_and_results is not None+ assert len(output.actions_and_results) == 1+ assert output.actions_and_results[0][0] is first+ assert output.actions_and_results[0][1][0].success is False+++async def test_failed_action_with_continue_flag_executes_remaining(monkeypatch: pytest.MonkeyPatch) -> None:+ # Mirror of the skip case: a failure result that opts out of stopping the batch+ # (stop_execution_on_failure=False) lets the loop run every action.+ first, second = _click("node-1"), _click("node-2")+ handler = AsyncMock(return_value=[ActionFailure(Exception("transient"), stop_execution_on_failure=False)])+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)++ step, output = await rig.run()++ # A tolerated failure (stop_execution_on_failure=False) does not fail the step —+ # every action runs and the step still completes.+ assert step.status == StepStatus.completed+ assert handler.await_count == 2+ assert output.actions_and_results is not None+ assert [action for action, _ in output.actions_and_results] == [first, second]+++async def test_skip_remaining_actions_stops_batch_but_step_completes(monkeypatch: pytest.MonkeyPatch) -> None:+ first, second = _click("node-1"), _click("node-2")+ handler = AsyncMock(return_value=[ActionResult(success=True, skip_remaining_actions=True)])+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[first, second], action_handler=handler)++ step, _output = await rig.run()++ assert step.status == StepStatus.completed+ assert handler.await_count == 1+++async def test_refresh_working_page_signal_reloads_and_skips_batch(monkeypatch: pytest.MonkeyPatch) -> None:+ rig = make_agent_step_rig(monkeypatch)+ rig.context.refresh_working_page = True++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ rig.browser_state.reload_page.assert_awaited_once()+ assert rig.action_handler.await_count == 0+ assert output.actions_and_results is not None+ assert output.actions_and_results[0][0].action_type == ActionType.RELOAD_PAGE+ assert rig.context.refresh_working_page is False+++async def test_unexpected_exception_returns_failed_step_instead_of_raising(monkeypatch: pytest.MonkeyPatch) -> None:+ handler = AsyncMock(side_effect=RuntimeError("browser exploded"))+ rig = make_agent_step_rig(monkeypatch, action_handler=handler)++ step, output = await rig.run()++ assert step.status == StepStatus.failed+ assert output.step_exception == "RuntimeError"+++async def test_successful_complete_action_with_extraction_goal_appends_extract(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ complete = CompleteAction(reasoning="goal reached")+ rig = make_agent_step_rig(monkeypatch, parsed_actions=[complete])+ monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task", AsyncMock(return_value=rig.task))+ extract = ExtractAction(+ reasoning="collect",+ data_extraction_goal=rig.task.data_extraction_goal,+ data_extraction_schema=None,+ )+ rig.agent.create_extract_action = AsyncMock(return_value=extract)++ step, output = await rig.run()++ assert step.status == StepStatus.completed+ rig.agent.create_extract_action.assert_awaited_once()+ assert rig.action_handler.await_count == 2+ assert output.actions_and_results is not None+ assert output.actions_and_results[-1][0] is extract+++async def test_parallel_verification_marks_speculative_original_status(monkeypatch: pytest.MonkeyPatch) -> None:+ rig = make_agent_step_rig(monkeypatch, disable_user_goal_check=False)++ step, _output = await rig.run()++ assert step.status == StepStatus.completed+ assert step.speculative_original_status == StepStatus.completed