Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-9553): retry OTP polling for full wall-clock timeout (#5878)
Aaron Perez2d5e5e4b727f
diff --git a/skyvern/config.py b/skyvern/config.pyindex 8f11297b1..abf3d6801 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -98,9 +98,9 @@ class Settings(BaseSettings):DATABASE_REPLICA_STRING: str | None = NoneDATABASE_STATEMENT_TIMEOUT_MS: int = 60000DISABLE_CONNECTION_POOL: bool = False- DATABASE_POOL_SIZE: int = 5- DATABASE_POOL_MAX_OVERFLOW: int = 10- DATABASE_POOL_TIMEOUT: int = 30+ DATABASE_POOL_SIZE: int = 20+ DATABASE_POOL_MAX_OVERFLOW: int = 20+ DATABASE_POOL_TIMEOUT: int = 10DATABASE_POOL_RECYCLE: int = 1800PROMPT_ACTION_HISTORY_WINDOW: int = 1TASK_RESPONSE_ACTION_SCREENSHOT_COUNT: int = 3@@ -425,7 +425,6 @@ class Settings(BaseSettings):TOTP_LIFESPAN_MINUTES: int = 10VERIFICATION_CODE_INITIAL_WAIT_TIME_SECS: int = 40VERIFICATION_CODE_POLLING_TIMEOUT_MINS: int = 15- VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES: int = 3# Bitwarden SettingsBITWARDEN_CLIENT_ID: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex 6d93d2f78..e012079a7 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -786,210 +786,227 @@ class WorkflowService::param max_steps_override: The max steps override for the workflow run, if any.:return: The created workflow run."""- # Validate the workflow and the organization- workflow = await self.get_workflow_by_permanent_id(- workflow_permanent_id=workflow_permanent_id,- organization_id=None if is_template_workflow else organization.organization_id,- version=version,- )- if workflow is None:- LOG.error(f"Workflow {workflow_permanent_id} not found", workflow_version=version)- raise WorkflowNotFound(workflow_permanent_id=workflow_permanent_id, version=version)- workflow_id = workflow.workflow_id- if workflow_request.proxy_location is None and workflow.proxy_location is not None:- workflow_request.proxy_location = workflow.proxy_location- if workflow_request.webhook_callback_url is None and workflow.webhook_callback_url is not None:- workflow_request.webhook_callback_url = workflow.webhook_callback_url- if workflow_request.extra_http_headers is None and workflow.extra_http_headers is not None:- workflow_request.extra_http_headers = workflow.extra_http_headers- if workflow_request.run_with is None:- workflow_request.run_with = workflow.run_with-- # Force ai_fallback=True for adaptive caching (code_version >= 2) runs.- # Adaptive caching requires AI fallback to self-heal when cached scripts break.- # Without this, a caller sending ai_fallback=false would silently disable recovery.- effective_code_version = (- workflow.code_version if workflow.code_version is not None else (2 if workflow.adaptive_caching else None)- )- if (effective_code_version or 0) >= 2 and (workflow_request.run_with == "code"):- if workflow_request.ai_fallback is False:- LOG.info(- "Overriding ai_fallback to True for adaptive caching run",- workflow_permanent_id=workflow_permanent_id,- request_run_with=workflow_request.run_with,- workflow_code_version=workflow.code_version,- )- workflow_request.ai_fallback = True-- # Inherit from ambient context so descendant runs (TriggerWorkflowBlock children)- # carry the parent's chat id forward without per-call plumbing. Resolved here so- # the same value reaches both the DB row and the new SkyvernContext below.- ambient_context: skyvern_context.SkyvernContext | None = skyvern_context.current()- resolved_copilot_session_id = (- copilot_session_id- if copilot_session_id is not None- else (ambient_context.copilot_session_id if ambient_context else None)- )+ async with app.DATABASE.workflow_runs.Session() as outer_session:+ # Validate the workflow and the organization+ workflow = await self.get_workflow_by_permanent_id(+ workflow_permanent_id=workflow_permanent_id,+ organization_id=None if is_template_workflow else organization.organization_id,+ version=version,+ )+ if workflow is None:+ LOG.error(f"Workflow {workflow_permanent_id} not found", workflow_version=version)+ raise WorkflowNotFound(workflow_permanent_id=workflow_permanent_id, version=version)+ workflow_id = workflow.workflow_id+ if workflow_request.proxy_location is None and workflow.proxy_location is not None:+ workflow_request.proxy_location = workflow.proxy_location+ if workflow_request.webhook_callback_url is None and workflow.webhook_callback_url is not None:+ workflow_request.webhook_callback_url = workflow.webhook_callback_url+ if workflow_request.extra_http_headers is None and workflow.extra_http_headers is not None:+ workflow_request.extra_http_headers = workflow.extra_http_headers+ if workflow_request.run_with is None:+ workflow_request.run_with = workflow.run_with++ # Force ai_fallback=True for adaptive caching (code_version >= 2) runs.+ # Adaptive caching requires AI fallback to self-heal when cached scripts break.+ # Without this, a caller sending ai_fallback=false would silently disable recovery.+ effective_code_version = (+ workflow.code_version+ if workflow.code_version is not None+ else (2 if workflow.adaptive_caching else None)+ )+ if (effective_code_version or 0) >= 2 and (workflow_request.run_with == "code"):+ if workflow_request.ai_fallback is False:+ LOG.info(+ "Overriding ai_fallback to True for adaptive caching run",+ workflow_permanent_id=workflow_permanent_id,+ request_run_with=workflow_request.run_with,+ workflow_code_version=workflow.code_version,+ )+ workflow_request.ai_fallback = True- # Create the workflow run and set skyvern context- workflow_run = await self.create_workflow_run(- workflow_request=workflow_request,- workflow_permanent_id=workflow_permanent_id,- workflow_id=workflow_id,- organization_id=organization.organization_id,- parent_workflow_run_id=parent_workflow_run_id,- sequential_key=workflow.sequential_key,- debug_session_id=debug_session_id,- code_gen=code_gen,- workflow_run_id=workflow_run_id,- trigger_type=trigger_type,- workflow_schedule_id=workflow_schedule_id,- ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,- copilot_session_id=resolved_copilot_session_id,- )- LOG.info(- f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",- request_id=request_id,- workflow_run_id=workflow_run.workflow_run_id,- workflow_id=workflow.workflow_id,- organization_id=workflow.organization_id,- proxy_location=workflow_request.proxy_location,- webhook_callback_url=workflow_request.webhook_callback_url,- max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,- ai_fallback=workflow_request.ai_fallback,- run_with=workflow_request.run_with,- code_gen=code_gen,- )- context: skyvern_context.SkyvernContext | None = skyvern_context.current()- current_run_id = context.run_id if context and context.run_id else workflow_run.workflow_run_id- root_workflow_run_id = (- context.root_workflow_run_id if context and context.root_workflow_run_id else workflow_run.workflow_run_id- )- skyvern_context.replace(- SkyvernContext(- organization_id=organization.organization_id,- organization_name=organization.organization_name,- request_id=request_id,+ # Inherit from ambient context so descendant runs (TriggerWorkflowBlock children)+ # carry the parent's chat id forward without per-call plumbing. Resolved here so+ # the same value reaches both the DB row and the new SkyvernContext below.+ ambient_context: skyvern_context.SkyvernContext | None = skyvern_context.current()+ resolved_copilot_session_id = (+ copilot_session_id+ if copilot_session_id is not None+ else (ambient_context.copilot_session_id if ambient_context else None)+ )++ # Create the workflow run and set skyvern context+ workflow_run = await self.create_workflow_run(+ workflow_request=workflow_request,+ workflow_permanent_id=workflow_permanent_id,workflow_id=workflow_id,- workflow_run_id=workflow_run.workflow_run_id,- root_workflow_run_id=root_workflow_run_id,- run_id=current_run_id,- workflow_permanent_id=workflow_run.workflow_permanent_id,- max_steps_override=max_steps_override,- max_screenshot_scrolls=workflow_request.max_screenshot_scrolls,- loop_internal_state=copy.deepcopy(context.loop_internal_state) if context else None,+ organization_id=organization.organization_id,+ parent_workflow_run_id=parent_workflow_run_id,+ sequential_key=workflow.sequential_key,+ debug_session_id=debug_session_id,+ code_gen=code_gen,+ workflow_run_id=workflow_run_id,+ trigger_type=trigger_type,+ workflow_schedule_id=workflow_schedule_id,+ ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,copilot_session_id=resolved_copilot_session_id,)- )-- # Check artifact bundling flag at workflow level so it applies to both agent and cached paths.- # See also: skyvern/forge/agent.py Agent.agent_step() checks per-task for standalone task runs.- new_context = skyvern_context.current()- if new_context:- try:- new_context.use_artifact_bundling = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(- "USE_ARTIFACT_BUNDLING",- workflow_run.workflow_run_id,- properties={"organization_id": organization.organization_id},- )- LOG.debug(- "USE_ARTIFACT_BUNDLING flag resolved for workflow",- use_artifact_bundling=new_context.use_artifact_bundling,- workflow_run_id=workflow_run.workflow_run_id,+ LOG.info(+ f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",+ request_id=request_id,+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_id=workflow.workflow_id,+ organization_id=workflow.organization_id,+ proxy_location=workflow_request.proxy_location,+ webhook_callback_url=workflow_request.webhook_callback_url,+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,+ ai_fallback=workflow_request.ai_fallback,+ run_with=workflow_request.run_with,+ code_gen=code_gen,+ )+ context: skyvern_context.SkyvernContext | None = skyvern_context.current()+ current_run_id = context.run_id if context and context.run_id else workflow_run.workflow_run_id+ root_workflow_run_id = (+ context.root_workflow_run_id+ if context and context.root_workflow_run_id+ else workflow_run.workflow_run_id+ )+ skyvern_context.replace(+ SkyvernContext(organization_id=organization.organization_id,- )- except Exception:- LOG.warning("Failed to check USE_ARTIFACT_BUNDLING flag for workflow", exc_info=True)- new_context.use_artifact_bundling = False-- # Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.- all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)- try:- missing_parameters: list[str] = []- workflow_parameter_values: list[tuple[WorkflowParameter, Any]] = []- for workflow_parameter in all_workflow_parameters:- if workflow_request.data and workflow_parameter.key in workflow_request.data:- request_body_value = workflow_request.data[workflow_parameter.key]- # Fall back to default value if the request explicitly sends null- # This supports API clients (e.g., n8n) that include the key with null value- if request_body_value is None and workflow_parameter.default_value is not None:- request_body_value = workflow_parameter.default_value- if self._is_missing_required_value(workflow_parameter, request_body_value):- missing_parameters.append(workflow_parameter.key)- continue- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:- if not isinstance(request_body_value, str):- raise InvalidCredentialId(f"<non-string value of type {type(request_body_value).__name__}>")- workflow_parameter_values.append((workflow_parameter, request_body_value))- elif workflow_parameter.default_value is not None:- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:- if not isinstance(workflow_parameter.default_value, str):- raise InvalidCredentialId(- f"<non-string value of type {type(workflow_parameter.default_value).__name__}>"- )- workflow_parameter_values.append((workflow_parameter, workflow_parameter.default_value))- else:- missing_parameters.append(workflow_parameter.key)-- if missing_parameters:- missing_list = ", ".join(sorted(missing_parameters))- raise MissingValueForParameter(- parameter_key=missing_list,- workflow_id=workflow.workflow_permanent_id,+ organization_name=organization.organization_name,+ request_id=request_id,+ workflow_id=workflow_id,workflow_run_id=workflow_run.workflow_run_id,+ root_workflow_run_id=root_workflow_run_id,+ run_id=current_run_id,+ workflow_permanent_id=workflow_run.workflow_permanent_id,+ max_steps_override=max_steps_override,+ max_screenshot_scrolls=workflow_request.max_screenshot_scrolls,+ loop_internal_state=copy.deepcopy(context.loop_internal_state) if context else None,+ copilot_session_id=resolved_copilot_session_id,)-- await self._validate_credential_ids(- [- value- for (workflow_parameter, value) in workflow_parameter_values- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID- ],- organization,)- if workflow_parameter_values:+ # Check artifact bundling flag at workflow level so it applies to both agent and cached paths.+ # See also: skyvern/forge/agent.py Agent.agent_step() checks per-task for standalone task runs.+ new_context = skyvern_context.current()+ if new_context:try:- await self.create_workflow_run_parameters(+ new_context.use_artifact_bundling = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(+ "USE_ARTIFACT_BUNDLING",+ workflow_run.workflow_run_id,+ properties={"organization_id": organization.organization_id},+ )+ LOG.debug(+ "USE_ARTIFACT_BUNDLING flag resolved for workflow",+ use_artifact_bundling=new_context.use_artifact_bundling,workflow_run_id=workflow_run.workflow_run_id,- workflow_parameter_values=workflow_parameter_values,+ organization_id=organization.organization_id,)- except SQLAlchemyError as batch_error:- # Batch failed — retry one-by-one to identify the exact failing parameter- for workflow_parameter, value in workflow_parameter_values:- try:- await self.create_workflow_run_parameter(- workflow_run_id=workflow_run.workflow_run_id,- workflow_parameter=workflow_parameter,- value=value,- )- except SQLAlchemyError as parameter_error:- raise WorkflowRunParameterPersistenceError(- parameter_key=workflow_parameter.key,- workflow_id=workflow.workflow_permanent_id,- workflow_run_id=workflow_run.workflow_run_id,- reason=self._format_parameter_persistence_error(parameter_error),- ) from parameter_error- # All individual inserts succeeded — the batch failure was transient- LOG.warning(- "Batch parameter insert failed but individual inserts succeeded",+ except Exception:+ LOG.warning("Failed to check USE_ARTIFACT_BUNDLING flag for workflow", exc_info=True)+ new_context.use_artifact_bundling = False++ # Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.+ all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)+ try:+ missing_parameters: list[str] = []+ workflow_parameter_values: list[tuple[WorkflowParameter, Any]] = []+ for workflow_parameter in all_workflow_parameters:+ if workflow_request.data and workflow_parameter.key in workflow_request.data:+ request_body_value = workflow_request.data[workflow_parameter.key]+ # Fall back to default value if the request explicitly sends null+ # This supports API clients (e.g., n8n) that include the key with null value+ if request_body_value is None and workflow_parameter.default_value is not None:+ request_body_value = workflow_parameter.default_value+ if self._is_missing_required_value(workflow_parameter, request_body_value):+ missing_parameters.append(workflow_parameter.key)+ continue+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:+ if not isinstance(request_body_value, str):+ raise InvalidCredentialId(+ f"<non-string value of type {type(request_body_value).__name__}>"+ )+ workflow_parameter_values.append((workflow_parameter, request_body_value))+ elif workflow_parameter.default_value is not None:+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:+ if not isinstance(workflow_parameter.default_value, str):+ raise InvalidCredentialId(+ f"<non-string value of type {type(workflow_parameter.default_value).__name__}>"+ )+ workflow_parameter_values.append((workflow_parameter, workflow_parameter.default_value))+ else:+ missing_parameters.append(workflow_parameter.key)++ if missing_parameters:+ missing_list = ", ".join(sorted(missing_parameters))+ raise MissingValueForParameter(+ parameter_key=missing_list,+ workflow_id=workflow.workflow_permanent_id,workflow_run_id=workflow_run.workflow_run_id,- batch_error=str(batch_error),)- except Exception as e:- LOG.exception(- f"Error while setting up workflow run {workflow_run.workflow_run_id}",- workflow_run_id=workflow_run.workflow_run_id,- )- failure_reason = f"Setup workflow failed. failure reason: {get_user_facing_exception_message(e)}"+ await self._validate_credential_ids(+ [+ value+ for (workflow_parameter, value) in workflow_parameter_values+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID+ ],+ organization,+ )- workflow_run = await self.mark_workflow_run_as_failed(- workflow_run_id=workflow_run.workflow_run_id, failure_reason=failure_reason- )- raise e+ if workflow_parameter_values:+ try:+ await self.create_workflow_run_parameters(+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_parameter_values=workflow_parameter_values,+ )+ except SQLAlchemyError as batch_error:+ # Roll back the failed transaction so the per-parameter fallback+ # (and any later mark_workflow_run_as_failed) can reuse the outer session.+ await outer_session.rollback()+ # Batch failed — retry one-by-one to identify the exact failing parameter+ for workflow_parameter, value in workflow_parameter_values:+ try:+ await self.create_workflow_run_parameter(+ workflow_run_id=workflow_run.workflow_run_id,+ workflow_parameter=workflow_parameter,+ value=value,+ )+ except SQLAlchemyError as parameter_error:+ raise WorkflowRunParameterPersistenceError(+ parameter_key=workflow_parameter.key,+ workflow_id=workflow.workflow_permanent_id,+ workflow_run_id=workflow_run.workflow_run_id,+ reason=self._format_parameter_persistence_error(parameter_error),+ ) from parameter_error+ # All individual inserts succeeded — the batch failure was transient+ LOG.warning(+ "Batch parameter insert failed but individual inserts succeeded",+ workflow_run_id=workflow_run.workflow_run_id,+ batch_error=str(batch_error),+ )+ except Exception as e:+ LOG.exception(+ f"Error while setting up workflow run {workflow_run.workflow_run_id}",+ workflow_run_id=workflow_run.workflow_run_id,+ )- return workflow_run+ # Discard any failed transaction state on the shared outer session before+ # mark_workflow_run_as_failed reuses it.+ try:+ await outer_session.rollback()+ except SQLAlchemyError:+ LOG.warning("Failed to rollback outer session during setup failure", exc_info=True)++ failure_reason = f"Setup workflow failed. failure reason: {get_user_facing_exception_message(e)}"++ workflow_run = await self.mark_workflow_run_as_failed(+ workflow_run_id=workflow_run.workflow_run_id, failure_reason=failure_reason+ )+ raise++ return workflow_run@staticmethoddef _format_parameter_persistence_error(error: SQLAlchemyError) -> str:diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 983c4242b..75cdfdcb1 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -218,17 +218,30 @@ async def poll_otp_value(totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,)- max_consecutive_failures = settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURESconsecutive_failures = 0+ last_error_reason: str | None = Nonewhile True:await asyncio.sleep(10)- # check timeoutif datetime.utcnow() > timeout_datetime:+ if consecutive_failures > 0 and last_error_reason is not None:+ LOG.warning(+ "Polling otp value timed out while webhook was still failing",+ consecutive_failures=consecutive_failures,+ last_error_reason=last_error_reason,+ )+ raise FailedToGetTOTPVerificationCode(+ task_id=task_id,+ workflow_run_id=workflow_run_id,+ workflow_id=workflow_id or workflow_permanent_id,+ totp_verification_url=totp_verification_url,+ totp_identifier=totp_identifier,+ reason=last_error_reason,+ )LOG.warning("Polling otp value timed out")raise NoTOTPVerificationCodeFound(task_id=task_id,workflow_run_id=workflow_run_id,- workflow_id=workflow_permanent_id,+ workflow_id=workflow_id or workflow_permanent_id,totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,)@@ -250,21 +263,21 @@ async def poll_otp_value(workflow_id=workflow_permanent_id,workflow_run_id=workflow_run_id,)- except FailedToGetTOTPVerificationCode:+ except FailedToGetTOTPVerificationCode as e:consecutive_failures += 1+ last_error_reason = e.reasonLOG.warning(- "OTP fetch failed, will retry",+ "OTP fetch failed, will retry until wall-clock timeout",consecutive_failures=consecutive_failures,- max_consecutive_failures=max_consecutive_failures,+ last_error_reason=e.reason,task_id=task_id,workflow_run_id=workflow_run_id,totp_verification_url=totp_verification_url,totp_identifier=totp_identifier,)- if consecutive_failures >= max_consecutive_failures:- raisecontinue- consecutive_failures = 0 # reset on any non-exception poll, even if otp_value is None+ consecutive_failures = 0+ last_error_reason = Noneif otp_value:LOG.info("Got otp value", otp_value=otp_value)return otp_valuediff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.pyindex d9d04576f..7ba8dc91f 100644--- a/tests/unit/test_otp_service.py+++ b/tests/unit/test_otp_service.py@@ -2,13 +2,14 @@from __future__ import annotations+from datetime import datetime, timedeltafrom types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pyotpimport pytest-from skyvern.exceptions import FailedToGetTOTPVerificationCode+from skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFoundfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import SkyvernContextfrom skyvern.services.otp_service import (@@ -117,7 +118,7 @@ def _mock_org_token() -> MagicMock:class TestPollOtpValueRetry:- """poll_otp_value should tolerate transient FailedToGetTOTPVerificationCode up to max consecutive failures."""+ """poll_otp_value retries fetch failures across the full wall-clock timeout window."""@pytest.mark.asyncio@patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)@@ -127,9 +128,8 @@ class TestPollOtpValueRetry:async def test_retries_on_transient_failure_then_succeeds(self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock) -> None:- """Fail twice, then return OTP on third attempt. Counter resets, OTP is returned."""+ """Fail twice, then return OTP on third attempt."""mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())otp = OTPValue(value="123456")@@ -148,58 +148,95 @@ class TestPollOtpValueRetry:assert mock_fetch.call_count == 3@pytest.mark.asyncio- @patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)- @patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock)- @patch("skyvern.services.otp_service.app")- @patch("skyvern.services.otp_service.settings")- async def test_raises_after_max_consecutive_failures(- self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock- ) -> None:- """After N consecutive failures, re-raise FailedToGetTOTPVerificationCode."""- mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3- mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())-- mock_fetch.side_effect = FailedToGetTOTPVerificationCode(reason="connection error")-- with pytest.raises(FailedToGetTOTPVerificationCode):- await poll_otp_value(+ async def test_does_not_short_circuit_during_extended_outage(self) -> None:+ """Persistent webhook errors must not exit polling before the wall-clock timeout fires (SKY-9553)."""+ base = datetime(2026, 1, 1, 12, 0, 0)+ utcnow_returns = [base] + [base + timedelta(seconds=60 * (i + 1)) for i in range(8)]++ with (+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.settings") as mock_settings,+ patch("skyvern.services.otp_service.datetime") as mock_datetime,+ ):+ mock_datetime.utcnow.side_effect = utcnow_returns+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())++ otp = OTPValue(value="424242")+ mock_fetch.side_effect = [+ *([FailedToGetTOTPVerificationCode(reason="connection refused")] * 7),+ otp,+ ]++ result = await poll_otp_value(organization_id="o_test",task_id="tsk_test",totp_verification_url="https://example.com/mfa",)- assert mock_fetch.call_count == 3+ assert result == otp+ assert mock_fetch.call_count == 8@pytest.mark.asyncio- @patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)- @patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock)- @patch("skyvern.services.otp_service.app")- @patch("skyvern.services.otp_service.settings")- async def test_counter_resets_after_success(- self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock- ) -> None:- """Fail twice, succeed (None), fail twice again, then return OTP. Counter resets between failure bursts."""- mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3- mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())+ async def test_raises_failed_with_reason_when_timeout_during_failure_streak(self) -> None:+ """When the wall-clock timeout fires while the webhook is still failing, surface the underlying reason."""+ base = datetime(2026, 1, 1, 12, 0, 0)+ utcnow_returns = [+ base, # start_datetime+ base + timedelta(seconds=30), # iter 1: not timed out, fetch fails+ base + timedelta(seconds=60), # iter 2: not timed out, fetch fails+ base + timedelta(minutes=16), # iter 3: timed out+ ]- otp = OTPValue(value="654321")- mock_fetch.side_effect = [- FailedToGetTOTPVerificationCode(reason="err"),- FailedToGetTOTPVerificationCode(reason="err"),- None, # success (no OTP yet, but resets counter)- FailedToGetTOTPVerificationCode(reason="err"),- FailedToGetTOTPVerificationCode(reason="err"),- otp,+ with (+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.settings") as mock_settings,+ patch("skyvern.services.otp_service.datetime") as mock_datetime,+ ):+ mock_datetime.utcnow.side_effect = utcnow_returns+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())+ mock_fetch.side_effect = FailedToGetTOTPVerificationCode(reason="connection refused")++ with pytest.raises(FailedToGetTOTPVerificationCode) as exc_info:+ await poll_otp_value(+ organization_id="o_test",+ task_id="tsk_test",+ totp_verification_url="https://example.com/mfa",+ )+ assert exc_info.value.reason == "connection refused"++ @pytest.mark.asyncio+ async def test_raises_no_otp_at_timeout_when_polls_were_clean(self) -> None:+ """When polling timed out without webhook errors (just no OTP yet), raise NoTOTPVerificationCodeFound."""+ base = datetime(2026, 1, 1, 12, 0, 0)+ utcnow_returns = [+ base,+ base + timedelta(seconds=30),+ base + timedelta(minutes=16),]- result = await poll_otp_value(- organization_id="o_test",- task_id="tsk_test",- totp_verification_url="https://example.com/mfa",- )- assert result == otp- assert mock_fetch.call_count == 6+ with (+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,+ patch("skyvern.services.otp_service.app") as mock_app,+ patch("skyvern.services.otp_service.settings") as mock_settings,+ patch("skyvern.services.otp_service.datetime") as mock_datetime,+ ):+ mock_datetime.utcnow.side_effect = utcnow_returns+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())+ mock_fetch.return_value = None++ with pytest.raises(NoTOTPVerificationCodeFound):+ await poll_otp_value(+ organization_id="o_test",+ task_id="tsk_test",+ totp_verification_url="https://example.com/mfa",+ )class _FakeWorkflowRunContext:diff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.pyindex c40ed7509..89c2a3c27 100644--- a/tests/unit/test_workflow_run_parameter_batching.py+++ b/tests/unit/test_workflow_run_parameter_batching.py@@ -367,3 +367,95 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_conassert current_context.root_workflow_run_id == "wr_root"assert current_context.loop_internal_state == loop_stateassert current_context.loop_internal_state is not loop_state+++async def test_setup_workflow_run_opens_one_outer_session() -> None:+ """setup_workflow_run wraps its body in exactly one outer ``Session()`` context."""++ params = [_make_workflow_parameter("k", default_value="v")]+ service, organization, _ = _make_service_with_mocks(workflow_parameters=params)++ session_open_count = 0++ class _Counter:+ async def __aenter__(self) -> _Counter:+ nonlocal session_open_count+ session_open_count += 1+ return self++ async def __aexit__(self, *_a: object) -> None:+ return None++ async def rollback(self) -> None:+ return None++ request = WorkflowRequestBody(data={"k": "v"})+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)+ mock_app.DATABASE.workflow_runs.Session = lambda: _Counter()+ await service.setup_workflow_run(+ request_id="req_test",+ workflow_request=request,+ workflow_permanent_id="wpid_test",+ organization=organization,+ )++ assert session_open_count == 1, (+ f"Expected exactly one outer Session() open in setup_workflow_run, got {session_open_count}"+ )+++async def test_setup_workflow_run_rolls_back_outer_session_on_batch_failure() -> None:+ """When the batched parameter insert raises, the outer session must be rolled back+ before the per-parameter fallback reuses it - otherwise the fallback runs on a+ session whose transaction is in error state."""++ params = [+ _make_workflow_parameter("a", default_value="1"),+ _make_workflow_parameter("b", default_value="2"),+ ]+ batch_error = IntegrityError("INSERT", {}, Exception("constraint failed"))++ fallback_call_index: list[int] = []+ rollback_index: list[int] = []+ call_counter = {"n": 0}++ async def _fallback_insert(*, workflow_run_id: str, workflow_parameter: WorkflowParameter, value: object) -> None:+ call_counter["n"] += 1+ fallback_call_index.append(call_counter["n"])++ service, organization, _ = _make_service_with_mocks(+ workflow_parameters=params,+ batch_side_effect=batch_error,+ )+ service.create_workflow_run_parameter = AsyncMock(side_effect=_fallback_insert) # type: ignore[method-assign]++ class _Session:+ async def __aenter__(self) -> _Session:+ return self++ async def __aexit__(self, *_a: object) -> None:+ return None++ async def rollback(self) -> None:+ call_counter["n"] += 1+ rollback_index.append(call_counter["n"])++ request = WorkflowRequestBody(data={"a": "1", "b": "2"})+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)+ mock_app.DATABASE.workflow_runs.Session = lambda: _Session()+ await service.setup_workflow_run(+ request_id="req_test",+ workflow_request=request,+ workflow_permanent_id="wpid_test",+ organization=organization,+ )++ assert rollback_index, "Expected rollback() to be called on the outer session after batch failure"+ assert fallback_call_index, "Expected the fallback path to run after rollback"+ assert rollback_index[0] < fallback_call_index[0], (+ f"rollback must precede fallback insert; got rollback at {rollback_index} fallback at {fallback_call_index}"+ )