Skip to content

Automate browser based workflows with AI

fix(SKY-9553): retry OTP polling for full wall-clock timeout (#5878)

Aaron Perez2d5e5e4b727f
diff --git a/skyvern/config.py b/skyvern/config.py
index 8f11297b1..abf3d6801 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -98,9 +98,9 @@ class Settings(BaseSettings):
DATABASE_REPLICA_STRING: str | None = None
DATABASE_STATEMENT_TIMEOUT_MS: int = 60000
DISABLE_CONNECTION_POOL: bool = False
- DATABASE_POOL_SIZE: int = 5
- DATABASE_POOL_MAX_OVERFLOW: int = 10
- DATABASE_POOL_TIMEOUT: int = 30
+ DATABASE_POOL_SIZE: int = 20
+ DATABASE_POOL_MAX_OVERFLOW: int = 20
+ DATABASE_POOL_TIMEOUT: int = 10
DATABASE_POOL_RECYCLE: int = 1800
PROMPT_ACTION_HISTORY_WINDOW: int = 1
TASK_RESPONSE_ACTION_SCREENSHOT_COUNT: int = 3
@@ -425,7 +425,6 @@ class Settings(BaseSettings):
TOTP_LIFESPAN_MINUTES: int = 10
VERIFICATION_CODE_INITIAL_WAIT_TIME_SECS: int = 40
VERIFICATION_CODE_POLLING_TIMEOUT_MINS: int = 15
- VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES: int = 3
# Bitwarden Settings
BITWARDEN_CLIENT_ID: str | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 6d93d2f78..e012079a7 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -786,210 +786,227 @@ class WorkflowService:
:param max_steps_override: The max steps override for the workflow run, if any.
:return: The created workflow run.
"""
- # Validate the workflow and the organization
- workflow = await self.get_workflow_by_permanent_id(
- workflow_permanent_id=workflow_permanent_id,
- organization_id=None if is_template_workflow else organization.organization_id,
- version=version,
- )
- if workflow is None:
- LOG.error(f"Workflow {workflow_permanent_id} not found", workflow_version=version)
- raise WorkflowNotFound(workflow_permanent_id=workflow_permanent_id, version=version)
- workflow_id = workflow.workflow_id
- if workflow_request.proxy_location is None and workflow.proxy_location is not None:
- workflow_request.proxy_location = workflow.proxy_location
- if workflow_request.webhook_callback_url is None and workflow.webhook_callback_url is not None:
- workflow_request.webhook_callback_url = workflow.webhook_callback_url
- if workflow_request.extra_http_headers is None and workflow.extra_http_headers is not None:
- workflow_request.extra_http_headers = workflow.extra_http_headers
- if workflow_request.run_with is None:
- workflow_request.run_with = workflow.run_with
-
- # Force ai_fallback=True for adaptive caching (code_version >= 2) runs.
- # Adaptive caching requires AI fallback to self-heal when cached scripts break.
- # Without this, a caller sending ai_fallback=false would silently disable recovery.
- effective_code_version = (
- workflow.code_version if workflow.code_version is not None else (2 if workflow.adaptive_caching else None)
- )
- if (effective_code_version or 0) >= 2 and (workflow_request.run_with == "code"):
- if workflow_request.ai_fallback is False:
- LOG.info(
- "Overriding ai_fallback to True for adaptive caching run",
- workflow_permanent_id=workflow_permanent_id,
- request_run_with=workflow_request.run_with,
- workflow_code_version=workflow.code_version,
- )
- workflow_request.ai_fallback = True
-
- # Inherit from ambient context so descendant runs (TriggerWorkflowBlock children)
- # carry the parent's chat id forward without per-call plumbing. Resolved here so
- # the same value reaches both the DB row and the new SkyvernContext below.
- ambient_context: skyvern_context.SkyvernContext | None = skyvern_context.current()
- resolved_copilot_session_id = (
- copilot_session_id
- if copilot_session_id is not None
- else (ambient_context.copilot_session_id if ambient_context else None)
- )
+ async with app.DATABASE.workflow_runs.Session() as outer_session:
+ # Validate the workflow and the organization
+ workflow = await self.get_workflow_by_permanent_id(
+ workflow_permanent_id=workflow_permanent_id,
+ organization_id=None if is_template_workflow else organization.organization_id,
+ version=version,
+ )
+ if workflow is None:
+ LOG.error(f"Workflow {workflow_permanent_id} not found", workflow_version=version)
+ raise WorkflowNotFound(workflow_permanent_id=workflow_permanent_id, version=version)
+ workflow_id = workflow.workflow_id
+ if workflow_request.proxy_location is None and workflow.proxy_location is not None:
+ workflow_request.proxy_location = workflow.proxy_location
+ if workflow_request.webhook_callback_url is None and workflow.webhook_callback_url is not None:
+ workflow_request.webhook_callback_url = workflow.webhook_callback_url
+ if workflow_request.extra_http_headers is None and workflow.extra_http_headers is not None:
+ workflow_request.extra_http_headers = workflow.extra_http_headers
+ if workflow_request.run_with is None:
+ workflow_request.run_with = workflow.run_with
+
+ # Force ai_fallback=True for adaptive caching (code_version >= 2) runs.
+ # Adaptive caching requires AI fallback to self-heal when cached scripts break.
+ # Without this, a caller sending ai_fallback=false would silently disable recovery.
+ effective_code_version = (
+ workflow.code_version
+ if workflow.code_version is not None
+ else (2 if workflow.adaptive_caching else None)
+ )
+ if (effective_code_version or 0) >= 2 and (workflow_request.run_with == "code"):
+ if workflow_request.ai_fallback is False:
+ LOG.info(
+ "Overriding ai_fallback to True for adaptive caching run",
+ workflow_permanent_id=workflow_permanent_id,
+ request_run_with=workflow_request.run_with,
+ workflow_code_version=workflow.code_version,
+ )
+ workflow_request.ai_fallback = True
- # Create the workflow run and set skyvern context
- workflow_run = await self.create_workflow_run(
- workflow_request=workflow_request,
- workflow_permanent_id=workflow_permanent_id,
- workflow_id=workflow_id,
- organization_id=organization.organization_id,
- parent_workflow_run_id=parent_workflow_run_id,
- sequential_key=workflow.sequential_key,
- debug_session_id=debug_session_id,
- code_gen=code_gen,
- workflow_run_id=workflow_run_id,
- trigger_type=trigger_type,
- workflow_schedule_id=workflow_schedule_id,
- ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
- copilot_session_id=resolved_copilot_session_id,
- )
- LOG.info(
- f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",
- request_id=request_id,
- workflow_run_id=workflow_run.workflow_run_id,
- workflow_id=workflow.workflow_id,
- organization_id=workflow.organization_id,
- proxy_location=workflow_request.proxy_location,
- webhook_callback_url=workflow_request.webhook_callback_url,
- max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,
- ai_fallback=workflow_request.ai_fallback,
- run_with=workflow_request.run_with,
- code_gen=code_gen,
- )
- context: skyvern_context.SkyvernContext | None = skyvern_context.current()
- current_run_id = context.run_id if context and context.run_id else workflow_run.workflow_run_id
- root_workflow_run_id = (
- context.root_workflow_run_id if context and context.root_workflow_run_id else workflow_run.workflow_run_id
- )
- skyvern_context.replace(
- SkyvernContext(
- organization_id=organization.organization_id,
- organization_name=organization.organization_name,
- request_id=request_id,
+ # Inherit from ambient context so descendant runs (TriggerWorkflowBlock children)
+ # carry the parent's chat id forward without per-call plumbing. Resolved here so
+ # the same value reaches both the DB row and the new SkyvernContext below.
+ ambient_context: skyvern_context.SkyvernContext | None = skyvern_context.current()
+ resolved_copilot_session_id = (
+ copilot_session_id
+ if copilot_session_id is not None
+ else (ambient_context.copilot_session_id if ambient_context else None)
+ )
+
+ # Create the workflow run and set skyvern context
+ workflow_run = await self.create_workflow_run(
+ workflow_request=workflow_request,
+ workflow_permanent_id=workflow_permanent_id,
workflow_id=workflow_id,
- workflow_run_id=workflow_run.workflow_run_id,
- root_workflow_run_id=root_workflow_run_id,
- run_id=current_run_id,
- workflow_permanent_id=workflow_run.workflow_permanent_id,
- max_steps_override=max_steps_override,
- max_screenshot_scrolls=workflow_request.max_screenshot_scrolls,
- loop_internal_state=copy.deepcopy(context.loop_internal_state) if context else None,
+ organization_id=organization.organization_id,
+ parent_workflow_run_id=parent_workflow_run_id,
+ sequential_key=workflow.sequential_key,
+ debug_session_id=debug_session_id,
+ code_gen=code_gen,
+ workflow_run_id=workflow_run_id,
+ trigger_type=trigger_type,
+ workflow_schedule_id=workflow_schedule_id,
+ ignore_inherited_workflow_system_prompt=ignore_inherited_workflow_system_prompt,
copilot_session_id=resolved_copilot_session_id,
)
- )
-
- # Check artifact bundling flag at workflow level so it applies to both agent and cached paths.
- # See also: skyvern/forge/agent.py Agent.agent_step() checks per-task for standalone task runs.
- new_context = skyvern_context.current()
- if new_context:
- try:
- new_context.use_artifact_bundling = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
- "USE_ARTIFACT_BUNDLING",
- workflow_run.workflow_run_id,
- properties={"organization_id": organization.organization_id},
- )
- LOG.debug(
- "USE_ARTIFACT_BUNDLING flag resolved for workflow",
- use_artifact_bundling=new_context.use_artifact_bundling,
- workflow_run_id=workflow_run.workflow_run_id,
+ LOG.info(
+ f"Created workflow run {workflow_run.workflow_run_id} for workflow {workflow.workflow_id}",
+ request_id=request_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_id=workflow.workflow_id,
+ organization_id=workflow.organization_id,
+ proxy_location=workflow_request.proxy_location,
+ webhook_callback_url=workflow_request.webhook_callback_url,
+ max_screenshot_scrolling_times=workflow_request.max_screenshot_scrolls,
+ ai_fallback=workflow_request.ai_fallback,
+ run_with=workflow_request.run_with,
+ code_gen=code_gen,
+ )
+ context: skyvern_context.SkyvernContext | None = skyvern_context.current()
+ current_run_id = context.run_id if context and context.run_id else workflow_run.workflow_run_id
+ root_workflow_run_id = (
+ context.root_workflow_run_id
+ if context and context.root_workflow_run_id
+ else workflow_run.workflow_run_id
+ )
+ skyvern_context.replace(
+ SkyvernContext(
organization_id=organization.organization_id,
- )
- except Exception:
- LOG.warning("Failed to check USE_ARTIFACT_BUNDLING flag for workflow", exc_info=True)
- new_context.use_artifact_bundling = False
-
- # Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.
- all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)
- try:
- missing_parameters: list[str] = []
- workflow_parameter_values: list[tuple[WorkflowParameter, Any]] = []
- for workflow_parameter in all_workflow_parameters:
- if workflow_request.data and workflow_parameter.key in workflow_request.data:
- request_body_value = workflow_request.data[workflow_parameter.key]
- # Fall back to default value if the request explicitly sends null
- # This supports API clients (e.g., n8n) that include the key with null value
- if request_body_value is None and workflow_parameter.default_value is not None:
- request_body_value = workflow_parameter.default_value
- if self._is_missing_required_value(workflow_parameter, request_body_value):
- missing_parameters.append(workflow_parameter.key)
- continue
- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:
- if not isinstance(request_body_value, str):
- raise InvalidCredentialId(f"<non-string value of type {type(request_body_value).__name__}>")
- workflow_parameter_values.append((workflow_parameter, request_body_value))
- elif workflow_parameter.default_value is not None:
- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:
- if not isinstance(workflow_parameter.default_value, str):
- raise InvalidCredentialId(
- f"<non-string value of type {type(workflow_parameter.default_value).__name__}>"
- )
- workflow_parameter_values.append((workflow_parameter, workflow_parameter.default_value))
- else:
- missing_parameters.append(workflow_parameter.key)
-
- if missing_parameters:
- missing_list = ", ".join(sorted(missing_parameters))
- raise MissingValueForParameter(
- parameter_key=missing_list,
- workflow_id=workflow.workflow_permanent_id,
+ organization_name=organization.organization_name,
+ request_id=request_id,
+ workflow_id=workflow_id,
workflow_run_id=workflow_run.workflow_run_id,
+ root_workflow_run_id=root_workflow_run_id,
+ run_id=current_run_id,
+ workflow_permanent_id=workflow_run.workflow_permanent_id,
+ max_steps_override=max_steps_override,
+ max_screenshot_scrolls=workflow_request.max_screenshot_scrolls,
+ loop_internal_state=copy.deepcopy(context.loop_internal_state) if context else None,
+ copilot_session_id=resolved_copilot_session_id,
)
-
- await self._validate_credential_ids(
- [
- value
- for (workflow_parameter, value) in workflow_parameter_values
- if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
- ],
- organization,
)
- if workflow_parameter_values:
+ # Check artifact bundling flag at workflow level so it applies to both agent and cached paths.
+ # See also: skyvern/forge/agent.py Agent.agent_step() checks per-task for standalone task runs.
+ new_context = skyvern_context.current()
+ if new_context:
try:
- await self.create_workflow_run_parameters(
+ new_context.use_artifact_bundling = await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
+ "USE_ARTIFACT_BUNDLING",
+ workflow_run.workflow_run_id,
+ properties={"organization_id": organization.organization_id},
+ )
+ LOG.debug(
+ "USE_ARTIFACT_BUNDLING flag resolved for workflow",
+ use_artifact_bundling=new_context.use_artifact_bundling,
workflow_run_id=workflow_run.workflow_run_id,
- workflow_parameter_values=workflow_parameter_values,
+ organization_id=organization.organization_id,
)
- except SQLAlchemyError as batch_error:
- # Batch failed — retry one-by-one to identify the exact failing parameter
- for workflow_parameter, value in workflow_parameter_values:
- try:
- await self.create_workflow_run_parameter(
- workflow_run_id=workflow_run.workflow_run_id,
- workflow_parameter=workflow_parameter,
- value=value,
- )
- except SQLAlchemyError as parameter_error:
- raise WorkflowRunParameterPersistenceError(
- parameter_key=workflow_parameter.key,
- workflow_id=workflow.workflow_permanent_id,
- workflow_run_id=workflow_run.workflow_run_id,
- reason=self._format_parameter_persistence_error(parameter_error),
- ) from parameter_error
- # All individual inserts succeeded — the batch failure was transient
- LOG.warning(
- "Batch parameter insert failed but individual inserts succeeded",
+ except Exception:
+ LOG.warning("Failed to check USE_ARTIFACT_BUNDLING flag for workflow", exc_info=True)
+ new_context.use_artifact_bundling = False
+
+ # Create all the workflow run parameters, AWSSecretParameter won't have workflow run parameters created.
+ all_workflow_parameters = await self.get_workflow_parameters(workflow_id=workflow.workflow_id)
+ try:
+ missing_parameters: list[str] = []
+ workflow_parameter_values: list[tuple[WorkflowParameter, Any]] = []
+ for workflow_parameter in all_workflow_parameters:
+ if workflow_request.data and workflow_parameter.key in workflow_request.data:
+ request_body_value = workflow_request.data[workflow_parameter.key]
+ # Fall back to default value if the request explicitly sends null
+ # This supports API clients (e.g., n8n) that include the key with null value
+ if request_body_value is None and workflow_parameter.default_value is not None:
+ request_body_value = workflow_parameter.default_value
+ if self._is_missing_required_value(workflow_parameter, request_body_value):
+ missing_parameters.append(workflow_parameter.key)
+ continue
+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:
+ if not isinstance(request_body_value, str):
+ raise InvalidCredentialId(
+ f"<non-string value of type {type(request_body_value).__name__}>"
+ )
+ workflow_parameter_values.append((workflow_parameter, request_body_value))
+ elif workflow_parameter.default_value is not None:
+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID:
+ if not isinstance(workflow_parameter.default_value, str):
+ raise InvalidCredentialId(
+ f"<non-string value of type {type(workflow_parameter.default_value).__name__}>"
+ )
+ workflow_parameter_values.append((workflow_parameter, workflow_parameter.default_value))
+ else:
+ missing_parameters.append(workflow_parameter.key)
+
+ if missing_parameters:
+ missing_list = ", ".join(sorted(missing_parameters))
+ raise MissingValueForParameter(
+ parameter_key=missing_list,
+ workflow_id=workflow.workflow_permanent_id,
workflow_run_id=workflow_run.workflow_run_id,
- batch_error=str(batch_error),
)
- except Exception as e:
- LOG.exception(
- f"Error while setting up workflow run {workflow_run.workflow_run_id}",
- workflow_run_id=workflow_run.workflow_run_id,
- )
- failure_reason = f"Setup workflow failed. failure reason: {get_user_facing_exception_message(e)}"
+ await self._validate_credential_ids(
+ [
+ value
+ for (workflow_parameter, value) in workflow_parameter_values
+ if workflow_parameter.workflow_parameter_type == WorkflowParameterType.CREDENTIAL_ID
+ ],
+ organization,
+ )
- workflow_run = await self.mark_workflow_run_as_failed(
- workflow_run_id=workflow_run.workflow_run_id, failure_reason=failure_reason
- )
- raise e
+ if workflow_parameter_values:
+ try:
+ await self.create_workflow_run_parameters(
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_parameter_values=workflow_parameter_values,
+ )
+ except SQLAlchemyError as batch_error:
+ # Roll back the failed transaction so the per-parameter fallback
+ # (and any later mark_workflow_run_as_failed) can reuse the outer session.
+ await outer_session.rollback()
+ # Batch failed — retry one-by-one to identify the exact failing parameter
+ for workflow_parameter, value in workflow_parameter_values:
+ try:
+ await self.create_workflow_run_parameter(
+ workflow_run_id=workflow_run.workflow_run_id,
+ workflow_parameter=workflow_parameter,
+ value=value,
+ )
+ except SQLAlchemyError as parameter_error:
+ raise WorkflowRunParameterPersistenceError(
+ parameter_key=workflow_parameter.key,
+ workflow_id=workflow.workflow_permanent_id,
+ workflow_run_id=workflow_run.workflow_run_id,
+ reason=self._format_parameter_persistence_error(parameter_error),
+ ) from parameter_error
+ # All individual inserts succeeded — the batch failure was transient
+ LOG.warning(
+ "Batch parameter insert failed but individual inserts succeeded",
+ workflow_run_id=workflow_run.workflow_run_id,
+ batch_error=str(batch_error),
+ )
+ except Exception as e:
+ LOG.exception(
+ f"Error while setting up workflow run {workflow_run.workflow_run_id}",
+ workflow_run_id=workflow_run.workflow_run_id,
+ )
- return workflow_run
+ # Discard any failed transaction state on the shared outer session before
+ # mark_workflow_run_as_failed reuses it.
+ try:
+ await outer_session.rollback()
+ except SQLAlchemyError:
+ LOG.warning("Failed to rollback outer session during setup failure", exc_info=True)
+
+ failure_reason = f"Setup workflow failed. failure reason: {get_user_facing_exception_message(e)}"
+
+ workflow_run = await self.mark_workflow_run_as_failed(
+ workflow_run_id=workflow_run.workflow_run_id, failure_reason=failure_reason
+ )
+ raise
+
+ return workflow_run
@staticmethod
def _format_parameter_persistence_error(error: SQLAlchemyError) -> str:
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index 983c4242b..75cdfdcb1 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -218,17 +218,30 @@ async def poll_otp_value(
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
)
- max_consecutive_failures = settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES
consecutive_failures = 0
+ last_error_reason: str | None = None
while True:
await asyncio.sleep(10)
- # check timeout
if datetime.utcnow() > timeout_datetime:
+ if consecutive_failures > 0 and last_error_reason is not None:
+ LOG.warning(
+ "Polling otp value timed out while webhook was still failing",
+ consecutive_failures=consecutive_failures,
+ last_error_reason=last_error_reason,
+ )
+ raise FailedToGetTOTPVerificationCode(
+ task_id=task_id,
+ workflow_run_id=workflow_run_id,
+ workflow_id=workflow_id or workflow_permanent_id,
+ totp_verification_url=totp_verification_url,
+ totp_identifier=totp_identifier,
+ reason=last_error_reason,
+ )
LOG.warning("Polling otp value timed out")
raise NoTOTPVerificationCodeFound(
task_id=task_id,
workflow_run_id=workflow_run_id,
- workflow_id=workflow_permanent_id,
+ workflow_id=workflow_id or workflow_permanent_id,
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
)
@@ -250,21 +263,21 @@ async def poll_otp_value(
workflow_id=workflow_permanent_id,
workflow_run_id=workflow_run_id,
)
- except FailedToGetTOTPVerificationCode:
+ except FailedToGetTOTPVerificationCode as e:
consecutive_failures += 1
+ last_error_reason = e.reason
LOG.warning(
- "OTP fetch failed, will retry",
+ "OTP fetch failed, will retry until wall-clock timeout",
consecutive_failures=consecutive_failures,
- max_consecutive_failures=max_consecutive_failures,
+ last_error_reason=e.reason,
task_id=task_id,
workflow_run_id=workflow_run_id,
totp_verification_url=totp_verification_url,
totp_identifier=totp_identifier,
)
- if consecutive_failures >= max_consecutive_failures:
- raise
continue
- consecutive_failures = 0 # reset on any non-exception poll, even if otp_value is None
+ consecutive_failures = 0
+ last_error_reason = None
if otp_value:
LOG.info("Got otp value", otp_value=otp_value)
return otp_value
diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.py
index d9d04576f..7ba8dc91f 100644
--- a/tests/unit/test_otp_service.py
+++ b/tests/unit/test_otp_service.py
@@ -2,13 +2,14 @@
from __future__ import annotations
+from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pyotp
import pytest
-from skyvern.exceptions import FailedToGetTOTPVerificationCode
+from skyvern.exceptions import FailedToGetTOTPVerificationCode, NoTOTPVerificationCodeFound
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.services.otp_service import (
@@ -117,7 +118,7 @@ def _mock_org_token() -> MagicMock:
class TestPollOtpValueRetry:
- """poll_otp_value should tolerate transient FailedToGetTOTPVerificationCode up to max consecutive failures."""
+ """poll_otp_value retries fetch failures across the full wall-clock timeout window."""
@pytest.mark.asyncio
@patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)
@@ -127,9 +128,8 @@ class TestPollOtpValueRetry:
async def test_retries_on_transient_failure_then_succeeds(
self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock
) -> None:
- """Fail twice, then return OTP on third attempt. Counter resets, OTP is returned."""
+ """Fail twice, then return OTP on third attempt."""
mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3
mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
otp = OTPValue(value="123456")
@@ -148,58 +148,95 @@ class TestPollOtpValueRetry:
assert mock_fetch.call_count == 3
@pytest.mark.asyncio
- @patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)
- @patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock)
- @patch("skyvern.services.otp_service.app")
- @patch("skyvern.services.otp_service.settings")
- async def test_raises_after_max_consecutive_failures(
- self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock
- ) -> None:
- """After N consecutive failures, re-raise FailedToGetTOTPVerificationCode."""
- mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3
- mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
-
- mock_fetch.side_effect = FailedToGetTOTPVerificationCode(reason="connection error")
-
- with pytest.raises(FailedToGetTOTPVerificationCode):
- await poll_otp_value(
+ async def test_does_not_short_circuit_during_extended_outage(self) -> None:
+ """Persistent webhook errors must not exit polling before the wall-clock timeout fires (SKY-9553)."""
+ base = datetime(2026, 1, 1, 12, 0, 0)
+ utcnow_returns = [base] + [base + timedelta(seconds=60 * (i + 1)) for i in range(8)]
+
+ with (
+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),
+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.settings") as mock_settings,
+ patch("skyvern.services.otp_service.datetime") as mock_datetime,
+ ):
+ mock_datetime.utcnow.side_effect = utcnow_returns
+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
+
+ otp = OTPValue(value="424242")
+ mock_fetch.side_effect = [
+ *([FailedToGetTOTPVerificationCode(reason="connection refused")] * 7),
+ otp,
+ ]
+
+ result = await poll_otp_value(
organization_id="o_test",
task_id="tsk_test",
totp_verification_url="https://example.com/mfa",
)
- assert mock_fetch.call_count == 3
+ assert result == otp
+ assert mock_fetch.call_count == 8
@pytest.mark.asyncio
- @patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock)
- @patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock)
- @patch("skyvern.services.otp_service.app")
- @patch("skyvern.services.otp_service.settings")
- async def test_counter_resets_after_success(
- self, mock_settings: MagicMock, mock_app: MagicMock, mock_fetch: AsyncMock, mock_sleep: AsyncMock
- ) -> None:
- """Fail twice, succeed (None), fail twice again, then return OTP. Counter resets between failure bursts."""
- mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
- mock_settings.VERIFICATION_CODE_POLLING_MAX_CONSECUTIVE_FAILURES = 3
- mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
+ async def test_raises_failed_with_reason_when_timeout_during_failure_streak(self) -> None:
+ """When the wall-clock timeout fires while the webhook is still failing, surface the underlying reason."""
+ base = datetime(2026, 1, 1, 12, 0, 0)
+ utcnow_returns = [
+ base, # start_datetime
+ base + timedelta(seconds=30), # iter 1: not timed out, fetch fails
+ base + timedelta(seconds=60), # iter 2: not timed out, fetch fails
+ base + timedelta(minutes=16), # iter 3: timed out
+ ]
- otp = OTPValue(value="654321")
- mock_fetch.side_effect = [
- FailedToGetTOTPVerificationCode(reason="err"),
- FailedToGetTOTPVerificationCode(reason="err"),
- None, # success (no OTP yet, but resets counter)
- FailedToGetTOTPVerificationCode(reason="err"),
- FailedToGetTOTPVerificationCode(reason="err"),
- otp,
+ with (
+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),
+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.settings") as mock_settings,
+ patch("skyvern.services.otp_service.datetime") as mock_datetime,
+ ):
+ mock_datetime.utcnow.side_effect = utcnow_returns
+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
+ mock_fetch.side_effect = FailedToGetTOTPVerificationCode(reason="connection refused")
+
+ with pytest.raises(FailedToGetTOTPVerificationCode) as exc_info:
+ await poll_otp_value(
+ organization_id="o_test",
+ task_id="tsk_test",
+ totp_verification_url="https://example.com/mfa",
+ )
+ assert exc_info.value.reason == "connection refused"
+
+ @pytest.mark.asyncio
+ async def test_raises_no_otp_at_timeout_when_polls_were_clean(self) -> None:
+ """When polling timed out without webhook errors (just no OTP yet), raise NoTOTPVerificationCodeFound."""
+ base = datetime(2026, 1, 1, 12, 0, 0)
+ utcnow_returns = [
+ base,
+ base + timedelta(seconds=30),
+ base + timedelta(minutes=16),
]
- result = await poll_otp_value(
- organization_id="o_test",
- task_id="tsk_test",
- totp_verification_url="https://example.com/mfa",
- )
- assert result == otp
- assert mock_fetch.call_count == 6
+ with (
+ patch("skyvern.services.otp_service.asyncio.sleep", new_callable=AsyncMock),
+ patch("skyvern.services.otp_service._get_otp_value_from_url", new_callable=AsyncMock) as mock_fetch,
+ patch("skyvern.services.otp_service.app") as mock_app,
+ patch("skyvern.services.otp_service.settings") as mock_settings,
+ patch("skyvern.services.otp_service.datetime") as mock_datetime,
+ ):
+ mock_datetime.utcnow.side_effect = utcnow_returns
+ mock_settings.VERIFICATION_CODE_POLLING_TIMEOUT_MINS = 15
+ mock_app.DATABASE.organizations.get_valid_org_auth_token = AsyncMock(return_value=_mock_org_token())
+ mock_fetch.return_value = None
+
+ with pytest.raises(NoTOTPVerificationCodeFound):
+ await poll_otp_value(
+ organization_id="o_test",
+ task_id="tsk_test",
+ totp_verification_url="https://example.com/mfa",
+ )
class _FakeWorkflowRunContext:
diff --git a/tests/unit/test_workflow_run_parameter_batching.py b/tests/unit/test_workflow_run_parameter_batching.py
index c40ed7509..89c2a3c27 100644
--- a/tests/unit/test_workflow_run_parameter_batching.py
+++ b/tests/unit/test_workflow_run_parameter_batching.py
@@ -367,3 +367,95 @@ async def test_setup_workflow_run_preserves_parent_loop_state_when_replacing_con
assert current_context.root_workflow_run_id == "wr_root"
assert current_context.loop_internal_state == loop_state
assert current_context.loop_internal_state is not loop_state
+
+
+async def test_setup_workflow_run_opens_one_outer_session() -> None:
+ """setup_workflow_run wraps its body in exactly one outer ``Session()`` context."""
+
+ params = [_make_workflow_parameter("k", default_value="v")]
+ service, organization, _ = _make_service_with_mocks(workflow_parameters=params)
+
+ session_open_count = 0
+
+ class _Counter:
+ async def __aenter__(self) -> _Counter:
+ nonlocal session_open_count
+ session_open_count += 1
+ return self
+
+ async def __aexit__(self, *_a: object) -> None:
+ return None
+
+ async def rollback(self) -> None:
+ return None
+
+ request = WorkflowRequestBody(data={"k": "v"})
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflow_runs.Session = lambda: _Counter()
+ await service.setup_workflow_run(
+ request_id="req_test",
+ workflow_request=request,
+ workflow_permanent_id="wpid_test",
+ organization=organization,
+ )
+
+ assert session_open_count == 1, (
+ f"Expected exactly one outer Session() open in setup_workflow_run, got {session_open_count}"
+ )
+
+
+async def test_setup_workflow_run_rolls_back_outer_session_on_batch_failure() -> None:
+ """When the batched parameter insert raises, the outer session must be rolled back
+ before the per-parameter fallback reuses it - otherwise the fallback runs on a
+ session whose transaction is in error state."""
+
+ params = [
+ _make_workflow_parameter("a", default_value="1"),
+ _make_workflow_parameter("b", default_value="2"),
+ ]
+ batch_error = IntegrityError("INSERT", {}, Exception("constraint failed"))
+
+ fallback_call_index: list[int] = []
+ rollback_index: list[int] = []
+ call_counter = {"n": 0}
+
+ async def _fallback_insert(*, workflow_run_id: str, workflow_parameter: WorkflowParameter, value: object) -> None:
+ call_counter["n"] += 1
+ fallback_call_index.append(call_counter["n"])
+
+ service, organization, _ = _make_service_with_mocks(
+ workflow_parameters=params,
+ batch_side_effect=batch_error,
+ )
+ service.create_workflow_run_parameter = AsyncMock(side_effect=_fallback_insert) # type: ignore[method-assign]
+
+ class _Session:
+ async def __aenter__(self) -> _Session:
+ return self
+
+ async def __aexit__(self, *_a: object) -> None:
+ return None
+
+ async def rollback(self) -> None:
+ call_counter["n"] += 1
+ rollback_index.append(call_counter["n"])
+
+ request = WorkflowRequestBody(data={"a": "1", "b": "2"})
+ with patch("skyvern.forge.sdk.workflow.service.app") as mock_app:
+ mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
+ mock_app.DATABASE.workflow_runs.Session = lambda: _Session()
+ await service.setup_workflow_run(
+ request_id="req_test",
+ workflow_request=request,
+ workflow_permanent_id="wpid_test",
+ organization=organization,
+ )
+
+ assert rollback_index, "Expected rollback() to be called on the outer session after batch failure"
+ assert fallback_call_index, "Expected the fallback path to run after rollback"
+ assert rollback_index[0] < fallback_call_index[0], (
+ f"rollback must precede fallback insert; got rollback at {rollback_index} fallback at {fallback_call_index}"
+ )