Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-48] fix TOTP verification UI: timer, expired code retry, 2FA banner suppression, and goal-text extraction (#4860)
Aaron Perez32ad9d369483
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex a738c809a..e16ad0b86 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -104,7 +104,11 @@ from skyvern.schemas.runs import CUA_ENGINES, RunEnginefrom skyvern.schemas.steps import AgentStepOutputfrom skyvern.services import run_service, service_utilsfrom skyvern.services.action_service import get_action_history-from skyvern.services.otp_service import poll_otp_value, try_generate_totp_from_credential+from skyvern.services.otp_service import (+ extract_totp_from_navigation_inputs,+ poll_otp_value,+ try_generate_totp_from_credential,+)from skyvern.utils.image_resizer import Resolutionfrom skyvern.utils.prompt_engine import MaxStepsReasonResponse, load_prompt_with_elementsfrom skyvern.webeye.actions.action_types import ActionType@@ -4524,9 +4528,16 @@ class ForgeAgent:should_enter_verification_code = json_response.get("should_enter_verification_code")if place_to_enter_verification_code and should_enter_verification_code and task.organization_id:LOG.info("Need verification code")- # Try credential TOTP first (highest priority, doesn't need totp_url/totp_identifier)+ # 1. Check navigation payload first for inline OTP.+ otp_value = extract_totp_from_navigation_inputs(task.navigation_payload)+ if otp_value:+ # Code was already in the payload the LLM saw, so its actions are already correct.+ # No need to re-prompt, generate from credentials, or poll.+ return json_response++ # 2. Then try to generate TOTP from credential if payload has no OTP.otp_value = try_generate_totp_from_credential(task.workflow_run_id)- # Fall back to webhook/totp_identifier+ # 3. Lastly, poll for OTP if organization has config and no OTP was found yet.if not otp_value:workflow_id = workflow_permanent_id = Noneif task.workflow_run_id:@@ -4543,9 +4554,12 @@ class ForgeAgent:totp_verification_url=task.totp_verification_url,totp_identifier=task.totp_identifier,)+if not otp_value or otp_value.get_otp_type() != OTPType.TOTP:return json_response+ # Store the code so _build_navigation_payload injects it, then re-prompt+ # the LLM so it generates actions that type the real code.current_context = skyvern_context.ensure_context()current_context.totp_codes[task.task_id] = otp_value.valuediff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 3b426dcb1..2c63600b1 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -1,4 +1,5 @@import asyncio+import refrom dataclasses import dataclassfrom datetime import datetime, timedelta@@ -17,6 +18,10 @@ from skyvern.forge.sdk.notification.factory import NotificationRegistryFactoryfrom skyvern.forge.sdk.schemas.totp_codes import OTPTypeLOG = structlog.get_logger()+_MFA_PARAMETER_KEY_HINTS = ("mfa", "otp", "verification")+_NON_ALNUM_PATTERN = re.compile(r"[^a-z0-9]")++MFANavigationPayload = dict | list | str | None@dataclass(slots=True)@@ -54,6 +59,62 @@ class OTPResultParsedByLLM(BaseModel):otp_value: str | None = Field(None, description="The OTP value.")+def _is_mfa_like_parameter_key(key: object) -> bool:+ """Return True when a payload key appears to represent an MFA/OTP parameter."""+ normalized_key = _NON_ALNUM_PATTERN.sub("", str(key).lower())+ return any(hint in normalized_key for hint in _MFA_PARAMETER_KEY_HINTS)+++def extract_totp_from_navigation_inputs(navigation_payload: MFANavigationPayload) -> OTPValue | None:+ """Extract TOTP from runtime navigation inputs.++ Runtime inline OTP extraction is intentionally payload-only.+ """+ # Early return if payload is not a traversable container+ if not isinstance(navigation_payload, (dict, list)):+ return None++ traversal_stack: list[dict | list | str] = [navigation_payload]+ visited_container_ids: set[int] = set()++ while traversal_stack:+ current_item = traversal_stack.pop()++ # Only save OTP if a string+ if isinstance(current_item, str):+ return OTPValue(value=current_item, type=OTPType.TOTP)++ # Check if we've already visited this container to avoid cycles+ current_id = id(current_item)+ if current_id in visited_container_ids:+ continue+ visited_container_ids.add(current_id)++ # For lists, add all nested containers to the stack for further traversal+ if isinstance(current_item, list):+ for item in reversed(current_item):+ if isinstance(item, (dict, list)):+ traversal_stack.append(item)+ continue++ # For dictionaries, process key-value pairs+ for key, value in reversed(list(current_item.items())):+ # Add nested containers (dicts/lists) to the stack regardless of key name+ if isinstance(value, (dict, list)):+ traversal_stack.append(value)+ # Only process string values if the key suggests it's MFA-related+ if not _is_mfa_like_parameter_key(key):+ continue+ if not isinstance(value, str):+ continue+ # If the value is a non-empty string under an MFA-like key, add it for evaluation+ candidate_value = value.strip()+ if candidate_value:+ traversal_stack.append(candidate_value)+ # No OTP code found after traversing the entire payload+ return None++async def parse_otp_login(content: str,organization_id: str,diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 2129fb6c1..574f8d955 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -14,7 +14,11 @@ from skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.models import Stepfrom skyvern.forge.sdk.schemas.tasks import Taskfrom skyvern.forge.sdk.schemas.totp_codes import OTPType-from skyvern.services.otp_service import poll_otp_value, try_generate_totp_from_credential+from skyvern.services.otp_service import (+ extract_totp_from_navigation_inputs,+ poll_otp_value,+ try_generate_totp_from_credential,+)from skyvern.utils.image_resizer import Resolution, scale_coordinatesfrom skyvern.webeye.actions.action_types import ActionTypefrom skyvern.webeye.actions.actions import (@@ -913,9 +917,12 @@ async def generate_cua_fallback_actions()elif skyvern_action_type == "get_verification_code":- # Try credential TOTP first (highest priority, doesn't need totp_url/totp_identifier)- otp_value = try_generate_totp_from_credential(task.workflow_run_id)- # Fall back to webhook/totp_identifier+ # 1. Check navigation payload first for inline OTP.+ otp_value = extract_totp_from_navigation_inputs(task.navigation_payload)+ # 2. Then try to generate TOTP from credential if payload has no OTP.+ if not otp_value:+ otp_value = try_generate_totp_from_credential(task.workflow_run_id)+ # 3. Lastly, poll for OTP if organization has config and no OTP was found yet.if not otp_value and task.organization_id:LOG.info("Getting verification code for CUA",@@ -940,6 +947,7 @@ async def generate_cua_fallback_actions(reasoning_suffix = f"Failed to get verification code. Reason: {e.reason}"reasoning = f"{reasoning}. {reasoning_suffix}" if reasoning else reasoning_suffix+ # OTP value obtained - use it as verification codeif otp_value and otp_value.get_otp_type() == OTPType.TOTP:verification_code = otp_value.valuereasoning = reasoning or f"Received verification code: {verification_code}"@@ -948,6 +956,7 @@ async def generate_cua_fallback_actions(reasoning=reasoning,intention=reasoning,)+ # Terminate the task since OTP is necessary but wasn't found/generatedelse:action = TerminateAction(reasoning=reasoning,diff --git a/tests/unit_tests/test_otp_no_config.py b/tests/unit_tests/test_otp_no_config.pyindex 33fa57bf8..930ed19e2 100644--- a/tests/unit_tests/test_otp_no_config.py+++ b/tests/unit_tests/test_otp_no_config.py@@ -9,9 +9,14 @@ from skyvern.forge.agent import ForgeAgentfrom skyvern.forge.sdk.db.agent_db import AgentDBfrom skyvern.forge.sdk.notification.local import LocalNotificationRegistryfrom skyvern.forge.sdk.routes.credentials import send_totp_code-from skyvern.forge.sdk.schemas.totp_codes import TOTPCodeCreate+from skyvern.forge.sdk.schemas.totp_codes import OTPType, TOTPCodeCreatefrom skyvern.schemas.runs import RunEngine-from skyvern.services.otp_service import OTPValue, _get_otp_value_by_run, poll_otp_value+from skyvern.services.otp_service import (+ OTPValue,+ _get_otp_value_by_run,+ extract_totp_from_navigation_inputs,+ poll_otp_value,+)@pytest.mark.asyncio@@ -72,6 +77,44 @@ async def test_get_otp_value_by_run_returns_none_when_no_codes():assert result is None+def test_extract_totp_from_navigation_inputs_prefers_payload_code_over_goal_text():+ """Payload alias code should be used even if goal text contains another code."""+ otp_value = extract_totp_from_navigation_inputs(+ {"mfaChoice": "520265"},+ )++ assert otp_value is not None+ assert otp_value.value == "520265"+ assert otp_value.get_otp_type() == OTPType.TOTP+++def test_extract_totp_from_navigation_inputs_ignores_navigation_goal_when_payload_missing():+ """Goal text alone should not produce inline OTP in payload-only mode."""+ otp_value = extract_totp_from_navigation_inputs(+ None,+ )++ assert otp_value is None+++def test_extract_totp_from_navigation_inputs_ignores_goal_input_action_when_payload_missing():+ """Input-action goal text should be ignored in payload-only mode."""+ otp_value = extract_totp_from_navigation_inputs(+ {},+ )++ assert otp_value is None+++def test_extract_totp_from_navigation_inputs_no_code_anywhere():+ """No code in payload or goal should return None."""+ otp_value = extract_totp_from_navigation_inputs(+ None,+ )++ assert otp_value is None++# === Task 3: poll_otp_value without identifier ===@@ -136,6 +179,76 @@ async def test_handle_potential_OTP_actions_without_totp_config():mock_handler.assert_called_once()+async def test_handle_potential_verification_code_uses_navigation_payload_and_skips_poll():+ """When payload includes MFA code, should consume it and skip poll_otp_value."""+ from skyvern.forge import agent as forge_agent_module++ agent = ForgeAgent.__new__(ForgeAgent)++ task = MagicMock()+ task.organization_id = "org_1"+ task.totp_verification_url = None+ task.totp_identifier = None+ task.task_id = "tsk_payload_code"+ task.workflow_run_id = None+ task.navigation_payload = {"mfaChoice": "520265"}+ task.llm_key = None++ step = MagicMock()+ step.step_id = "step_1"+ step.order = 0++ scraped_page = MagicMock()+ scraped_page.screenshots = []+ browser_state = MagicMock()+ json_response = {+ "should_enter_verification_code": True,+ "place_to_enter_verification_code": "input#otp-code",+ "actions": [+ {"action_type": "INPUT_TEXT", "id": "AAAb", "reasoning": "Enter MFA code", "text": "520265"},+ ],+ }++ original_app_inst = object.__getattribute__(forge_agent_module.app, "_inst")+ object.__setattr__(forge_agent_module.app, "_inst", MagicMock(LLM_API_HANDLER=AsyncMock()))+ try:+ with (+ patch("skyvern.forge.agent.try_generate_totp_from_credential") as mock_credential_totp,+ patch("skyvern.forge.agent.poll_otp_value", new_callable=AsyncMock) as mock_poll,+ patch("skyvern.forge.agent.skyvern_context") as mock_skyvern_context,+ patch("skyvern.forge.agent.service_utils.is_cua_task", new_callable=AsyncMock, return_value=False),+ patch(+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",+ return_value=AsyncMock(return_value={"actions": []}),+ ),+ patch.object(+ agent,+ "_build_extract_action_prompt",+ new_callable=AsyncMock,+ return_value=("prompt", False, "extract-actions"),+ ),+ ):+ mock_context = MagicMock()+ mock_context.totp_codes = {}+ mock_skyvern_context.ensure_context.return_value = mock_context+ mock_skyvern_context.current.return_value = mock_context++ await agent.handle_potential_verification_code(+ task=task,+ step=step,+ scraped_page=scraped_page,+ browser_state=browser_state,+ json_response=json_response,+ )+ finally:+ object.__setattr__(forge_agent_module.app, "_inst", original_app_inst)++ mock_credential_totp.assert_not_called()+ mock_poll.assert_not_called()+ # Early return path: code already visible to LLM in payload, no context storage or re-prompt needed++@pytest.mark.asyncioasync def test_handle_potential_OTP_actions_skips_magic_link_without_totp_config():"""Magic links should still require TOTP config."""@@ -504,6 +617,85 @@ async def test_poll_otp_value_publishes_required_event_for_task():assert resolved["task_id"] == "tsk_1"+async def test_reprompt_with_verification_code_calls_llm():+ """When poll_otp_value returns a TOTP code, handle_potential_verification_code should+ store the code in context, re-prompt the LLM with verification_code_check=False, and+ return the LLM's new response."""+ from skyvern.forge import agent as forge_agent_module++ agent = ForgeAgent.__new__(ForgeAgent)++ task = MagicMock()+ task.organization_id = "org_1"+ task.totp_verification_url = None+ task.totp_identifier = None+ task.task_id = "tsk_reprompt"+ task.workflow_run_id = None+ task.navigation_payload = None+ task.llm_key = None++ step = MagicMock()+ step.step_id = "step_1"+ step.order = 0++ scraped_page = MagicMock()+ scraped_page.screenshots = []+ browser_state = MagicMock()+ json_response = {+ "should_enter_verification_code": True,+ "place_to_enter_verification_code": "input#otp-code",+ "actions": [],+ }++ llm_response = {"actions": [{"action_type": "input_text", "text": "999888"}]}++ original_app_inst = object.__getattribute__(forge_agent_module.app, "_inst")+ object.__setattr__(forge_agent_module.app, "_inst", MagicMock(LLM_API_HANDLER=AsyncMock()))+ try:+ with (+ patch("skyvern.forge.agent.extract_totp_from_navigation_inputs", return_value=None),+ patch("skyvern.forge.agent.try_generate_totp_from_credential", return_value=None),+ patch(+ "skyvern.forge.agent.poll_otp_value",+ new_callable=AsyncMock,+ return_value=OTPValue(value="999888", type=OTPType.TOTP),+ ),+ patch("skyvern.forge.agent.skyvern_context") as mock_skyvern_context,+ patch("skyvern.forge.agent.service_utils.is_cua_task", new_callable=AsyncMock, return_value=False),+ patch(+ "skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",+ return_value=AsyncMock(return_value=llm_response),+ ),+ patch.object(+ agent,+ "_build_extract_action_prompt",+ new_callable=AsyncMock,+ return_value=("prompt", False, "extract-actions"),+ ) as mock_build,+ ):+ mock_context = MagicMock()+ mock_context.totp_codes = {}+ mock_skyvern_context.ensure_context.return_value = mock_context+ mock_skyvern_context.current.return_value = mock_context++ result = await agent.handle_potential_verification_code(+ task=task,+ step=step,+ scraped_page=scraped_page,+ browser_state=browser_state,+ json_response=json_response,+ )+ finally:+ object.__setattr__(forge_agent_module.app, "_inst", original_app_inst)++ mock_build.assert_called_once()+ _, kwargs = mock_build.call_args+ assert kwargs["verification_code_check"] is False+ assert mock_context.totp_codes["tsk_reprompt"] == "999888"+ assert result == llm_response++@pytest.mark.asyncioasync def test_poll_otp_value_publishes_required_event_for_workflow_run():"""poll_otp_value should publish verification_code_required when workflow run waiting state is set."""