Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12590): treat off-schema OTP parser output as 400, not retryable 502 (#7509)
LawyZheng91db8f4cf489
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 27729a220..3bd743d1d 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -5,7 +5,7 @@ from datetime import datetime, timedeltafrom typing import TYPE_CHECKING, Anyimport structlog-from pydantic import BaseModel, Field+from pydantic import BaseModel, Field, ValidationErrorif TYPE_CHECKING:from skyvern.forge.sdk.schemas.tasks import Task@@ -86,7 +86,18 @@ async def parse_otp_login(resp = await app.SECONDARY_LLM_API_HANDLER(prompt=prompt, prompt_name="parse-otp-login", organization_id=organization_id)- otp_result = OTPResultParsedByLLM.model_validate(resp)+ try:+ otp_result = OTPResultParsedByLLM.model_validate(resp)+ except ValidationError as e:+ # Off-schema JSON from a successful LLM call is unparseable content (-> caller's 400+ # path), not a backend outage (-> 502). Log the exception type only; the raw response+ # can carry OTP/PII.+ LOG.warning(+ "OTP login parser returned off-schema response",+ organization_id=organization_id,+ exception_type=type(e).__name__,+ )+ return NoneLOG.info("OTP Login Parser Response",enforced_otp_type=enforced_otp_type,diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.pyindex 982b4c4b5..eb4fbaf05 100644--- a/tests/unit/test_credentials_totp_route_logging.py+++ b/tests/unit/test_credentials_totp_route_logging.py@@ -122,3 +122,33 @@ async def test_send_totp_code_parser_exception_log_redacts_raw_exception_contextassert error_log["exception_type"] == "RuntimeError"assert "content" not in error_log_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "246810")+++async def test_send_totp_code_off_schema_llm_response_yields_400_not_502(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from skyvern.services import otp_service++ raw_identifier = "[email protected]"+ raw_content = "Long email body requesting OTP 424242 for [email protected]."+ create_otp_code = AsyncMock()+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))++ async def off_schema_handler(*_args: object, **_kwargs: object) -> dict[str, object]:+ return {"otp_type": "totp", "otp_value": "424242"}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", off_schema_handler, raising=False)++ with structlog.testing.capture_logs() as logs:+ with pytest.raises(HTTPException) as exc_info:+ await credentials.send_totp_code(+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert exc_info.value.status_code == 400+ assert exc_info.value.detail == "Failed to parse otp login"+ create_otp_code.assert_not_awaited()+ _assert_raw_values_not_logged(logs, raw_identifier, raw_content, "424242")diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.pyindex aafe2e99d..29410a329 100644--- a/tests/unit/test_otp_service.py+++ b/tests/unit/test_otp_service.py@@ -471,6 +471,52 @@ class TestParseOtpLogin:assert parser["otp_value_found"] is Trueassert parser["otp_length"] == len(raw)+ @pytest.mark.asyncio+ async def test_off_schema_response_returns_none_without_leaking_payload(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from skyvern.services import otp_service++ raw_content = "Long email body requesting OTP 424242 to finish sign in."+ off_schema_resp = {"otp_type": "totp", "otp_value": "424242"}++ async def fake_handler(*_args: object, **_kwargs: object) -> dict[str, object]:+ return off_schema_resp++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", fake_handler, raising=False)++ with structlog.testing.capture_logs() as logs:+ result = await parse_otp_login(content=raw_content, organization_id="o_test")++ assert result is None++ warning = next((r for r in logs if r.get("exception_type") == "ValidationError"), None)+ assert warning is not None+ assert warning["organization_id"] == "o_test"+ assert "content" not in warning+ assert "resp" not in warning+ for record in logs:+ assert raw_content not in repr(record)+ assert "424242" not in repr(record)+ for value in record.values():+ assert raw_content not in str(value)+ assert "424242" not in str(value)++ @pytest.mark.asyncio+ async def test_backend_provider_error_propagates_not_swallowed(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError+ from skyvern.services import otp_service++ async def failing_handler(*_args: object, **_kwargs: object) -> dict[str, object]:+ raise LLMProviderError("gemini-2.5-flash")++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", failing_handler, raising=False)++ with pytest.raises(LLMProviderError):+ await parse_otp_login(content="raw email body that exceeds ten chars", organization_id="o_test")+class TestPollOtpValueRetry:"""poll_otp_value retries fetch failures across the full wall-clock timeout window."""