Skip to content

Automate browser based workflows with AI

fix(SKY-12590): treat off-schema OTP parser output as 400, not retryable 502 (#7509)

LawyZheng91db8f4cf489
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index 27729a220..3bd743d1d 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -5,7 +5,7 @@ from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any
import structlog
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
if TYPE_CHECKING:
from skyvern.forge.sdk.schemas.tasks import Task
@@ -86,7 +86,18 @@ async def parse_otp_login(
resp = await app.SECONDARY_LLM_API_HANDLER(
prompt=prompt, prompt_name="parse-otp-login", organization_id=organization_id
)
- otp_result = OTPResultParsedByLLM.model_validate(resp)
+ try:
+ otp_result = OTPResultParsedByLLM.model_validate(resp)
+ except ValidationError as e:
+ # Off-schema JSON from a successful LLM call is unparseable content (-> caller's 400
+ # path), not a backend outage (-> 502). Log the exception type only; the raw response
+ # can carry OTP/PII.
+ LOG.warning(
+ "OTP login parser returned off-schema response",
+ organization_id=organization_id,
+ exception_type=type(e).__name__,
+ )
+ return None
LOG.info(
"OTP Login Parser Response",
enforced_otp_type=enforced_otp_type,
diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.py
index 982b4c4b5..eb4fbaf05 100644
--- a/tests/unit/test_credentials_totp_route_logging.py
+++ b/tests/unit/test_credentials_totp_route_logging.py
@@ -122,3 +122,33 @@ async def test_send_totp_code_parser_exception_log_redacts_raw_exception_context
assert error_log["exception_type"] == "RuntimeError"
assert "content" not in error_log
_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "246810")
+
+
+async def test_send_totp_code_off_schema_llm_response_yields_400_not_502(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.services import otp_service
+
+ raw_identifier = "[email protected]"
+ raw_content = "Long email body requesting OTP 424242 for [email protected]."
+ create_otp_code = AsyncMock()
+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code))
+
+ async def off_schema_handler(*_args: object, **_kwargs: object) -> dict[str, object]:
+ return {"otp_type": "totp", "otp_value": "424242"}
+
+ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")
+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", off_schema_handler, raising=False)
+
+ with structlog.testing.capture_logs() as logs:
+ with pytest.raises(HTTPException) as exc_info:
+ await credentials.send_totp_code(
+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),
+ curr_org=SimpleNamespace(organization_id="o_test"),
+ )
+
+ assert exc_info.value.status_code == 400
+ assert exc_info.value.detail == "Failed to parse otp login"
+ create_otp_code.assert_not_awaited()
+ _assert_raw_values_not_logged(logs, raw_identifier, raw_content, "424242")
diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.py
index aafe2e99d..29410a329 100644
--- a/tests/unit/test_otp_service.py
+++ b/tests/unit/test_otp_service.py
@@ -471,6 +471,52 @@ class TestParseOtpLogin:
assert parser["otp_value_found"] is True
assert parser["otp_length"] == len(raw)
+ @pytest.mark.asyncio
+ async def test_off_schema_response_returns_none_without_leaking_payload(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.services import otp_service
+
+ raw_content = "Long email body requesting OTP 424242 to finish sign in."
+ off_schema_resp = {"otp_type": "totp", "otp_value": "424242"}
+
+ async def fake_handler(*_args: object, **_kwargs: object) -> dict[str, object]:
+ return off_schema_resp
+
+ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")
+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", fake_handler, raising=False)
+
+ with structlog.testing.capture_logs() as logs:
+ result = await parse_otp_login(content=raw_content, organization_id="o_test")
+
+ assert result is None
+
+ warning = next((r for r in logs if r.get("exception_type") == "ValidationError"), None)
+ assert warning is not None
+ assert warning["organization_id"] == "o_test"
+ assert "content" not in warning
+ assert "resp" not in warning
+ for record in logs:
+ assert raw_content not in repr(record)
+ assert "424242" not in repr(record)
+ for value in record.values():
+ assert raw_content not in str(value)
+ assert "424242" not in str(value)
+
+ @pytest.mark.asyncio
+ async def test_backend_provider_error_propagates_not_swallowed(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
+ from skyvern.services import otp_service
+
+ async def failing_handler(*_args: object, **_kwargs: object) -> dict[str, object]:
+ raise LLMProviderError("gemini-2.5-flash")
+
+ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")
+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", failing_handler, raising=False)
+
+ with pytest.raises(LLMProviderError):
+ await parse_otp_login(content="raw email body that exceeds ten chars", organization_id="o_test")
+
class TestPollOtpValueRetry:
"""poll_otp_value retries fetch failures across the full wall-clock timeout window."""