Skip to content

Automate browser based workflows with AI

feat: enable raw OTP fallback by default, remove TOTP_RAW_FALLBACK_ENABLED (#7866)

LawyZhengb670a1ec54d8
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.json
index 059ffc007..59cc70096 100644
--- a/docs/api-reference/openapi.json
+++ b/docs/api-reference/openapi.json
@@ -4247,16 +4247,6 @@
}
}
},
- "202": {
- "description": "Raw OTP content accepted for later parsing",
- "content": {
- "application/json": {
- "schema": {
- "$ref": "#/components/schemas/RawTOTPCodeAccepted"
- }
- }
- }
- },
"402": {
"description": "Insufficient credits to parse OTP content"
},
diff --git a/skyvern/config.py b/skyvern/config.py
index 50e9a682a..f4591619d 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -601,7 +601,6 @@ class Settings(BaseSettings):
# TOTP Settings
TOTP_LIFESPAN_MINUTES: int = 10
- TOTP_RAW_FALLBACK_ENABLED: bool = False
TOTP_RAW_CONTENT_MAX_LENGTH: int = 65536
VERIFICATION_CODE_INITIAL_WAIT_TIME_SECS: int = 40
VERIFICATION_CODE_POLLING_TIMEOUT_MINS: int = 15
diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.py
index a85e14b3c..36f12761b 100644
--- a/skyvern/forge/sdk/routes/credentials.py
+++ b/skyvern/forge/sdk/routes/credentials.py
@@ -412,19 +412,13 @@ async def fetch_credential_item_background(item_id: str) -> None:
@legacy_base_router.post(
"/totp",
- responses={
- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},
- 402: {"description": "Insufficient credits to parse OTP content"},
- },
+ responses={402: {"description": "Insufficient credits to parse OTP content"}},
)
@legacy_base_router.post("/totp/", include_in_schema=False)
@base_router.post(
"/credentials/totp",
response_model=TOTPCode,
- responses={
- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},
- 402: {"description": "Insufficient credits to parse OTP content"},
- },
+ responses={402: {"description": "Insufficient credits to parse OTP content"}},
summary="Send TOTP code",
description="Forward a TOTP (2FA, MFA) email or sms message containing the code to Skyvern. This endpoint stores the code in database so that Skyvern can use it while running tasks/workflows.",
tags=["Credentials"],
@@ -443,10 +437,7 @@ async def fetch_credential_item_background(item_id: str) -> None:
@base_router.post(
"/credentials/totp/",
response_model=TOTPCode,
- responses={
- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},
- 402: {"description": "Insufficient credits to parse OTP content"},
- },
+ responses={402: {"description": "Insufficient credits to parse OTP content"}},
include_in_schema=False,
)
async def send_totp_code(
@@ -490,9 +481,7 @@ async def send_totp_code(
otp_value = None
parse_exception_type_name = type(e).__name__
- if otp_parse_skipped_for_insufficient_credits and (
- not settings.TOTP_RAW_FALLBACK_ENABLED or len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH
- ):
+ if otp_parse_skipped_for_insufficient_credits and len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH:
raise HTTPException(
status_code=status.HTTP_402_PAYMENT_REQUIRED,
detail="Insufficient credits to parse OTP content",
@@ -519,17 +508,6 @@ async def send_totp_code(
)
if not otp_value:
- if not settings.TOTP_RAW_FALLBACK_ENABLED:
- LOG.error(
- "Failed to parse otp login",
- organization_id=curr_org.organization_id,
- totp_identifier=redacted_totp_identifier,
- task_id=data.task_id,
- workflow_id=data.workflow_id,
- workflow_run_id=data.workflow_run_id,
- content_length=len(data.content),
- )
- raise HTTPException(status_code=400, detail="Failed to parse otp login")
if len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH:
raise HTTPException(status_code=400, detail="Failed to parse otp login")
raw_row = await app.DATABASE.otp.create_raw_otp_code(
@@ -549,9 +527,9 @@ async def send_totp_code(
totp_code_id=raw_row.totp_code_id,
content_length=len(data.content),
)
- pending = RawTOTPCodeAccepted(totp_code_id=raw_row.totp_code_id)
+ pending = RawTOTPCodeAccepted.from_raw_row(raw_row)
return Response( # type: ignore[return-value]
- content=pending.model_dump_json(), status_code=202, media_type="application/json"
+ content=pending.model_dump_json(), status_code=200, media_type="application/json"
)
return await app.DATABASE.otp.create_otp_code(
diff --git a/skyvern/forge/sdk/schemas/totp_codes.py b/skyvern/forge/sdk/schemas/totp_codes.py
index 60e1a1ad4..4df894f50 100644
--- a/skyvern/forge/sdk/schemas/totp_codes.py
+++ b/skyvern/forge/sdk/schemas/totp_codes.py
@@ -101,6 +101,22 @@ class RawTOTPCode(BaseModel):
expired_at: datetime | None
-class RawTOTPCodeAccepted(BaseModel):
- totp_code_id: str
+class RawTOTPCodeAccepted(TOTPCode):
status: Literal["raw_pending"] = "raw_pending"
+
+ @classmethod
+ def from_raw_row(cls, raw_row: RawTOTPCode) -> "RawTOTPCodeAccepted":
+ return cls(
+ totp_code_id=raw_row.totp_code_id,
+ totp_identifier=raw_row.totp_identifier,
+ organization_id=raw_row.organization_id,
+ content=raw_row.content,
+ code="",
+ task_id=raw_row.task_id,
+ workflow_id=raw_row.workflow_id,
+ workflow_run_id=raw_row.workflow_run_id,
+ source=raw_row.source,
+ created_at=raw_row.created_at,
+ modified_at=raw_row.created_at,
+ expired_at=raw_row.expired_at,
+ )
diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.py
index 1d935e661..ca522e72a 100644
--- a/tests/unit/test_credentials_totp_route_logging.py
+++ b/tests/unit/test_credentials_totp_route_logging.py
@@ -1,4 +1,5 @@
import json
+from datetime import datetime, timezone
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -8,7 +9,22 @@ from fastapi import FastAPI, HTTPException
from skyvern.forge.sdk.routes import credentials
from skyvern.forge.sdk.routes.routers import base_router, legacy_base_router
-from skyvern.forge.sdk.schemas.totp_codes import TOTPCodeCreate
+from skyvern.forge.sdk.schemas.totp_codes import RawTOTPCode, TOTPCodeCreate
+
+
+def _raw_row(totp_code_id: str = "otp_raw") -> RawTOTPCode:
+ return RawTOTPCode(
+ totp_code_id=totp_code_id,
+ totp_identifier="[email protected]",
+ organization_id="o_test",
+ content="Long email body requesting OTP for the account.",
+ task_id=None,
+ workflow_id=None,
+ workflow_run_id=None,
+ source=None,
+ created_at=datetime(2025, 1, 1, tzinfo=timezone.utc),
+ expired_at=None,
+ )
def _database_with_otp_create(
@@ -62,48 +78,25 @@ async def test_send_totp_code_parse_failure_log_redacts_identifier_and_content(
raw_identifier = "[email protected]"
raw_content = "Use 135790 to finish sign in for [email protected]."
create_otp_code = AsyncMock()
- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))
+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))
monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))
monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))
- assert type(credentials.settings).model_fields["TOTP_RAW_FALLBACK_ENABLED"].default is False
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", False)
with structlog.testing.capture_logs() as logs:
- with pytest.raises(HTTPException) as exc_info:
- await credentials.send_totp_code(
- TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),
- curr_org=SimpleNamespace(organization_id="o_test"),
- )
+ response = await credentials.send_totp_code(
+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),
+ curr_org=SimpleNamespace(organization_id="o_test"),
+ )
- assert exc_info.value.status_code == 400
- assert exc_info.value.detail == "Failed to parse otp login"
- assert exc_info.value.__cause__ is None
- assert exc_info.value.__context__ is None
+ assert response.status_code == 200
create_otp_code.assert_not_awaited()
- create_raw_otp_code.assert_not_awaited()
-
- error_log = next((r for r in logs if r.get("event") == "Failed to parse otp login"), None)
- assert error_log is not None
- assert error_log["organization_id"] == "o_test"
- assert error_log["totp_identifier"] == "[REDACTED_OTP_IDENTIFIER]"
- assert error_log["content_length"] == len(raw_content)
- assert "content" not in error_log
- assert all(record.get("event") != "Skipping OTP parse; organization has insufficient credits" for record in logs)
+ create_raw_otp_code.assert_awaited_once()
_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "135790")
@pytest.mark.asyncio
- ("raw_fallback_enabled", "raw_content_max_length"),
- [
- (False, 1_000),
- (True, 10),
- ],
-)
-async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback_is_unavailable(
+async def test_send_totp_code_insufficient_credits_returns_402_when_raw_content_over_cap(
monkeypatch: pytest.MonkeyPatch,
- raw_fallback_enabled: bool,
- raw_content_max_length: int,
) -> None:
raw_identifier = "[email protected]"
raw_content = "Long relayed message without a verification code."
@@ -125,8 +118,7 @@ async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback
),
)
monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", raw_fallback_enabled)
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", raw_content_max_length)
+ monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", 10)
with structlog.testing.capture_logs() as logs:
with pytest.raises(HTTPException) as exc_info:
@@ -149,13 +141,13 @@ async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback
@pytest.mark.asyncio
-async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallback(
+async def test_send_totp_code_insufficient_credits_persists_raw_within_cap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
raw_identifier = "[email protected]"
raw_content = "Long relayed message without a verification code."
create_otp_code = AsyncMock()
- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))
+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))
llm_handler = AsyncMock()
charge = AsyncMock()
monkeypatch.setattr(
@@ -172,7 +164,6 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbac
),
)
monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)
with structlog.testing.capture_logs() as logs:
response = await credentials.send_totp_code(
@@ -180,8 +171,11 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbac
curr_org=SimpleNamespace(organization_id="o_test"),
)
- assert response.status_code == 202
- assert json.loads(response.body) == {"totp_code_id": "otp_raw", "status": "raw_pending"}
+ assert response.status_code == 200
+ body = json.loads(response.body)
+ assert body["totp_code_id"] == "otp_raw"
+ assert body["status"] == "raw_pending"
+ assert body["code"] == ""
assert all(record.get("event") != "Failed to parse otp login" for record in logs)
assert any(record.get("event") == "Skipping OTP parse; organization has insufficient credits" for record in logs)
_assert_raw_values_not_logged(logs, raw_identifier, raw_content)
@@ -191,18 +185,18 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbac
create_raw_otp_code.assert_awaited_once()
-def test_send_totp_code_openapi_declares_distinct_success_and_credit_responses() -> None:
+def test_send_totp_code_openapi_declares_200_totp_code_and_402_no_202() -> None:
fastapi_app = FastAPI()
fastapi_app.include_router(base_router, prefix="/v1")
fastapi_app.include_router(legacy_base_router, prefix="/api/v1")
responses = fastapi_app.openapi()["paths"]["/v1/credentials/totp"]["post"]["responses"]
assert responses["200"]["content"]["application/json"]["schema"]["$ref"].endswith("/TOTPCode")
- assert responses["202"]["content"]["application/json"]["schema"]["$ref"].endswith("/RawTOTPCodeAccepted")
+ assert "202" not in responses
assert responses["402"]["description"] == "Insufficient credits to parse OTP content"
legacy_route = next(route for route in legacy_base_router.routes if route.path == "/totp")
- assert legacy_route.responses[202]["model"].__name__ == "RawTOTPCodeAccepted"
+ assert 202 not in legacy_route.responses
assert legacy_route.responses[402]["description"] == "Insufficient credits to parse OTP content"
@@ -249,7 +243,7 @@ async def test_send_totp_code_parser_exception_log_redacts_raw_exception_context
@pytest.mark.asyncio
-async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(
+async def test_send_totp_code_off_schema_llm_response_yields_200_not_502(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from skyvern.services import otp_service
@@ -257,9 +251,8 @@ async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(
raw_identifier = "[email protected]"
raw_content = "Long email body requesting OTP 424242 for [email protected]."
create_otp_code = AsyncMock()
- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))
+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))
monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)
async def off_schema_handler(*_args: object, **_kwargs: object) -> dict[str, object]:
return {"otp_type": "totp", "otp_value": "424242"}
@@ -273,20 +266,50 @@ async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(
curr_org=SimpleNamespace(organization_id="o_test"),
)
- assert response.status_code == 202
- assert json.loads(response.body) == {"totp_code_id": "otp_raw", "status": "raw_pending"}
+ assert response.status_code == 200
+ body = json.loads(response.body)
+ assert body["totp_code_id"] == "otp_raw"
+ assert body["status"] == "raw_pending"
+ assert body["code"] == ""
create_otp_code.assert_not_awaited()
create_raw_otp_code.assert_awaited_once()
_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "424242")
+async def test_send_totp_code_raw_fallback_body_validates_vendored_totp_code(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from skyvern.client.types.totp_code import TotpCode as VendoredTotpCode
+
+ raw_identifier = "[email protected]"
+ raw_content = "Long email body requesting OTP for the account."
+ create_otp_code = AsyncMock()
+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))
+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))
+ monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))
+
+ response = await credentials.send_totp_code(
+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),
+ curr_org=SimpleNamespace(organization_id="o_test"),
+ )
+
+ assert response.status_code == 200
+ body = json.loads(response.body)
+ parsed = VendoredTotpCode.model_validate(body)
+ assert parsed.totp_code_id == "otp_raw"
+ assert parsed.code == ""
+ assert (parsed.model_extra or {})["status"] == "raw_pending"
+ create_otp_code.assert_not_awaited()
+ create_raw_otp_code.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_send_totp_code_parse_miss_over_cap_stays_400(monkeypatch: pytest.MonkeyPatch) -> None:
create_otp_code = AsyncMock()
create_raw_otp_code = AsyncMock()
monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))
monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))
- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)
monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", 12)
with pytest.raises(HTTPException) as exc_info: