Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: enable raw OTP fallback by default, remove TOTP_RAW_FALLBACK_ENABLED (#7866)
LawyZhengb670a1ec54d8
diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex 059ffc007..59cc70096 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -4247,16 +4247,6 @@}}},- "202": {- "description": "Raw OTP content accepted for later parsing",- "content": {- "application/json": {- "schema": {- "$ref": "#/components/schemas/RawTOTPCodeAccepted"- }- }- }- },"402": {"description": "Insufficient credits to parse OTP content"},diff --git a/skyvern/config.py b/skyvern/config.pyindex 50e9a682a..f4591619d 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -601,7 +601,6 @@ class Settings(BaseSettings):# TOTP SettingsTOTP_LIFESPAN_MINUTES: int = 10- TOTP_RAW_FALLBACK_ENABLED: bool = FalseTOTP_RAW_CONTENT_MAX_LENGTH: int = 65536VERIFICATION_CODE_INITIAL_WAIT_TIME_SECS: int = 40VERIFICATION_CODE_POLLING_TIMEOUT_MINS: int = 15diff --git a/skyvern/forge/sdk/routes/credentials.py b/skyvern/forge/sdk/routes/credentials.pyindex a85e14b3c..36f12761b 100644--- a/skyvern/forge/sdk/routes/credentials.py+++ b/skyvern/forge/sdk/routes/credentials.py@@ -412,19 +412,13 @@ async def fetch_credential_item_background(item_id: str) -> None:@legacy_base_router.post("/totp",- responses={- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},- 402: {"description": "Insufficient credits to parse OTP content"},- },+ responses={402: {"description": "Insufficient credits to parse OTP content"}},)@legacy_base_router.post("/totp/", include_in_schema=False)@base_router.post("/credentials/totp",response_model=TOTPCode,- responses={- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},- 402: {"description": "Insufficient credits to parse OTP content"},- },+ responses={402: {"description": "Insufficient credits to parse OTP content"}},summary="Send TOTP code",description="Forward a TOTP (2FA, MFA) email or sms message containing the code to Skyvern. This endpoint stores the code in database so that Skyvern can use it while running tasks/workflows.",tags=["Credentials"],@@ -443,10 +437,7 @@ async def fetch_credential_item_background(item_id: str) -> None:@base_router.post("/credentials/totp/",response_model=TOTPCode,- responses={- 202: {"model": RawTOTPCodeAccepted, "description": "Raw OTP content accepted for later parsing"},- 402: {"description": "Insufficient credits to parse OTP content"},- },+ responses={402: {"description": "Insufficient credits to parse OTP content"}},include_in_schema=False,)async def send_totp_code(@@ -490,9 +481,7 @@ async def send_totp_code(otp_value = Noneparse_exception_type_name = type(e).__name__- if otp_parse_skipped_for_insufficient_credits and (- not settings.TOTP_RAW_FALLBACK_ENABLED or len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH- ):+ if otp_parse_skipped_for_insufficient_credits and len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH:raise HTTPException(status_code=status.HTTP_402_PAYMENT_REQUIRED,detail="Insufficient credits to parse OTP content",@@ -519,17 +508,6 @@ async def send_totp_code()if not otp_value:- if not settings.TOTP_RAW_FALLBACK_ENABLED:- LOG.error(- "Failed to parse otp login",- organization_id=curr_org.organization_id,- totp_identifier=redacted_totp_identifier,- task_id=data.task_id,- workflow_id=data.workflow_id,- workflow_run_id=data.workflow_run_id,- content_length=len(data.content),- )- raise HTTPException(status_code=400, detail="Failed to parse otp login")if len(data.content) > settings.TOTP_RAW_CONTENT_MAX_LENGTH:raise HTTPException(status_code=400, detail="Failed to parse otp login")raw_row = await app.DATABASE.otp.create_raw_otp_code(@@ -549,9 +527,9 @@ async def send_totp_code(totp_code_id=raw_row.totp_code_id,content_length=len(data.content),)- pending = RawTOTPCodeAccepted(totp_code_id=raw_row.totp_code_id)+ pending = RawTOTPCodeAccepted.from_raw_row(raw_row)return Response( # type: ignore[return-value]- content=pending.model_dump_json(), status_code=202, media_type="application/json"+ content=pending.model_dump_json(), status_code=200, media_type="application/json")return await app.DATABASE.otp.create_otp_code(diff --git a/skyvern/forge/sdk/schemas/totp_codes.py b/skyvern/forge/sdk/schemas/totp_codes.pyindex 60e1a1ad4..4df894f50 100644--- a/skyvern/forge/sdk/schemas/totp_codes.py+++ b/skyvern/forge/sdk/schemas/totp_codes.py@@ -101,6 +101,22 @@ class RawTOTPCode(BaseModel):expired_at: datetime | None-class RawTOTPCodeAccepted(BaseModel):- totp_code_id: str+class RawTOTPCodeAccepted(TOTPCode):status: Literal["raw_pending"] = "raw_pending"++ @classmethod+ def from_raw_row(cls, raw_row: RawTOTPCode) -> "RawTOTPCodeAccepted":+ return cls(+ totp_code_id=raw_row.totp_code_id,+ totp_identifier=raw_row.totp_identifier,+ organization_id=raw_row.organization_id,+ content=raw_row.content,+ code="",+ task_id=raw_row.task_id,+ workflow_id=raw_row.workflow_id,+ workflow_run_id=raw_row.workflow_run_id,+ source=raw_row.source,+ created_at=raw_row.created_at,+ modified_at=raw_row.created_at,+ expired_at=raw_row.expired_at,+ )diff --git a/tests/unit/test_credentials_totp_route_logging.py b/tests/unit/test_credentials_totp_route_logging.pyindex 1d935e661..ca522e72a 100644--- a/tests/unit/test_credentials_totp_route_logging.py+++ b/tests/unit/test_credentials_totp_route_logging.py@@ -1,4 +1,5 @@import json+from datetime import datetime, timezonefrom types import SimpleNamespacefrom unittest.mock import AsyncMock@@ -8,7 +9,22 @@ from fastapi import FastAPI, HTTPExceptionfrom skyvern.forge.sdk.routes import credentialsfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_router-from skyvern.forge.sdk.schemas.totp_codes import TOTPCodeCreate+from skyvern.forge.sdk.schemas.totp_codes import RawTOTPCode, TOTPCodeCreate+++def _raw_row(totp_code_id: str = "otp_raw") -> RawTOTPCode:+ return RawTOTPCode(+ totp_code_id=totp_code_id,+ totp_identifier="[email protected]",+ organization_id="o_test",+ content="Long email body requesting OTP for the account.",+ task_id=None,+ workflow_id=None,+ workflow_run_id=None,+ source=None,+ created_at=datetime(2025, 1, 1, tzinfo=timezone.utc),+ expired_at=None,+ )def _database_with_otp_create(@@ -62,48 +78,25 @@ async def test_send_totp_code_parse_failure_log_redacts_identifier_and_content(raw_identifier = "[email protected]"raw_content = "Use 135790 to finish sign in for [email protected]."create_otp_code = AsyncMock()- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))- assert type(credentials.settings).model_fields["TOTP_RAW_FALLBACK_ENABLED"].default is False- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", False)with structlog.testing.capture_logs() as logs:- with pytest.raises(HTTPException) as exc_info:- await credentials.send_totp_code(- TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),- curr_org=SimpleNamespace(organization_id="o_test"),- )+ response = await credentials.send_totp_code(+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )- assert exc_info.value.status_code == 400- assert exc_info.value.detail == "Failed to parse otp login"- assert exc_info.value.__cause__ is None- assert exc_info.value.__context__ is None+ assert response.status_code == 200create_otp_code.assert_not_awaited()- create_raw_otp_code.assert_not_awaited()-- error_log = next((r for r in logs if r.get("event") == "Failed to parse otp login"), None)- assert error_log is not None- assert error_log["organization_id"] == "o_test"- assert error_log["totp_identifier"] == "[REDACTED_OTP_IDENTIFIER]"- assert error_log["content_length"] == len(raw_content)- assert "content" not in error_log- assert all(record.get("event") != "Skipping OTP parse; organization has insufficient credits" for record in logs)+ create_raw_otp_code.assert_awaited_once()_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "135790")@pytest.mark.asyncio- ("raw_fallback_enabled", "raw_content_max_length"),- [- (False, 1_000),- (True, 10),- ],-)-async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback_is_unavailable(+async def test_send_totp_code_insufficient_credits_returns_402_when_raw_content_over_cap(monkeypatch: pytest.MonkeyPatch,- raw_fallback_enabled: bool,- raw_content_max_length: int,) -> None:raw_identifier = "[email protected]"raw_content = "Long relayed message without a verification code."@@ -125,8 +118,7 @@ async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback),)monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", raw_fallback_enabled)- monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", raw_content_max_length)+ monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", 10)with structlog.testing.capture_logs() as logs:with pytest.raises(HTTPException) as exc_info:@@ -149,13 +141,13 @@ async def test_send_totp_code_insufficient_credits_returns_402_when_raw_fallback@pytest.mark.asyncio-async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallback(+async def test_send_totp_code_insufficient_credits_persists_raw_within_cap(monkeypatch: pytest.MonkeyPatch,) -> None:raw_identifier = "[email protected]"raw_content = "Long relayed message without a verification code."create_otp_code = AsyncMock()- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))llm_handler = AsyncMock()charge = AsyncMock()monkeypatch.setattr(@@ -172,7 +164,6 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbac),)monkeypatch.setattr(credentials.app, "SECONDARY_LLM_API_HANDLER", llm_handler)- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)with structlog.testing.capture_logs() as logs:response = await credentials.send_totp_code(@@ -180,8 +171,11 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbaccurr_org=SimpleNamespace(organization_id="o_test"),)- assert response.status_code == 202- assert json.loads(response.body) == {"totp_code_id": "otp_raw", "status": "raw_pending"}+ assert response.status_code == 200+ body = json.loads(response.body)+ assert body["totp_code_id"] == "otp_raw"+ assert body["status"] == "raw_pending"+ assert body["code"] == ""assert all(record.get("event") != "Failed to parse otp login" for record in logs)assert any(record.get("event") == "Skipping OTP parse; organization has insufficient credits" for record in logs)_assert_raw_values_not_logged(logs, raw_identifier, raw_content)@@ -191,18 +185,18 @@ async def test_send_totp_code_insufficient_credits_preserves_enabled_raw_fallbaccreate_raw_otp_code.assert_awaited_once()-def test_send_totp_code_openapi_declares_distinct_success_and_credit_responses() -> None:+def test_send_totp_code_openapi_declares_200_totp_code_and_402_no_202() -> None:fastapi_app = FastAPI()fastapi_app.include_router(base_router, prefix="/v1")fastapi_app.include_router(legacy_base_router, prefix="/api/v1")responses = fastapi_app.openapi()["paths"]["/v1/credentials/totp"]["post"]["responses"]assert responses["200"]["content"]["application/json"]["schema"]["$ref"].endswith("/TOTPCode")- assert responses["202"]["content"]["application/json"]["schema"]["$ref"].endswith("/RawTOTPCodeAccepted")+ assert "202" not in responsesassert responses["402"]["description"] == "Insufficient credits to parse OTP content"legacy_route = next(route for route in legacy_base_router.routes if route.path == "/totp")- assert legacy_route.responses[202]["model"].__name__ == "RawTOTPCodeAccepted"+ assert 202 not in legacy_route.responsesassert legacy_route.responses[402]["description"] == "Insufficient credits to parse OTP content"@@ -249,7 +243,7 @@ async def test_send_totp_code_parser_exception_log_redacts_raw_exception_context@pytest.mark.asyncio-async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(+async def test_send_totp_code_off_schema_llm_response_yields_200_not_502(monkeypatch: pytest.MonkeyPatch,) -> None:from skyvern.services import otp_service@@ -257,9 +251,8 @@ async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(raw_identifier = "[email protected]"raw_content = "Long email body requesting OTP 424242 for [email protected]."create_otp_code = AsyncMock()- create_raw_otp_code = AsyncMock(return_value=SimpleNamespace(totp_code_id="otp_raw"))+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)async def off_schema_handler(*_args: object, **_kwargs: object) -> dict[str, object]:return {"otp_type": "totp", "otp_value": "424242"}@@ -273,20 +266,50 @@ async def test_send_totp_code_off_schema_llm_response_yields_202_not_502(curr_org=SimpleNamespace(organization_id="o_test"),)- assert response.status_code == 202- assert json.loads(response.body) == {"totp_code_id": "otp_raw", "status": "raw_pending"}+ assert response.status_code == 200+ body = json.loads(response.body)+ assert body["totp_code_id"] == "otp_raw"+ assert body["status"] == "raw_pending"+ assert body["code"] == ""create_otp_code.assert_not_awaited()create_raw_otp_code.assert_awaited_once()_assert_raw_values_not_logged(logs, raw_identifier, raw_content, "424242")+async def test_send_totp_code_raw_fallback_body_validates_vendored_totp_code(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from skyvern.client.types.totp_code import TotpCode as VendoredTotpCode++ raw_identifier = "[email protected]"+ raw_content = "Long email body requesting OTP for the account."+ create_otp_code = AsyncMock()+ create_raw_otp_code = AsyncMock(return_value=_raw_row("otp_raw"))+ monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))+ monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))++ response = await credentials.send_totp_code(+ TOTPCodeCreate(totp_identifier=raw_identifier, content=raw_content),+ curr_org=SimpleNamespace(organization_id="o_test"),+ )++ assert response.status_code == 200+ body = json.loads(response.body)+ parsed = VendoredTotpCode.model_validate(body)+ assert parsed.totp_code_id == "otp_raw"+ assert parsed.code == ""+ assert (parsed.model_extra or {})["status"] == "raw_pending"+ create_otp_code.assert_not_awaited()+ create_raw_otp_code.assert_awaited_once()++@pytest.mark.asyncioasync def test_send_totp_code_parse_miss_over_cap_stays_400(monkeypatch: pytest.MonkeyPatch) -> None:create_otp_code = AsyncMock()create_raw_otp_code = AsyncMock()monkeypatch.setattr(credentials.app, "DATABASE", _database_with_otp_create(create_otp_code, create_raw_otp_code))monkeypatch.setattr(credentials, "parse_otp_login", AsyncMock(return_value=None))- monkeypatch.setattr(credentials.settings, "TOTP_RAW_FALLBACK_ENABLED", True)monkeypatch.setattr(credentials.settings, "TOTP_RAW_CONTENT_MAX_LENGTH", 12)with pytest.raises(HTTPException) as exc_info: