Skip to content

Automate browser based workflows with AI

SKY-10560: route TOTP verification requests through NAT egress proxy (#6274)

Shuchang Zhengd0939e007223
diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.py
index 33ae079bd..c436c0cee 100644
--- a/skyvern/forge/agent_functions.py
+++ b/skyvern/forge/agent_functions.py
@@ -2,9 +2,11 @@ import asyncio
import copy
import hashlib
import os
+from dataclasses import dataclass, field
from datetime import timedelta
from typing import TYPE_CHECKING, Any, Dict, List
+import aiohttp
import httpx
import structlog
from playwright.async_api import Frame, Page
@@ -53,6 +55,20 @@ CSS_SHAPE_CONVERTION_ATTEMPTS = 1
INVALID_SHAPE = "N/A"
+@dataclass
+class TOTPVerificationResponse:
+ """Normalized response shape for the TOTP verification seam.
+
+ Decouples the seam contract from any specific HTTP client so the OSS
+ direct path (aiohttp) and the cloud proxy path (NATEgressProxyClient)
+ can both produce a response the helper consumes the same way.
+ """
+
+ status_code: int
+ body: str
+ headers: dict[str, str] = field(default_factory=dict)
+
+
def _remove_rect(element: dict) -> None:
if "rect" in element:
del element["rect"]
@@ -1108,6 +1124,23 @@ class AgentFunction:
timeout=httpx.Timeout(timeout_seconds),
)
+ async def post_totp_verification_request(
+ self,
+ url: str,
+ payload: str,
+ headers: dict[str, str],
+ timeout_seconds: float = 30.0,
+ organization_id: str | None = None,
+ ) -> TOTPVerificationResponse:
+ async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout_seconds)) as session:
+ async with session.post(url, data=payload, headers=headers) as response:
+ body = await response.text()
+ return TOTPVerificationResponse(
+ status_code=response.status,
+ body=body,
+ headers=dict(response.headers),
+ )
+
async def upload_file_to_customer_storage(
self,
file_path: str,
diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.py
index 56d2f3f05..5044d159a 100644
--- a/skyvern/services/otp_service.py
+++ b/skyvern/services/otp_service.py
@@ -1,9 +1,9 @@
import asyncio
+import json
import re
from datetime import datetime, timedelta
from typing import TYPE_CHECKING, Any
-import aiohttp
import pyotp
import structlog
from pydantic import BaseModel, Field
@@ -175,32 +175,44 @@ async def _post_totp_verification_url(
url: str,
signed_payload: str,
headers: dict[str, str],
+ organization_id: str,
max_attempts: int = _TOTP_WEBHOOK_REQUEST_MAX_ATTEMPTS,
retry_timeout: float = _TOTP_WEBHOOK_REQUEST_RETRY_TIMEOUT_SECONDS,
timeout: int = DEFAULT_REQUEST_TIMEOUT,
) -> _TOTPWebhookPostResponse:
- async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=timeout)) as session:
- for attempt in range(max_attempts):
+ # Routed through app.AGENT_FUNCTION so cloud egresses via the NAT proxy
+ # (static IP), matching webhook and file-upload delivery.
+ for attempt in range(max_attempts):
+ try:
+ response = await app.AGENT_FUNCTION.post_totp_verification_request(
+ url=url,
+ payload=signed_payload,
+ headers=headers,
+ timeout_seconds=timeout,
+ organization_id=organization_id,
+ )
+ # Content-Type gate: only trust an explicit non-JSON header to mean
+ # "this is not JSON". Missing header (e.g. proxy responses, which
+ # don't preserve upstream headers) falls through to optimistic JSON
+ # parsing — customer TOTP endpoints contractually return JSON.
+ content_type = response.headers.get("content-type", "").lower()
+ if content_type and "json" not in content_type:
+ return response.status_code, response.headers, response.body, False
try:
- async with session.post(url, data=signed_payload, headers=headers) as response:
- response_headers = dict(response.headers)
- try:
- response_body = await response.json()
- return response.status, response_headers, response_body, True
- except (aiohttp.ContentTypeError, ValueError):
- response_body = await response.text()
- return response.status, response_headers, response_body, False
- except Exception:
- LOG.debug(
- "TOTP webhook request attempt failed",
- endpoint_url=url,
- attempt=attempt + 1,
- max_attempts=max_attempts,
- exc_info=True,
- )
- if attempt < max_attempts - 1 and retry_timeout > 0:
- await asyncio.sleep(retry_timeout)
- raise _TOTPWebhookRequestError(f"Failed post request url={url}")
+ return response.status_code, response.headers, json.loads(response.body), True
+ except (json.JSONDecodeError, ValueError):
+ return response.status_code, response.headers, response.body, False
+ except Exception:
+ LOG.debug(
+ "TOTP webhook request attempt failed",
+ endpoint_url=url,
+ attempt=attempt + 1,
+ max_attempts=max_attempts,
+ exc_info=True,
+ )
+ if attempt < max_attempts - 1 and retry_timeout > 0:
+ await asyncio.sleep(retry_timeout)
+ raise _TOTPWebhookRequestError(f"Failed post request url={url}")
def _try_generate_totp_for_credential(
@@ -386,6 +398,7 @@ async def _get_otp_value_from_url(
url=url,
signed_payload=signed_data.signed_payload,
headers=signed_data.headers,
+ organization_id=organization_id,
)
except Exception as e:
LOG.error("Failed to get otp value from url", totp_verification_url=url, exc_info=True)
diff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.py
index 9af17ea36..eb0b86bdf 100644
--- a/tests/unit/test_otp_service.py
+++ b/tests/unit/test_otp_service.py
@@ -583,3 +583,159 @@ class TestTryGenerateTotpFromCredential:
with skyvern_context.scoped(_scoped_context(active="credentials")):
assert try_generate_totp_from_credential("wr_test") is None
+
+
+class TestPostTotpVerificationUrlSeam:
+ """`_post_totp_verification_url` must route through app.AGENT_FUNCTION so the
+ cloud override can egress via the NAT proxy (static IP), like webhook/file-upload."""
+
+ @pytest.mark.asyncio
+ async def test_routes_through_agent_function_and_adapts_json(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.agent_functions import TOTPVerificationResponse
+ from skyvern.services import otp_service
+
+ seam = AsyncMock(
+ return_value=TOTPVerificationResponse(
+ status_code=200,
+ body='{"verification_code": "123456"}',
+ headers={"content-type": "application/json"},
+ )
+ )
+ monkeypatch.setattr(
+ otp_service.app, "AGENT_FUNCTION", SimpleNamespace(post_totp_verification_request=seam), raising=False
+ )
+
+ status, _headers, body, is_json = await otp_service._post_totp_verification_url(
+ url="https://example.com/totp",
+ signed_payload="{}",
+ headers={"X-Sig": "abc"},
+ organization_id="o_test",
+ retry_timeout=0,
+ )
+
+ assert status == 200
+ assert is_json is True
+ assert body == {"verification_code": "123456"}
+ seam.assert_awaited_once()
+ kwargs = seam.await_args.kwargs
+ assert kwargs["url"] == "https://example.com/totp"
+ assert kwargs["payload"] == "{}"
+ assert kwargs["headers"] == {"X-Sig": "abc"}
+ assert kwargs["organization_id"] == "o_test"
+
+ @pytest.mark.asyncio
+ async def test_non_json_response_marks_is_json_false(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ from skyvern.forge.agent_functions import TOTPVerificationResponse
+ from skyvern.services import otp_service
+
+ seam = AsyncMock(
+ return_value=TOTPVerificationResponse(
+ status_code=200,
+ body="not a json body",
+ headers={"content-type": "text/plain"},
+ )
+ )
+ monkeypatch.setattr(
+ otp_service.app, "AGENT_FUNCTION", SimpleNamespace(post_totp_verification_request=seam), raising=False
+ )
+
+ status, _headers, body, is_json = await otp_service._post_totp_verification_url(
+ url="https://example.com/totp",
+ signed_payload="{}",
+ headers={},
+ organization_id="o_test",
+ retry_timeout=0,
+ )
+
+ assert status == 200
+ assert is_json is False
+ assert body == "not a json body"
+
+ @pytest.mark.asyncio
+ async def test_json_body_with_non_json_content_type_marks_is_json_false(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """Explicit non-JSON Content-Type must trump the JSON-shaped body so a
+ misconfigured customer endpoint still trips the non-JSON contract error
+ downstream instead of being silently accepted."""
+ from skyvern.forge.agent_functions import TOTPVerificationResponse
+ from skyvern.services import otp_service
+
+ seam = AsyncMock(
+ return_value=TOTPVerificationResponse(
+ status_code=200,
+ body='{"verification_code": "123456"}',
+ headers={"content-type": "text/plain"},
+ )
+ )
+ monkeypatch.setattr(
+ otp_service.app, "AGENT_FUNCTION", SimpleNamespace(post_totp_verification_request=seam), raising=False
+ )
+
+ status, _headers, body, is_json = await otp_service._post_totp_verification_url(
+ url="https://example.com/totp",
+ signed_payload="{}",
+ headers={},
+ organization_id="o_test",
+ retry_timeout=0,
+ )
+
+ assert status == 200
+ assert is_json is False
+ assert body == '{"verification_code": "123456"}'
+
+ @pytest.mark.asyncio
+ async def test_missing_content_type_falls_through_to_optimistic_json_parse(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ """NATEgressProxyClient.deliver_webhook doesn't preserve upstream response
+ headers, so the proxy path produces TOTPVerificationResponse(headers={}).
+ The helper must still parse a JSON-shaped body as JSON in that case."""
+ from skyvern.forge.agent_functions import TOTPVerificationResponse
+ from skyvern.services import otp_service
+
+ seam = AsyncMock(
+ return_value=TOTPVerificationResponse(
+ status_code=200,
+ body='{"verification_code": "123456"}',
+ headers={},
+ )
+ )
+ monkeypatch.setattr(
+ otp_service.app, "AGENT_FUNCTION", SimpleNamespace(post_totp_verification_request=seam), raising=False
+ )
+
+ status, _headers, body, is_json = await otp_service._post_totp_verification_url(
+ url="https://example.com/totp",
+ signed_payload="{}",
+ headers={},
+ organization_id="o_test",
+ retry_timeout=0,
+ )
+
+ assert status == 200
+ assert is_json is True
+ assert body == {"verification_code": "123456"}
+
+ @pytest.mark.asyncio
+ async def test_retries_then_raises_on_persistent_exception(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import httpx
+
+ from skyvern.services import otp_service
+
+ seam = AsyncMock(side_effect=httpx.ConnectError("boom"))
+ monkeypatch.setattr(
+ otp_service.app, "AGENT_FUNCTION", SimpleNamespace(post_totp_verification_request=seam), raising=False
+ )
+
+ with pytest.raises(otp_service._TOTPWebhookRequestError):
+ await otp_service._post_totp_verification_url(
+ url="https://example.com/totp",
+ signed_payload="{}",
+ headers={},
+ organization_id="o_test",
+ max_attempts=2,
+ retry_timeout=0,
+ )
+
+ assert seam.await_count == 2