Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Store OTP/magic-link values verbatim instead of LLM-retyped (fixes magic-link 400s) (#8062)
Celal Zamanoğlub31f2dd64347
diff --git a/skyvern/forge/prompts/skyvern/parse-otp-login.j2 b/skyvern/forge/prompts/skyvern/parse-otp-login.j2index aafe81ffc..c2e117fcf 100644--- a/skyvern/forge/prompts/skyvern/parse-otp-login.j2+++ b/skyvern/forge/prompts/skyvern/parse-otp-login.j2@@ -1,4 +1,4 @@-You receive either an email or a text message containing an OTP(like TOTP, Magic Link) to verify the login. Your job is to parse the content, identify the OTP type and value. The value must be from the content.+You receive either an email or a text message containing an OTP(like TOTP, Magic Link) to verify the login. Your job is to parse the content, identify the OTP type and value. The value must be from the content. Copy the value EXACTLY as it appears in the content, character for character — do NOT retype, reformat, correct, or shorten it. Magic-link URLs are long and a single changed character makes them invalid.{% if enforced_otp_type %}IMPORTANT: The user has specified they expect a "{{ enforced_otp_type }}" type OTP. You MUST extract the {{ enforced_otp_type }} value from the content, even if other OTP types are present. Ignore any other OTP types in the content. If no {{ enforced_otp_type }} is found, set otp_value_found to false.{% else %}diff --git a/skyvern/services/otp_service.py b/skyvern/services/otp_service.pyindex 01b406b4d..ba51bdff6 100644--- a/skyvern/services/otp_service.py+++ b/skyvern/services/otp_service.py@@ -1,4 +1,5 @@import asyncio+import difflibimport jsonimport refrom dataclasses import dataclass, field@@ -41,6 +42,23 @@ _MFA_PARAMETER_KEY_HINTS = ("mfa", "otp", "verification")# "totpidentifier" matches "otp" but carries a lookup key, not a 6-digit code._MFA_METADATA_KEY_HINTS = ("identifier", "url", "secret", "seed", "key")_NON_ALNUM_PATTERN = re.compile(r"[^a-z0-9]")+# OTP/magic-link values are copied verbatim from the source, never re-typed by the LLM: a high-entropy+# JWT loses one character to the classic rn->m confusion and its signature breaks. URL boundaries stop at+# whitespace and HTML/quote delimiters so an href="...">Sign body doesn't swallow the trailing markup.+_BARE_URL_PATTERN = re.compile(r"""^https?://[^\s"'<>)\]}]+$""", re.IGNORECASE)+_URL_IN_TEXT_PATTERN = re.compile(r"""https?://[^\s"'<>)\]}]+""", re.IGNORECASE)+# Every HTML spelling of a query-separating "&": named, decimal and hex. The semicolon is required so a+# plain-text parameter named after a legacy entity (?copy=1, ?amp=1) is left byte-exact.+_AMPERSAND_ENTITY_PATTERN = re.compile(r"&(?:amp|#0*38|#x0*26);", re.IGNORECASE)+_OTP_CODE_PATTERN = re.compile(r"\b\d{4,8}\b")+_CODE_SEPARATOR_PATTERN = re.compile(r"[\s\-]")+# A code as the source displays it, so a prefix of one is never mistaken for a whole one. A hyphen binds+# into the token ("AB-12" is one code, not "AB"). A space joins only equal-sized groups of 3-4 digits, the+# shape used to make a code readable ("123 456"); anything else keeps its own token, so a code followed by+# an unrelated number ("123456 10 minutes") is never welded into a value that appears nowhere in the source.+_CODE_CANDIDATE_PATTERN = re.compile(+ r"(?<![A-Za-z0-9])\d{3,4}(?:[ \t]\d{3,4})+(?![A-Za-z0-9])|[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*"+)_EXPECTED_TOTP_WEBHOOK_RESPONSE_SHAPE = '{"verification_code":"123456"}'# Recovers the verification_code value when the surrounding JSON is malformed# (e.g. unescaped quotes inside a relayed email). Assumes verification_code is@@ -87,6 +105,71 @@ class OTPResultParsedByLLM(BaseModel):otp_value: str | None = Field(None, description="The OTP value.")+def _clean_url(url: str) -> str:+ """Normalize a URL pulled from message text to what a browser would actually navigate to: strip+ trailing sentence punctuation a prose body attaches ("...verify.") and decode the ampersands an HTML+ href carries, so a query separator is the real & rather than the encoded source bytes. A full+ html.unescape would corrupt a plain-text link whose query parameter is a legacy entity (©=1)."""+ return _AMPERSAND_ENTITY_PATTERN.sub("&", url).rstrip(".,;:!?")+++def _verbatim_otp_value(content: str, otp_type: OTPType | None, llm_value: str | None) -> str | None:+ """Return an OTP/magic-link value copied byte-for-byte from ``content``. The LLM is trusted to LOCATE+ the value, never to transcribe it: the stored value is always a COMPLETE candidate extracted from the+ source (a whole URL for magic links, a whole digit run for codes), selected as the one closest to what+ the LLM returned. This defeats one-character corruption AND truncation of a long token, and — when a+ message carries several URLs (logo, unsubscribe) or number runs (order #, year) — recovers the one the+ LLM actually located rather than the first in the text."""+ if not llm_value:+ return None++ if otp_type == OTPType.MAGIC_LINK or (+ otp_type is None and llm_value.strip().lower().startswith(("http://", "https://"))+ ):+ urls = [_clean_url(url) for url in _URL_IN_TEXT_PATTERN.findall(content)]+ cleaned_value = _clean_url(llm_value)+ if cleaned_value in urls:+ return cleaned_value+ closest = difflib.get_close_matches(cleaned_value, urls, n=1, cutoff=0.5)+ return closest[0] if closest else None++ # Codes: accept only a value equal to a WHOLE candidate from the source, compared with display+ # separators removed ("123 456", "123-456") so a correctly-read code is kept as the digits the site+ # expects. A truncation equals no whole candidate, so it reaches digit-run recovery instead of storage.+ stripped_value = _CODE_SEPARATOR_PATTERN.sub("", llm_value)+ if not stripped_value:+ return None+ # The same code quoted twice (subject and body) is one candidate, not two competing ones.+ candidates = list(+ dict.fromkeys(+ _CODE_SEPARATOR_PATTERN.sub("", candidate) for candidate in _CODE_CANDIDATE_PATTERN.findall(content)+ )+ )+ if stripped_value in candidates:+ return stripped_value+ numeric_candidates = [candidate for candidate in candidates if _OTP_CODE_PATTERN.fullmatch(candidate)]+ # A truncation is a prefix of the code it came from, so resolve prefixes before similarity: scoring+ # alone ranks a shorter unrelated number ("1234" from an order line) above the code it truncated.+ # Several prefix matches means the source cannot say which code was located; return nothing.+ prefixed = [candidate for candidate in numeric_candidates if candidate.startswith(stripped_value)]+ if prefixed:+ return prefixed[0] if len(prefixed) == 1 else None+ # Approximate recovery never guesses: an equally-similar runner-up ("123456" and "123457" against a+ # misread "123458") means the source cannot say which code was located, so let polling retry instead.+ scored = sorted(+ (+ (difflib.SequenceMatcher(None, candidate, stripped_value).ratio(), candidate)+ for candidate in numeric_candidates+ ),+ reverse=True,+ )+ if not scored or scored[0][0] < 0.6:+ return None+ if len(scored) > 1 and scored[1][0] == scored[0][0]:+ return None+ return scored[0][1]++async def parse_otp_login(content: str,organization_id: str,@@ -97,6 +180,13 @@ async def parse_otp_login(if not await app.AGENT_FUNCTION.has_sufficient_credit_for_otp_parse(organization_id):LOG.info("Skipping OTP parse; organization has insufficient credits", organization_id=organization_id)raise InsufficientCreditsForOTPParse+ # A bare magic-link URL is the entire message: store it verbatim and skip the LLM entirely, so a+ # high-entropy token is never re-transcribed (the rn->m corruption that breaks its signature). Still+ # a billable parse — charge as usual so the customer's per-parse bill is unchanged by the fast path.+ stripped_content = content.strip()+ if enforced_otp_type in (None, OTPType.MAGIC_LINK) and _BARE_URL_PATTERN.match(stripped_content):+ await app.AGENT_FUNCTION.charge_for_otp_parse(organization_id)+ return OTPValue(value=_clean_url(stripped_content), type=OTPType.MAGIC_LINK)prompt = prompt_engine.load_prompt("parse-otp-login",content=content,@@ -128,7 +218,9 @@ async def parse_otp_login(otp_length=len(otp_result.otp_value) if otp_result.otp_value else 0,)if otp_result.otp_value_found and otp_result.otp_value:- return OTPValue(value=otp_result.otp_value, type=otp_result.otp_type)+ verbatim_value = _verbatim_otp_value(content, otp_result.otp_type, otp_result.otp_value)+ if verbatim_value:+ return OTPValue(value=verbatim_value, type=otp_result.otp_type)return Nonediff --git a/tests/unit/test_otp_service.py b/tests/unit/test_otp_service.pyindex 573935eb6..1c2e8444b 100644--- a/tests/unit/test_otp_service.py+++ b/tests/unit/test_otp_service.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.services import otp_servicefrom skyvern.services.otp_service import (OTPValue,+ _clean_url,_get_otp_value_from_db,_get_otp_value_from_url,_is_mfa_like_parameter_key,@@ -513,6 +514,58 @@ class TestGetOtpValueFromUrl:assert all(record.get("event") != "Failed to parse otp login from the totp url" for record in logs)+class TestCleanUrl:+ """Every HTML spelling of "&" must decode; nothing else may be touched."""++ @pytest.mark.parametrize(+ ("raw", "expected"),+ [+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ ("https://e.test/v?a=1&token=abc", "https://e.test/v?a=1&token=abc"),+ # Legacy entity names used as ordinary query parameters must survive byte-exact.+ ("https://e.test/v?token=abc©=1", "https://e.test/v?token=abc©=1"),+ ("https://e.test/v?token=abc&=1", "https://e.test/v?token=abc&=1"),+ ("https://e.test/v?token=abc×=2", "https://e.test/v?token=abc×=2"),+ ("https://e.test/v?token=abc.", "https://e.test/v?token=abc"),+ ],+ )+ def test_decodes_only_ampersands(self, raw: str, expected: str) -> None:+ assert _clean_url(raw) == expected+++def _patch_otp_llm(+ monkeypatch: pytest.MonkeyPatch,+ *,+ otp_type: str,+ otp_value: str | None,+) -> AsyncMock:+ """Patch the prompt/LLM/billing seams of parse_otp_login and return the LLM handler mock."""+ handler = AsyncMock(+ return_value={+ "reasoning": "r",+ "otp_type": otp_type,+ "otp_value_found": otp_value is not None,+ "otp_value": otp_value,+ }+ )+ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )+ return handler++class TestParseOtpLogin:@pytest.mark.asyncioasync def test_parser_log_omits_raw_code_and_reasoning_keeps_metadata(@@ -535,7 +588,7 @@ class TestParseOtpLogin:monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", fake_handler, raising=False)with structlog.testing.capture_logs() as logs:- result = await parse_otp_login(content="raw email body", organization_id="o_test")+ result = await parse_otp_login(content="raw email body 135790", organization_id="o_test")assert result == OTPValue(value=raw, type="totp")@@ -610,6 +663,387 @@ class TestParseOtpLogin:assert result is Nonecharge.assert_awaited_once_with("o_test")+ @pytest.mark.asyncio+ async def test_bare_magic_link_url_stored_verbatim_without_llm(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ # A ~1000-char Clerk magic link; the LLM must never see it (re-typing corrupts one char -> bad sig).+ magic_link = "https://clerk.skyvern.com/v1/verify?token=" + "a1B2c3D4" * 120+ llm_handler = AsyncMock()+ charge = AsyncMock()+ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", llm_handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=charge,+ ),+ raising=False,+ )++ result = await parse_otp_login(content=f" {magic_link} ", organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # byte-exact, trimmed+ assert result.type == OTPType.MAGIC_LINK+ llm_handler.assert_not_awaited() # LLM skipped entirely+ charge.assert_awaited_once_with("o_test") # still a billable parse — bill unchanged++ @pytest.mark.asyncio+ async def test_corrupted_llm_magic_link_recovered_verbatim_from_content(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from skyvern.services import otp_service++ magic_link = "https://clerk.skyvern.com/v1/verify?token=" + "Zm9vYmFy" * 100+ content = f"Hi there, click to sign in:\n{magic_link}\nThanks!" # blob -> LLM path, not the fast path+ corrupted = magic_link.replace("skyvern", "skyverm") # the classic rn->m corruption++ async def corrupting_handler(*_args: object, **_kwargs: object) -> dict[str, object]:+ return {+ "reasoning": "found link",+ "otp_type": "magic_link",+ "otp_value_found": True,+ "otp_value": corrupted,+ }++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", corrupting_handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # corrupted LLM value rejected; verbatim recovered from content+ assert corrupted not in result.value++ @pytest.mark.asyncio+ async def test_plain_code_email_still_parses(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ content = "Your verification code is 428913. It expires in 10 minutes."++ async def code_handler(*_args: object, **_kwargs: object) -> dict[str, object]:+ return {"reasoning": "found code", "otp_type": "totp", "otp_value_found": True, "otp_value": "428913"}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", code_handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == "428913"+ assert result.type == OTPType.TOTP++ @pytest.mark.asyncio+ async def test_recovers_the_located_url_among_decoys_and_html(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ magic_link = "https://clerk.skyvern.com/v1/verify?token=" + "Zm9vYmFy" * 100+ content = ( # tracking/logo + unsubscribe decoys, real link inside an href="...">Sign anchor+ "https://track.example.com/o.png?id=987654\n"+ f'<a href="{magic_link}">Sign in</a>\n'+ "https://example.com/unsubscribe?u=42"+ )+ corrupted = magic_link.replace("skyvern", "skyverm") # one-char corruption of the located link++ async def handler(*_a: object, **_k: object) -> dict[str, object]:+ return {"reasoning": "r", "otp_type": "magic_link", "otp_value_found": True, "otp_value": corrupted}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # the located link, not the logo/unsubscribe decoys+ assert '">Sign' not in result.value # href boundary respected — no trailing markup++ @pytest.mark.asyncio+ async def test_recovers_complete_url_when_llm_truncates(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ magic_link = "https://clerk.skyvern.com/v1/verify?token=" + "Zm9vYmFy" * 100+ content = f"Sign in: {magic_link}"+ truncated = magic_link[:600] # an exact prefix — a substring of content, but incomplete++ async def handler(*_a: object, **_k: object) -> dict[str, object]:+ return {"reasoning": "r", "otp_type": "magic_link", "otp_value_found": True, "otp_value": truncated}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # complete URL, not the truncated prefix the LLM returned++ @pytest.mark.asyncio+ async def test_spaced_numeric_code_returned_without_separators(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ content = "Your code is 123 456 - enter it to continue."++ async def handler(*_a: object, **_k: object) -> dict[str, object]:+ return {"reasoning": "r", "otp_type": "totp", "otp_value_found": True, "otp_value": "123456"}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == "123456" # separators stripped — the digits the site expects++ @pytest.mark.asyncio+ async def test_recovers_url_stripping_trailing_sentence_punctuation(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ magic_link = "https://clerk.skyvern.com/v1/verify?token=" + "Zm9vYmFy" * 100+ content = f"Please click {magic_link}." # trailing period is prose, not part of the URL+ corrupted = magic_link.replace("skyvern", "skyverm")++ async def handler(*_a: object, **_k: object) -> dict[str, object]:+ return {"reasoning": "r", "otp_type": "magic_link", "otp_value_found": True, "otp_value": corrupted}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # trailing period stripped+ assert not result.value.endswith(".")++ @pytest.mark.asyncio+ async def test_recovers_url_decoding_html_entities(self, monkeypatch: pytest.MonkeyPatch) -> None:+ from skyvern.services import otp_service++ decoded = "https://clerk.skyvern.com/v1/verify?token=" + "Zm9vYmFy" * 100 + "&redirect=/home"+ encoded_in_body = decoded.replace("&", "&")+ content = f'<a href="{encoded_in_body}">Sign in</a>' # raw HTML email body+ corrupted = decoded.replace("skyvern", "skyverm")++ async def handler(*_a: object, **_k: object) -> dict[str, object]:+ return {"reasoning": "r", "otp_type": "magic_link", "otp_value_found": True, "otp_value": corrupted}++ monkeypatch.setattr(otp_service.prompt_engine, "load_prompt", lambda *a, **k: "prompt")+ monkeypatch.setattr(otp_service.app, "SECONDARY_LLM_API_HANDLER", handler, raising=False)+ monkeypatch.setattr(+ otp_service.app,+ "AGENT_FUNCTION",+ SimpleNamespace(+ has_sufficient_credit_for_otp_parse=AsyncMock(return_value=True),+ charge_for_otp_parse=AsyncMock(),+ ),+ raising=False,+ )++ result = await parse_otp_login(content=content, organization_id="o_test")++ assert result is not None+ assert result.value == decoded # & decoded to &, href boundary respected+ assert "&" not in result.value++ @pytest.mark.asyncio+ async def test_bare_url_fast_path_strips_trailing_sentence_punctuation(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ magic_link = "https://example.test/verify?token=abc"+ handler = _patch_otp_llm(monkeypatch, otp_type="magic_link", otp_value=None)++ result = await parse_otp_login(content=f"{magic_link}.", organization_id="o_test")++ assert result == OTPValue(value=magic_link, type=OTPType.MAGIC_LINK) # period dropped, link still signs+ handler.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_bare_url_fast_path_keeps_legacy_entity_query_parameter(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ magic_link = "https://example.test/verify?token=abc©=1" # "©" is also a legacy HTML entity+ _patch_otp_llm(monkeypatch, otp_type="magic_link", otp_value=None)++ result = await parse_otp_login(content=magic_link, organization_id="o_test")++ assert result == OTPValue(value=magic_link, type=OTPType.MAGIC_LINK) # not "...abc©=1"++ @pytest.mark.asyncio+ async def test_bare_url_fast_path_accepts_uppercase_scheme(self, monkeypatch: pytest.MonkeyPatch) -> None:+ magic_link = "HTTPS://example.test/verify?token=abc"+ handler = _patch_otp_llm(monkeypatch, otp_type="magic_link", otp_value=None)++ result = await parse_otp_login(content=magic_link, organization_id="o_test")++ assert result == OTPValue(value=magic_link, type=OTPType.MAGIC_LINK)+ handler.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_uppercase_scheme_url_in_text_recovered(self, monkeypatch: pytest.MonkeyPatch) -> None:+ magic_link = "HTTPS://example.test/verify?token=abc123"+ _patch_otp_llm(monkeypatch, otp_type="magic_link", otp_value=magic_link)++ result = await parse_otp_login(content=f"Sign in here: {magic_link} and welcome.", organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # candidate extraction is scheme-case-insensitive++ @pytest.mark.asyncio+ async def test_truncated_numeric_code_recovers_complete_run(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="12345")++ result = await parse_otp_login(content="Your verification code is 123456.", organization_id="o_test")++ assert result is not None+ assert result.value == "123456" # the complete run, not the LLM's truncation++ @pytest.mark.asyncio+ async def test_truncation_ending_at_a_display_separator_recovers(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="123")++ result = await parse_otp_login(content="Your code is 123-456", organization_id="o_test")++ assert result is not None+ assert result.value == "123456" # a separator is not a token boundary for a code++ @pytest.mark.asyncio+ async def test_truncation_ending_at_a_space_separator_recovers(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="123")++ result = await parse_otp_login(content="Your code is 123 456 to continue.", organization_id="o_test")++ assert result is not None+ assert result.value == "123456"++ @pytest.mark.asyncio+ async def test_ambiguous_truncation_returns_nothing_rather_than_a_decoy(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="123")++ result = await parse_otp_login(content="Order 1234. Your code is 123-456", organization_id="o_test")++ assert result is None # "1234" scores higher than "123456"; serving an order number would be worse++ @pytest.mark.asyncio+ async def test_adjacent_numeric_field_is_not_welded_onto_the_code(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="123456")++ result = await parse_otp_login(content="Use code 123456 10 minutes remaining", organization_id="o_test")++ assert result is not None+ assert result.value == "123456" # not "12345610", a value that appears nowhere in the source++ @pytest.mark.asyncio+ async def test_tied_approximate_matches_return_nothing(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="123458")++ result = await parse_otp_login(content="Codes 123456 and 123457 were issued", organization_id="o_test")++ assert result is None # both score 0.833; picking either would be a guess++ @pytest.mark.asyncio+ async def test_code_repeated_in_the_message_is_one_candidate(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="12345")++ result = await parse_otp_login(+ content="Your code is 123456. Reminder: code 123456 expires soon.", organization_id="o_test"+ )++ assert result is not None+ assert result.value == "123456" # duplicates are not competing candidates++ @pytest.mark.asyncio+ async def test_bare_url_with_trailing_markup_is_not_fast_pathed(self, monkeypatch: pytest.MonkeyPatch) -> None:+ magic_link = "https://example.test/verify?token=abc"+ handler = _patch_otp_llm(monkeypatch, otp_type="magic_link", otp_value=magic_link)++ result = await parse_otp_login(content=f"{magic_link}<br>", organization_id="o_test")++ assert result is not None+ assert result.value == magic_link # markup excluded, not stored as part of the link+ handler.assert_awaited() # markup means it is not a bare URL, so the LLM path runs++ @pytest.mark.asyncio+ async def test_alphanumeric_truncation_at_separator_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="AB")++ result = await parse_otp_login(content="Your code is AB-12", organization_id="o_test")++ assert result is None++ @pytest.mark.asyncio+ async def test_alphanumeric_code_accepted_when_complete(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="AB12CD")++ result = await parse_otp_login(content="Your code is AB12CD - it expires soon.", organization_id="o_test")++ assert result is not None+ assert result.value == "AB12CD"++ @pytest.mark.asyncio+ async def test_truncated_alphanumeric_code_is_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _patch_otp_llm(monkeypatch, otp_type="totp", otp_value="AB12C")++ result = await parse_otp_login(content="Your code is AB12CD - it expires soon.", organization_id="o_test")++ assert result is None # no digit-run candidate to recover from; better than storing a short code+@pytest.mark.asyncioasync def test_does_not_charge_when_llm_call_raises(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError