Skip to content

Automate browser based workflows with AI

Copilot: recognize a credential ID someone pasted, however the message is worded (#8094)

Andrew Neilsona03968cb369d
diff --git a/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.py b/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.py
new file mode 100644
index 000000000..9a4146aa2
--- /dev/null
+++ b/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.py
@@ -0,0 +1,31 @@
+"""add browser runtime to workflow runs
+
+Revision ID: ec34d7f81ad4
+Revises: 1ee67c216fca
+Create Date: 2026-08-06 22:32:54.952831+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "ec34d7f81ad4"
+down_revision: Union[str, None] = "1ee67c216fca"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("workflow_runs", sa.Column("browser_runtime", sa.String(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("workflow_runs", "browser_runtime")
+ # ### end Alembic commands ###
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 7b17ac491..0b03fea5d 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -56,6 +56,10 @@ class SkyvernException(Exception):
# so the concrete name stays in logs/monitoring but never reaches end users.
return type(self).__name__
+ @property
+ def message_is_user_facing(self) -> bool:
+ return False
+
class SkyvernPageAnalysisTimeout(SkyvernException):
pass
@@ -643,6 +647,9 @@ class UnknownErrorWhileCreatingBrowserContext(SkyvernException):
@staticmethod
def _get_detail(exception: Exception) -> str:
+ if isinstance(exception, SkyvernException) and exception.message_is_user_facing:
+ return exception.message or "Unexpected browser creation failure."
+
if isinstance(exception, CdpConnectionConfigurationError):
return exception.message or str(exception)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 6efc81059..954840e6f 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -234,14 +234,6 @@ _POSTFIX_CREDENTIAL_TOKEN_RE = re.compile(
r"\b(?:use|using|with)\s+(?:my\s+|the\s+)?(?:saved\s+)?([A-Za-z0-9_.@:-]{2,100})\s+credential\b",
re.I,
)
-_EXPLICIT_CREDENTIAL_ID_CONTEXT_RE = re.compile(
- r"\b(?:use|using|with|select|choose)\s+"
- r"(?:(?:my|the|a)\s+)?(?:(?:saved|stored)\s+)?(?:credential(?:\s+id)?\s*)?"
- r"(?::\s*)?$",
- re.I,
-)
-_CREDENTIAL_ID_LABEL_CONTEXT_RE = re.compile(r"\bcredential\s+id\s*$", re.I)
-_CREDENTIAL_ID_COORDINATOR_RE = re.compile(r"\s*,?\s*(?:and|or)\s*$", re.I)
_CREDENTIAL_REPLACEMENT_TARGET_RES = (
re.compile(
r"\bswitch(?:ing)?\s+to\b"
@@ -4606,6 +4598,11 @@ def _classifier_ref_stated_in_message(policy: RequestPolicy, user_message: str)
def _explicit_credential_ids(user_message: str) -> list[str]:
+ """Credential IDs the user's own message hands to this turn.
+
+ A ``cred_`` token is a structural identifier rather than English, so writing one is the
+ authority signal; the prose around it can only withdraw it, never qualify it.
+ """
text = _credential_authority_text(user_message or "")
explicit: list[str] = []
matches = sorted(
@@ -4618,7 +4615,6 @@ def _explicit_credential_ids(user_message: str) -> list[str]:
for match, credential_id in matches
if _credential_reference_is_negated(text, match.start())
}
- last_explicit_end: int | None = None
for match, credential_id in matches:
if last_negated_mention.get(credential_id, -1) > match.start():
continue
@@ -4626,18 +4622,7 @@ def _explicit_credential_ids(user_message: str) -> list[str]:
text, match.start()
):
continue
- context = text[max(0, match.start() - 64) : match.start()]
- if (
- _EXPLICIT_CREDENTIAL_ID_CONTEXT_RE.search(context)
- or _CREDENTIAL_ID_LABEL_CONTEXT_RE.search(context)
- or match.group(0).strip() == text.strip()
- or (
- last_explicit_end is not None
- and _CREDENTIAL_ID_COORDINATOR_RE.fullmatch(text[last_explicit_end : match.start()])
- )
- ):
- explicit.append(credential_id)
- last_explicit_end = match.end()
+ explicit.append(credential_id)
return _clean_list(explicit)
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index c4e9c1121..035eb19c5 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -746,6 +746,7 @@ class WorkflowRunModel(Base):
totp_identifier = Column(String)
max_screenshot_scrolling_times = Column(Integer, nullable=True)
max_elapsed_time_minutes = Column(Integer, nullable=True)
+ browser_runtime = Column(String, nullable=True)
extra_http_headers = Column(JSON, nullable=True)
cdp_connect_headers = Column(JSON, nullable=True)
browser_address = Column(String, nullable=True, index=True)
diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.py
index 093cb75a1..22198154f 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py
@@ -302,6 +302,7 @@ class WorkflowRunsRepository(BaseRepository):
ai_fallback: bool | None = None,
depends_on_workflow_run_id: str | None = None,
browser_session_id: str | None = None,
+ browser_runtime: str | None = None,
waiting_for_verification_code: bool | None = None,
verification_code_identifier: str | None = None,
verification_code_polling_started_at: datetime | None = None,
@@ -399,6 +400,8 @@ class WorkflowRunsRepository(BaseRepository):
workflow_run.depends_on_workflow_run_id = depends_on_workflow_run_id
if browser_session_id:
workflow_run.browser_session_id = browser_session_id
+ if browser_runtime is not None:
+ workflow_run.browser_runtime = browser_runtime
if browser_address:
workflow_run.browser_address = browser_address
if extra_http_headers is not None:
diff --git a/skyvern/forge/sdk/services/google_gmail_service.py b/skyvern/forge/sdk/services/google_gmail_service.py
index b208a5924..3abc2df17 100644
--- a/skyvern/forge/sdk/services/google_gmail_service.py
+++ b/skyvern/forge/sdk/services/google_gmail_service.py
@@ -1,3 +1,4 @@
+from collections.abc import Collection
from dataclasses import dataclass
from datetime import datetime, timezone
from math import ceil
@@ -19,6 +20,8 @@ _OTP_QUERY_TERMS = (
'OR signin OR "log in" OR "log-in" OR login OR "magic link" OR "access link")'
)
_SAFE_EMAIL_QUERY_IDENTIFIER = SAFE_EMAIL_ADDRESS_PATTERN
+_MAX_EXTERNALIZED_TEXT_PART_FETCHES = 4
+_MAX_EXTERNALIZED_TEXT_PART_BYTES = 256 * 1024
_decode = gmail_client.decode
_get_json = gmail_client.get_json
_payload_text = gmail_client.payload_text
@@ -29,6 +32,15 @@ class GmailMessageCandidate:
message_id: str
content: str
internal_date: datetime | None = None
+ hydration_failed: bool = False
+
+
+@dataclass(frozen=True)
+class _BodyExtraction:
+ texts: tuple[str, ...]
+ externalized_parts_fetched: int
+ externalized_parts_failed: int
+ externalized_parts_skipped: int
async def fetch_profile_email(
@@ -55,7 +67,11 @@ async def fetch_profile_email(
return await _fetch(client)
async with httpx.AsyncClient(timeout=20.0) as owned_client:
return await _fetch(owned_client)
- except (GmailAPIError, ValueError):
+ except (GmailAPIError, ValueError) as exc:
+ log_fields: dict[str, str | int | None] = {"exception_type": type(exc).__name__}
+ if isinstance(exc, GmailAPIError):
+ log_fields.update(status=exc.status, code=exc.code)
+ LOG.warning("Failed to fetch Gmail profile email", **log_fields)
return None
@@ -86,7 +102,106 @@ def _internal_date(message: dict[str, Any]) -> datetime | None:
return None
-def _candidate(message: dict[str, Any]) -> GmailMessageCandidate | None:
+def _externalized_text_parts(payload: dict[str, Any]) -> list[tuple[str, int | None]]:
+ externalized_parts: list[tuple[str, int | None]] = []
+ raw_body = payload.get("body")
+ body = raw_body if isinstance(raw_body, dict) else {}
+ mime_type = str(payload.get("mimeType") or "").lower()
+ attachment_id = body.get("attachmentId")
+ if (
+ mime_type in {"text/plain", "text/html"}
+ and not payload.get("filename")
+ and not body.get("data")
+ and isinstance(attachment_id, str)
+ and attachment_id
+ ):
+ size = body.get("size")
+ externalized_parts.append((attachment_id, size if isinstance(size, int) else None))
+ for part in payload.get("parts") or []:
+ if isinstance(part, dict):
+ externalized_parts.extend(_externalized_text_parts(part))
+ return externalized_parts
+
+
+async def _extract_otp_body_texts(
+ client: httpx.AsyncClient,
+ *,
+ access_token: str,
+ message_id: str,
+ payload: dict[str, Any],
+) -> _BodyExtraction:
+ texts = list(_payload_text(payload))
+ fetched = 0
+ failed = 0
+ skipped = 0
+ attempts = 0
+ # Shared per-message budget so the assembled content stays bounded for the parse prompt.
+ remaining = _MAX_EXTERNALIZED_TEXT_PART_BYTES
+ for attachment_id, size in _externalized_text_parts(payload):
+ if size is not None and size > remaining:
+ skipped += 1
+ continue
+ if attempts >= _MAX_EXTERNALIZED_TEXT_PART_FETCHES:
+ skipped += 1
+ continue
+ attempts += 1
+ try:
+ body = await _get_json(
+ client,
+ (
+ f"{GMAIL_API_BASE}/users/me/messages/{quote(message_id, safe='')}/attachments/"
+ f"{quote(attachment_id, safe='')}"
+ ),
+ access_token=access_token,
+ )
+ except (GmailAPIError, ValueError) as exc:
+ if isinstance(exc, GmailAPIError) and exc.code == "reconnect_required":
+ raise
+ log_fields: dict[str, str | int | None] = {
+ "message_id": message_id,
+ "exception_type": type(exc).__name__,
+ }
+ if isinstance(exc, GmailAPIError):
+ log_fields.update(status=exc.status, code=exc.code)
+ LOG.warning("Failed to fetch externalized Gmail OTP body part", **log_fields)
+ failed += 1
+ continue
+ if not isinstance(body, dict):
+ LOG.warning("Malformed externalized Gmail OTP body part", message_id=message_id)
+ failed += 1
+ continue
+ data = body.get("data")
+ response_size = body.get("size")
+ if isinstance(response_size, int) and response_size > remaining:
+ LOG.warning("Rejected oversized externalized Gmail OTP body part", message_id=message_id)
+ failed += 1
+ continue
+ if not isinstance(data, str) or not data:
+ LOG.warning("Malformed externalized Gmail OTP body part", message_id=message_id)
+ failed += 1
+ continue
+ if len(data) > 4 * ((remaining + 2) // 3):
+ LOG.warning("Rejected oversized externalized Gmail OTP body data", message_id=message_id)
+ failed += 1
+ continue
+ decoded = _decode(data)
+ if not decoded:
+ failed += 1
+ continue
+ texts.append(decoded)
+ fetched += 1
+ remaining -= len(decoded)
+ return _BodyExtraction(
+ texts=tuple(texts),
+ externalized_parts_fetched=fetched,
+ externalized_parts_failed=failed,
+ externalized_parts_skipped=skipped,
+ )
+
+
+def _candidate(
+ message: dict[str, Any], body_texts: tuple[str, ...], *, hydration_failed: bool = False
+) -> GmailMessageCandidate | None:
message_id = message.get("id")
if not isinstance(message_id, str):
return None
@@ -103,12 +218,17 @@ def _candidate(message: dict[str, Any]) -> GmailMessageCandidate | None:
for part in [
f"Subject: {headers['subject']}" if headers.get("subject") else "",
f"Snippet: {snippet}" if snippet else "",
- "\n".join(_payload_text(payload)),
+ "\n".join(body_texts),
]
if part
).strip()
return (
- GmailMessageCandidate(message_id=message_id, content=content, internal_date=_internal_date(message))
+ GmailMessageCandidate(
+ message_id=message_id,
+ content=content,
+ internal_date=_internal_date(message),
+ hydration_failed=hydration_failed,
+ )
if content
else None
)
@@ -121,11 +241,13 @@ async def search_recent_otp_messages(
created_after: datetime | None = None,
max_results: int = 10,
client: httpx.AsyncClient | None = None,
+ excluded_message_ids: Collection[str] | None = None,
) -> list[GmailMessageCandidate]:
query = _build_query(totp_identifier, created_after=created_after)
if query is None:
return []
cutoff = _as_utc(created_after) if created_after else None
+ excluded_ids = set(excluded_message_ids or ())
async def _search(client_: httpx.AsyncClient) -> list[GmailMessageCandidate]:
payload = await _get_json(
@@ -139,19 +261,45 @@ async def search_recent_otp_messages(
message_id = ref.get("id") if isinstance(ref, dict) else None
if not isinstance(message_id, str):
continue
+ if message_id in excluded_ids:
+ continue
message = await _get_json(
client_,
f"{GMAIL_API_BASE}/users/me/messages/{quote(message_id, safe='')}",
access_token=access_token,
params={"format": "full"},
)
- candidate = _candidate(message)
- if not candidate:
+ internal_date = _internal_date(message)
+ if not internal_date:
+ LOG.debug("Skipping Gmail OTP candidate without internalDate", message_id=message_id)
continue
- if not candidate.internal_date:
- LOG.debug("Skipping Gmail OTP candidate without internalDate", message_id=candidate.message_id)
+ if cutoff and internal_date < cutoff:
continue
- if cutoff and candidate.internal_date < cutoff:
+ raw_message_payload = message.get("payload")
+ message_payload = raw_message_payload if isinstance(raw_message_payload, dict) else {}
+ extraction = await _extract_otp_body_texts(
+ client_,
+ access_token=access_token,
+ message_id=message_id,
+ payload=message_payload,
+ )
+ candidate = _candidate(
+ message,
+ extraction.texts,
+ hydration_failed=(
+ extraction.externalized_parts_failed > 0 and extraction.externalized_parts_fetched == 0
+ ),
+ )
+ LOG.info(
+ "Fetched Gmail OTP candidate",
+ message_id=message_id,
+ content_length=len(candidate.content) if candidate else 0,
+ body_parts_extracted=len(extraction.texts),
+ externalized_parts_fetched=extraction.externalized_parts_fetched,
+ externalized_parts_failed=extraction.externalized_parts_failed,
+ externalized_parts_skipped=extraction.externalized_parts_skipped,
+ )
+ if not candidate:
continue
candidates_.append(candidate)
return candidates_
diff --git a/skyvern/services/email/gmail_client.py b/skyvern/services/email/gmail_client.py
index 34e904908..87b43d966 100644
--- a/skyvern/services/email/gmail_client.py
+++ b/skyvern/services/email/gmail_client.py
@@ -6,11 +6,13 @@ from email.utils import parsedate_to_datetime
from typing import Any
import httpx
+import structlog
GMAIL_API_BASE = "https://gmail.googleapis.com/gmail/v1"
_RETRYABLE_STATUSES = {429, 500, 502, 503, 504}
_MAX_ATTEMPTS = 3
_MAX_BACKOFF_SECONDS = 5.0
+LOG = structlog.get_logger()
class GmailAPIError(RuntimeError):
@@ -92,10 +94,18 @@ async def get_json(
def decode(data: str | None) -> str:
if not data:
return ""
+ exception_type: str | None = None
try:
- return base64.urlsafe_b64decode(f"{data}{'=' * (-len(data) % 4)}").decode("utf-8", errors="replace")
- except (binascii.Error, UnicodeDecodeError, ValueError):
- return ""
+ decoded = base64.urlsafe_b64decode(f"{data}{'=' * (-len(data) % 4)}").decode("utf-8", errors="replace")
+ except (binascii.Error, UnicodeDecodeError, ValueError) as exc:
+ decoded = ""
+ exception_type = type(exc).__name__
+ if not decoded:
+ log_fields: dict[str, int | str] = {"encoded_length": len(data)}
+ if exception_type:
+ log_fields["exception_type"] = exception_type
+ LOG.warning("Gmail body data decoded to empty content", **log_fields)
+ return decoded
def payload_text(payload: dict[str, Any]) -> list[str]:
diff --git a/skyvern/services/otp_email.py b/skyvern/services/otp_email.py
index ecf9d3079..33beca52a 100644
--- a/skyvern/services/otp_email.py
+++ b/skyvern/services/otp_email.py
@@ -15,6 +15,8 @@ if TYPE_CHECKING:
from skyvern.forge.agent_functions import AgentFunction
MAX_SEEN_EMAIL_MESSAGE_IDS = 500
+MAX_HYDRATION_FAILURE_WITHHOLDS = 2
+_HYDRATION_FAILURE_COUNTS_STATE_KEY = "gmail_hydration_failure_counts"
@dataclass(frozen=True)
@@ -157,6 +159,7 @@ class GmailOTPSource:
created_after=created_after,
max_results=max_results,
client=client,
+ excluded_message_ids=excluded_message_ids,
)
except google_gmail_service.GmailAPIError as exc:
raise EmailOTPSearchError(
@@ -165,14 +168,27 @@ class GmailOTPSource:
status=exc.status,
code=exc.code,
) from exc
- return [
- EmailOTPCandidate(
- message_id=candidate.message_id,
- content=candidate.content,
+ withhold_counts: dict[str, int] = context.provider_state.setdefault(credential_id, {}).setdefault(
+ _HYDRATION_FAILURE_COUNTS_STATE_KEY, {}
+ )
+ results: list[EmailOTPCandidate] = []
+ for candidate in candidates:
+ if candidate.message_id in excluded_message_ids:
+ continue
+ if candidate.hydration_failed:
+ withholds = withhold_counts.get(candidate.message_id, 0)
+ if withholds < MAX_HYDRATION_FAILURE_WITHHOLDS:
+ # Withhold while hydration can still recover, then degrade to parsing
+ # the assembled content so a permanent failure cannot stall the poll.
+ withhold_counts[candidate.message_id] = withholds + 1
+ continue
+ results.append(
+ EmailOTPCandidate(
+ message_id=candidate.message_id,
+ content=candidate.content,
+ )
)
- for candidate in candidates
- if candidate.message_id not in excluded_message_ids
- ]
+ return results
class OutlookOTPSource:
diff --git a/tests/unit/google/test_otp_service_gmail.py b/tests/unit/google/test_otp_service_gmail.py
index 1e96d6a1c..8ef3f236a 100644
--- a/tests/unit/google/test_otp_service_gmail.py
+++ b/tests/unit/google/test_otp_service_gmail.py
@@ -58,6 +58,45 @@ async def test_gmail_source_filters_seen_candidates_for_current_credential(monke
)
assert [candidate.message_id for candidate in results] == ["unseen"]
+ assert search_messages.await_args is not None
+ assert search_messages.await_args.kwargs["excluded_message_ids"] == {"seen"}
+
+
+async def test_gmail_source_withholds_hydration_failed_candidate_then_degrades(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ source = otp_email.GmailOTPSource(AsyncMock(return_value=SimpleNamespace(token="AT")))
+ flagged = google_gmail_service.GmailMessageCandidate(
+ message_id="msg_flaky",
+ content="Subject: Sign in\nSnippet: Continue signing in",
+ internal_date=datetime.now(timezone.utc),
+ hydration_failed=True,
+ )
+ monkeypatch.setattr(
+ otp_email.google_gmail_service,
+ "search_recent_otp_messages",
+ AsyncMock(return_value=[flagged]),
+ )
+ context = EmailOTPVerificationContext().for_source("gmail")
+
+ async def search() -> list[otp_email.EmailOTPCandidate]:
+ async with httpx.AsyncClient() as client:
+ return await source.search_recent_otp_messages(
+ organization_id="org_1",
+ credential_id="goac_1",
+ totp_identifier="[email protected]",
+ created_after=None,
+ max_results=5,
+ context=context,
+ client=client,
+ )
+
+ withheld_rounds = [await search() for _ in range(otp_email.MAX_HYDRATION_FAILURE_WITHHOLDS)]
+ degraded = await search()
+
+ assert withheld_rounds == [[], []]
+ assert [candidate.message_id for candidate in degraded] == ["msg_flaky"]
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 925795395..219a4ee2d 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -799,14 +799,20 @@ async def test_negated_explicit_id_does_not_resolve(user_message: str) -> None:
@pytest.mark.parametrize(
"user_message",
[
+ # The two shapes users actually send, which the prose gate rejected (SKY-13552).
+ "cred_prod\n\ncan you use this credential to log into the portal?",
+ "Here's a credential you can use to test: cred_prod",
+ # Shapes that already resolved, kept so the widening did not narrow anything.
+ "use cred_prod to log in",
+ "cred_prod",
+ # Previously rejected for want of a use/with phrase; an ID the user typed is supplied.
"Use this as a documentation example: cred_prod.",
- "Use the staging workflow; the notes mention cred_prod.",
- "Do not use the old workflow because it mentions cred_prod.",
'Document the string "cred_prod" in the workflow notes.',
],
)
-async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str) -> None:
- by_ids = AsyncMock(side_effect=AssertionError("unrelated credential IDs must not be resolved"))
+async def test_stated_credential_id_resolves_whatever_the_prose(user_message: str) -> None:
+ """A cred_ token the user typed is the supply; only negation withdraws it."""
+ by_ids = AsyncMock(return_value=[_cred("prod-login", "cred_prod")])
policy = RequestPolicy(
credential_input_kind="credential_id",
credential_refs=["cred_prod"],
@@ -819,8 +825,9 @@ async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str)
get_credentials_by_ids=by_ids,
)
- assert policy.resolved_credentials == []
- by_ids.assert_not_awaited()
+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_prod"]
+ assert policy.requires_user_clarification is False
+ by_ids.assert_awaited_once_with(["cred_prod"], organization_id="o_test")
@pytest.mark.asyncio