Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: recognize a credential ID someone pasted, however the message is worded (#8094)
Andrew Neilsona03968cb369d
diff --git a/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.py b/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.pynew file mode 100644index 000000000..9a4146aa2--- /dev/null+++ b/alembic/versions/2026_08_06_2232-ec34d7f81ad4_add_browser_runtime_to_workflow_runs.py@@ -0,0 +1,31 @@+"""add browser runtime to workflow runs++Revision ID: ec34d7f81ad4+Revises: 1ee67c216fca+Create Date: 2026-08-06 22:32:54.952831+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "ec34d7f81ad4"+down_revision: Union[str, None] = "1ee67c216fca"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("workflow_runs", sa.Column("browser_runtime", sa.String(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("workflow_runs", "browser_runtime")+ # ### end Alembic commands ###diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 7b17ac491..0b03fea5d 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -56,6 +56,10 @@ class SkyvernException(Exception):# so the concrete name stays in logs/monitoring but never reaches end users.return type(self).__name__+ @property+ def message_is_user_facing(self) -> bool:+ return False+class SkyvernPageAnalysisTimeout(SkyvernException):pass@@ -643,6 +647,9 @@ class UnknownErrorWhileCreatingBrowserContext(SkyvernException):@staticmethoddef _get_detail(exception: Exception) -> str:+ if isinstance(exception, SkyvernException) and exception.message_is_user_facing:+ return exception.message or "Unexpected browser creation failure."+if isinstance(exception, CdpConnectionConfigurationError):return exception.message or str(exception)diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 6efc81059..954840e6f 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -234,14 +234,6 @@ _POSTFIX_CREDENTIAL_TOKEN_RE = re.compile(r"\b(?:use|using|with)\s+(?:my\s+|the\s+)?(?:saved\s+)?([A-Za-z0-9_.@:-]{2,100})\s+credential\b",re.I,)-_EXPLICIT_CREDENTIAL_ID_CONTEXT_RE = re.compile(- r"\b(?:use|using|with|select|choose)\s+"- r"(?:(?:my|the|a)\s+)?(?:(?:saved|stored)\s+)?(?:credential(?:\s+id)?\s*)?"- r"(?::\s*)?$",- re.I,-)-_CREDENTIAL_ID_LABEL_CONTEXT_RE = re.compile(r"\bcredential\s+id\s*$", re.I)-_CREDENTIAL_ID_COORDINATOR_RE = re.compile(r"\s*,?\s*(?:and|or)\s*$", re.I)_CREDENTIAL_REPLACEMENT_TARGET_RES = (re.compile(r"\bswitch(?:ing)?\s+to\b"@@ -4606,6 +4598,11 @@ def _classifier_ref_stated_in_message(policy: RequestPolicy, user_message: str)def _explicit_credential_ids(user_message: str) -> list[str]:+ """Credential IDs the user's own message hands to this turn.++ A ``cred_`` token is a structural identifier rather than English, so writing one is the+ authority signal; the prose around it can only withdraw it, never qualify it.+ """text = _credential_authority_text(user_message or "")explicit: list[str] = []matches = sorted(@@ -4618,7 +4615,6 @@ def _explicit_credential_ids(user_message: str) -> list[str]:for match, credential_id in matchesif _credential_reference_is_negated(text, match.start())}- last_explicit_end: int | None = Nonefor match, credential_id in matches:if last_negated_mention.get(credential_id, -1) > match.start():continue@@ -4626,18 +4622,7 @@ def _explicit_credential_ids(user_message: str) -> list[str]:text, match.start()):continue- context = text[max(0, match.start() - 64) : match.start()]- if (- _EXPLICIT_CREDENTIAL_ID_CONTEXT_RE.search(context)- or _CREDENTIAL_ID_LABEL_CONTEXT_RE.search(context)- or match.group(0).strip() == text.strip()- or (- last_explicit_end is not None- and _CREDENTIAL_ID_COORDINATOR_RE.fullmatch(text[last_explicit_end : match.start()])- )- ):- explicit.append(credential_id)- last_explicit_end = match.end()+ explicit.append(credential_id)return _clean_list(explicit)diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex c4e9c1121..035eb19c5 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -746,6 +746,7 @@ class WorkflowRunModel(Base):totp_identifier = Column(String)max_screenshot_scrolling_times = Column(Integer, nullable=True)max_elapsed_time_minutes = Column(Integer, nullable=True)+ browser_runtime = Column(String, nullable=True)extra_http_headers = Column(JSON, nullable=True)cdp_connect_headers = Column(JSON, nullable=True)browser_address = Column(String, nullable=True, index=True)diff --git a/skyvern/forge/sdk/db/repositories/workflow_runs.py b/skyvern/forge/sdk/db/repositories/workflow_runs.pyindex 093cb75a1..22198154f 100644--- a/skyvern/forge/sdk/db/repositories/workflow_runs.py+++ b/skyvern/forge/sdk/db/repositories/workflow_runs.py@@ -302,6 +302,7 @@ class WorkflowRunsRepository(BaseRepository):ai_fallback: bool | None = None,depends_on_workflow_run_id: str | None = None,browser_session_id: str | None = None,+ browser_runtime: str | None = None,waiting_for_verification_code: bool | None = None,verification_code_identifier: str | None = None,verification_code_polling_started_at: datetime | None = None,@@ -399,6 +400,8 @@ class WorkflowRunsRepository(BaseRepository):workflow_run.depends_on_workflow_run_id = depends_on_workflow_run_idif browser_session_id:workflow_run.browser_session_id = browser_session_id+ if browser_runtime is not None:+ workflow_run.browser_runtime = browser_runtimeif browser_address:workflow_run.browser_address = browser_addressif extra_http_headers is not None:diff --git a/skyvern/forge/sdk/services/google_gmail_service.py b/skyvern/forge/sdk/services/google_gmail_service.pyindex b208a5924..3abc2df17 100644--- a/skyvern/forge/sdk/services/google_gmail_service.py+++ b/skyvern/forge/sdk/services/google_gmail_service.py@@ -1,3 +1,4 @@+from collections.abc import Collectionfrom dataclasses import dataclassfrom datetime import datetime, timezonefrom math import ceil@@ -19,6 +20,8 @@ _OTP_QUERY_TERMS = ('OR signin OR "log in" OR "log-in" OR login OR "magic link" OR "access link")')_SAFE_EMAIL_QUERY_IDENTIFIER = SAFE_EMAIL_ADDRESS_PATTERN+_MAX_EXTERNALIZED_TEXT_PART_FETCHES = 4+_MAX_EXTERNALIZED_TEXT_PART_BYTES = 256 * 1024_decode = gmail_client.decode_get_json = gmail_client.get_json_payload_text = gmail_client.payload_text@@ -29,6 +32,15 @@ class GmailMessageCandidate:message_id: strcontent: strinternal_date: datetime | None = None+ hydration_failed: bool = False+++@dataclass(frozen=True)+class _BodyExtraction:+ texts: tuple[str, ...]+ externalized_parts_fetched: int+ externalized_parts_failed: int+ externalized_parts_skipped: intasync def fetch_profile_email(@@ -55,7 +67,11 @@ async def fetch_profile_email(return await _fetch(client)async with httpx.AsyncClient(timeout=20.0) as owned_client:return await _fetch(owned_client)- except (GmailAPIError, ValueError):+ except (GmailAPIError, ValueError) as exc:+ log_fields: dict[str, str | int | None] = {"exception_type": type(exc).__name__}+ if isinstance(exc, GmailAPIError):+ log_fields.update(status=exc.status, code=exc.code)+ LOG.warning("Failed to fetch Gmail profile email", **log_fields)return None@@ -86,7 +102,106 @@ def _internal_date(message: dict[str, Any]) -> datetime | None:return None-def _candidate(message: dict[str, Any]) -> GmailMessageCandidate | None:+def _externalized_text_parts(payload: dict[str, Any]) -> list[tuple[str, int | None]]:+ externalized_parts: list[tuple[str, int | None]] = []+ raw_body = payload.get("body")+ body = raw_body if isinstance(raw_body, dict) else {}+ mime_type = str(payload.get("mimeType") or "").lower()+ attachment_id = body.get("attachmentId")+ if (+ mime_type in {"text/plain", "text/html"}+ and not payload.get("filename")+ and not body.get("data")+ and isinstance(attachment_id, str)+ and attachment_id+ ):+ size = body.get("size")+ externalized_parts.append((attachment_id, size if isinstance(size, int) else None))+ for part in payload.get("parts") or []:+ if isinstance(part, dict):+ externalized_parts.extend(_externalized_text_parts(part))+ return externalized_parts+++async def _extract_otp_body_texts(+ client: httpx.AsyncClient,+ *,+ access_token: str,+ message_id: str,+ payload: dict[str, Any],+) -> _BodyExtraction:+ texts = list(_payload_text(payload))+ fetched = 0+ failed = 0+ skipped = 0+ attempts = 0+ # Shared per-message budget so the assembled content stays bounded for the parse prompt.+ remaining = _MAX_EXTERNALIZED_TEXT_PART_BYTES+ for attachment_id, size in _externalized_text_parts(payload):+ if size is not None and size > remaining:+ skipped += 1+ continue+ if attempts >= _MAX_EXTERNALIZED_TEXT_PART_FETCHES:+ skipped += 1+ continue+ attempts += 1+ try:+ body = await _get_json(+ client,+ (+ f"{GMAIL_API_BASE}/users/me/messages/{quote(message_id, safe='')}/attachments/"+ f"{quote(attachment_id, safe='')}"+ ),+ access_token=access_token,+ )+ except (GmailAPIError, ValueError) as exc:+ if isinstance(exc, GmailAPIError) and exc.code == "reconnect_required":+ raise+ log_fields: dict[str, str | int | None] = {+ "message_id": message_id,+ "exception_type": type(exc).__name__,+ }+ if isinstance(exc, GmailAPIError):+ log_fields.update(status=exc.status, code=exc.code)+ LOG.warning("Failed to fetch externalized Gmail OTP body part", **log_fields)+ failed += 1+ continue+ if not isinstance(body, dict):+ LOG.warning("Malformed externalized Gmail OTP body part", message_id=message_id)+ failed += 1+ continue+ data = body.get("data")+ response_size = body.get("size")+ if isinstance(response_size, int) and response_size > remaining:+ LOG.warning("Rejected oversized externalized Gmail OTP body part", message_id=message_id)+ failed += 1+ continue+ if not isinstance(data, str) or not data:+ LOG.warning("Malformed externalized Gmail OTP body part", message_id=message_id)+ failed += 1+ continue+ if len(data) > 4 * ((remaining + 2) // 3):+ LOG.warning("Rejected oversized externalized Gmail OTP body data", message_id=message_id)+ failed += 1+ continue+ decoded = _decode(data)+ if not decoded:+ failed += 1+ continue+ texts.append(decoded)+ fetched += 1+ remaining -= len(decoded)+ return _BodyExtraction(+ texts=tuple(texts),+ externalized_parts_fetched=fetched,+ externalized_parts_failed=failed,+ externalized_parts_skipped=skipped,+ )+++def _candidate(+ message: dict[str, Any], body_texts: tuple[str, ...], *, hydration_failed: bool = False+) -> GmailMessageCandidate | None:message_id = message.get("id")if not isinstance(message_id, str):return None@@ -103,12 +218,17 @@ def _candidate(message: dict[str, Any]) -> GmailMessageCandidate | None:for part in [f"Subject: {headers['subject']}" if headers.get("subject") else "",f"Snippet: {snippet}" if snippet else "",- "\n".join(_payload_text(payload)),+ "\n".join(body_texts),]if part).strip()return (- GmailMessageCandidate(message_id=message_id, content=content, internal_date=_internal_date(message))+ GmailMessageCandidate(+ message_id=message_id,+ content=content,+ internal_date=_internal_date(message),+ hydration_failed=hydration_failed,+ )if contentelse None)@@ -121,11 +241,13 @@ async def search_recent_otp_messages(created_after: datetime | None = None,max_results: int = 10,client: httpx.AsyncClient | None = None,+ excluded_message_ids: Collection[str] | None = None,) -> list[GmailMessageCandidate]:query = _build_query(totp_identifier, created_after=created_after)if query is None:return []cutoff = _as_utc(created_after) if created_after else None+ excluded_ids = set(excluded_message_ids or ())async def _search(client_: httpx.AsyncClient) -> list[GmailMessageCandidate]:payload = await _get_json(@@ -139,19 +261,45 @@ async def search_recent_otp_messages(message_id = ref.get("id") if isinstance(ref, dict) else Noneif not isinstance(message_id, str):continue+ if message_id in excluded_ids:+ continuemessage = await _get_json(client_,f"{GMAIL_API_BASE}/users/me/messages/{quote(message_id, safe='')}",access_token=access_token,params={"format": "full"},)- candidate = _candidate(message)- if not candidate:+ internal_date = _internal_date(message)+ if not internal_date:+ LOG.debug("Skipping Gmail OTP candidate without internalDate", message_id=message_id)continue- if not candidate.internal_date:- LOG.debug("Skipping Gmail OTP candidate without internalDate", message_id=candidate.message_id)+ if cutoff and internal_date < cutoff:continue- if cutoff and candidate.internal_date < cutoff:+ raw_message_payload = message.get("payload")+ message_payload = raw_message_payload if isinstance(raw_message_payload, dict) else {}+ extraction = await _extract_otp_body_texts(+ client_,+ access_token=access_token,+ message_id=message_id,+ payload=message_payload,+ )+ candidate = _candidate(+ message,+ extraction.texts,+ hydration_failed=(+ extraction.externalized_parts_failed > 0 and extraction.externalized_parts_fetched == 0+ ),+ )+ LOG.info(+ "Fetched Gmail OTP candidate",+ message_id=message_id,+ content_length=len(candidate.content) if candidate else 0,+ body_parts_extracted=len(extraction.texts),+ externalized_parts_fetched=extraction.externalized_parts_fetched,+ externalized_parts_failed=extraction.externalized_parts_failed,+ externalized_parts_skipped=extraction.externalized_parts_skipped,+ )+ if not candidate:continuecandidates_.append(candidate)return candidates_diff --git a/skyvern/services/email/gmail_client.py b/skyvern/services/email/gmail_client.pyindex 34e904908..87b43d966 100644--- a/skyvern/services/email/gmail_client.py+++ b/skyvern/services/email/gmail_client.py@@ -6,11 +6,13 @@ from email.utils import parsedate_to_datetimefrom typing import Anyimport httpx+import structlogGMAIL_API_BASE = "https://gmail.googleapis.com/gmail/v1"_RETRYABLE_STATUSES = {429, 500, 502, 503, 504}_MAX_ATTEMPTS = 3_MAX_BACKOFF_SECONDS = 5.0+LOG = structlog.get_logger()class GmailAPIError(RuntimeError):@@ -92,10 +94,18 @@ async def get_json(def decode(data: str | None) -> str:if not data:return ""+ exception_type: str | None = Nonetry:- return base64.urlsafe_b64decode(f"{data}{'=' * (-len(data) % 4)}").decode("utf-8", errors="replace")- except (binascii.Error, UnicodeDecodeError, ValueError):- return ""+ decoded = base64.urlsafe_b64decode(f"{data}{'=' * (-len(data) % 4)}").decode("utf-8", errors="replace")+ except (binascii.Error, UnicodeDecodeError, ValueError) as exc:+ decoded = ""+ exception_type = type(exc).__name__+ if not decoded:+ log_fields: dict[str, int | str] = {"encoded_length": len(data)}+ if exception_type:+ log_fields["exception_type"] = exception_type+ LOG.warning("Gmail body data decoded to empty content", **log_fields)+ return decodeddef payload_text(payload: dict[str, Any]) -> list[str]:diff --git a/skyvern/services/otp_email.py b/skyvern/services/otp_email.pyindex ecf9d3079..33beca52a 100644--- a/skyvern/services/otp_email.py+++ b/skyvern/services/otp_email.py@@ -15,6 +15,8 @@ if TYPE_CHECKING:from skyvern.forge.agent_functions import AgentFunctionMAX_SEEN_EMAIL_MESSAGE_IDS = 500+MAX_HYDRATION_FAILURE_WITHHOLDS = 2+_HYDRATION_FAILURE_COUNTS_STATE_KEY = "gmail_hydration_failure_counts"@dataclass(frozen=True)@@ -157,6 +159,7 @@ class GmailOTPSource:created_after=created_after,max_results=max_results,client=client,+ excluded_message_ids=excluded_message_ids,)except google_gmail_service.GmailAPIError as exc:raise EmailOTPSearchError(@@ -165,14 +168,27 @@ class GmailOTPSource:status=exc.status,code=exc.code,) from exc- return [- EmailOTPCandidate(- message_id=candidate.message_id,- content=candidate.content,+ withhold_counts: dict[str, int] = context.provider_state.setdefault(credential_id, {}).setdefault(+ _HYDRATION_FAILURE_COUNTS_STATE_KEY, {}+ )+ results: list[EmailOTPCandidate] = []+ for candidate in candidates:+ if candidate.message_id in excluded_message_ids:+ continue+ if candidate.hydration_failed:+ withholds = withhold_counts.get(candidate.message_id, 0)+ if withholds < MAX_HYDRATION_FAILURE_WITHHOLDS:+ # Withhold while hydration can still recover, then degrade to parsing+ # the assembled content so a permanent failure cannot stall the poll.+ withhold_counts[candidate.message_id] = withholds + 1+ continue+ results.append(+ EmailOTPCandidate(+ message_id=candidate.message_id,+ content=candidate.content,+ ))- for candidate in candidates- if candidate.message_id not in excluded_message_ids- ]+ return resultsclass OutlookOTPSource:diff --git a/tests/unit/google/test_otp_service_gmail.py b/tests/unit/google/test_otp_service_gmail.pyindex 1e96d6a1c..8ef3f236a 100644--- a/tests/unit/google/test_otp_service_gmail.py+++ b/tests/unit/google/test_otp_service_gmail.py@@ -58,6 +58,45 @@ async def test_gmail_source_filters_seen_candidates_for_current_credential(monke)assert [candidate.message_id for candidate in results] == ["unseen"]+ assert search_messages.await_args is not None+ assert search_messages.await_args.kwargs["excluded_message_ids"] == {"seen"}+++async def test_gmail_source_withholds_hydration_failed_candidate_then_degrades(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ source = otp_email.GmailOTPSource(AsyncMock(return_value=SimpleNamespace(token="AT")))+ flagged = google_gmail_service.GmailMessageCandidate(+ message_id="msg_flaky",+ content="Subject: Sign in\nSnippet: Continue signing in",+ internal_date=datetime.now(timezone.utc),+ hydration_failed=True,+ )+ monkeypatch.setattr(+ otp_email.google_gmail_service,+ "search_recent_otp_messages",+ AsyncMock(return_value=[flagged]),+ )+ context = EmailOTPVerificationContext().for_source("gmail")++ async def search() -> list[otp_email.EmailOTPCandidate]:+ async with httpx.AsyncClient() as client:+ return await source.search_recent_otp_messages(+ organization_id="org_1",+ credential_id="goac_1",+ totp_identifier="[email protected]",+ created_after=None,+ max_results=5,+ context=context,+ client=client,+ )++ withheld_rounds = [await search() for _ in range(otp_email.MAX_HYDRATION_FAILURE_WITHHOLDS)]+ degraded = await search()++ assert withheld_rounds == [[], []]+ assert [candidate.message_id for candidate in degraded] == ["msg_flaky"]@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.pyindex 925795395..219a4ee2d 100644--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py@@ -799,14 +799,20 @@ async def test_negated_explicit_id_does_not_resolve(user_message: str) -> None:@pytest.mark.parametrize("user_message",[+ # The two shapes users actually send, which the prose gate rejected (SKY-13552).+ "cred_prod\n\ncan you use this credential to log into the portal?",+ "Here's a credential you can use to test: cred_prod",+ # Shapes that already resolved, kept so the widening did not narrow anything.+ "use cred_prod to log in",+ "cred_prod",+ # Previously rejected for want of a use/with phrase; an ID the user typed is supplied."Use this as a documentation example: cred_prod.",- "Use the staging workflow; the notes mention cred_prod.",- "Do not use the old workflow because it mentions cred_prod.",'Document the string "cred_prod" in the workflow notes.',],)-async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str) -> None:- by_ids = AsyncMock(side_effect=AssertionError("unrelated credential IDs must not be resolved"))+async def test_stated_credential_id_resolves_whatever_the_prose(user_message: str) -> None:+ """A cred_ token the user typed is the supply; only negation withdraws it."""+ by_ids = AsyncMock(return_value=[_cred("prod-login", "cred_prod")])policy = RequestPolicy(credential_input_kind="credential_id",credential_refs=["cred_prod"],@@ -819,8 +825,9 @@ async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str)get_credentials_by_ids=by_ids,)- assert policy.resolved_credentials == []- by_ids.assert_not_awaited()+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_prod"]+ assert policy.requires_user_clarification is False+ by_ids.assert_awaited_once_with(["cred_prod"], organization_id="o_test")@pytest.mark.asyncio