Skip to content

Automate browser based workflows with AI

Remove three Copilot sign-in fields that were always false (SKY-14054) (#8211)

Andrew Neilson0cbfce0b88fb
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 1507fc27f..c98996633 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -82,7 +82,6 @@ from skyvern.forge.sdk.copilot.context import (
finalize_observation_context,
parsed_ask_refs,
record_approved_credentials_in_global_llm_context,
- record_signin_email_in_global_llm_context,
sanitize_global_llm_context_for_prompt,
)
from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason, preflight_credential_pause
@@ -1014,10 +1013,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ "\n```\nThis block contains safety and request facts, not permission or a mandatory next action. "
+ "If `raw_secret_handling` is `redacted_draft`, build only from the redacted request, do not run "
+ "blocks, and tell the user to store the redacted secret as a saved credential before testing. "
- + "If `resolved_credentials` are present, use those `credential_id` values. "
- + "If `resolved_signin_email` is present, that address signs in: bind it to the workflow as a "
- + "workflow input parameter with that value as the default, so the saved workflow can be re-run "
- + "under a different address, and do not ask for a saved password credential for that login."
+ + "If `resolved_credentials` are present, use those `credential_id` values."
+ _runtime_verification_evidence_prompt(ctx)
+ _recorded_build_test_outcome_prompt(ctx)
+ _code_authoring_repair_context_prompt(ctx)
@@ -1402,12 +1398,7 @@ def _make_agent_result(
normal translate-result, missing-SDK fallback, unexpected-error fallback).
"""
final_context = (
- record_signin_email_in_global_llm_context(
- ctx,
- record_approved_credentials_in_global_llm_context(
- ctx, finalize_observation_context(ctx, global_llm_context)
- ),
- )
+ record_approved_credentials_in_global_llm_context(ctx, finalize_observation_context(ctx, global_llm_context))
if ctx is not None
else global_llm_context
)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 4d2353ade..76c6fe5f2 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
import re
import uuid
-from collections.abc import Collection, Mapping, Sequence
+from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Literal, get_args
@@ -251,12 +251,6 @@ class StructuredContext(BaseModel):
]
entrypoint_url: str | None = None
- # Passwordless sign-in identity, carried so a follow-up turn the classifier reads as an
- # ordinary login does not fall back to demanding a password the site does not have.
- # Scoped by host: reusing it anywhere else would suppress the password ask on a site
- # that never established it was passwordless.
- signin_email: str = ""
- signin_email_host: str = ""
def to_json_str(self) -> str:
payload = self.model_dump(mode="json")
@@ -511,35 +505,13 @@ def record_approved_credentials_in_global_llm_context(ctx: CopilotContext, raw_c
return sc.to_json_str()
-def record_signin_email_in_global_llm_context(ctx: CopilotContext, raw_context: str | None) -> str | None:
- policy = ctx.request_policy
- if policy is None or not policy.resolved_signin_email or not policy.resolved_signin_host:
- return raw_context
- sc = StructuredContext.from_json_str(raw_context)
- if sc.signin_email == policy.resolved_signin_email and sc.signin_email_host == policy.resolved_signin_host:
- return raw_context
- sc.signin_email = policy.resolved_signin_email
- sc.signin_email_host = policy.resolved_signin_host
- return sc.to_json_str()
-
-
-def prior_signin_email_from_context(raw_context: str | None, hosts: Collection[str]) -> str:
- """The carried address, only when this turn targets the host it was resolved for."""
- sc = StructuredContext.from_json_str(raw_context)
- if not sc.signin_email or sc.signin_email_host not in hosts:
- return ""
- return sc.signin_email
-
-
def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) -> StructuredContext:
"""Take the model's context but keep the server-owned fields server-owned.
Approval is recorded only from server-resolved credentials; an entry the model
supplied would be promoted into `resolved_credentials` on the next turn and clear
the unapproved-credential gate for a credential the user never named. Membership
- of the org is not evidence the user named it. A model-authored `signin_email` is the
- same promotion in the other gate: it marks the next turn passwordless and suppresses
- the password-credential ask for a site that does need one.
+ of the org is not evidence the user named it.
`carried_trajectory` is the record of what the browser was observed doing, so an
entry the model wrote would enter the factual record as an observation nothing made.
@@ -557,8 +529,6 @@ def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) ->
elif isinstance(model_raw, str):
structured = StructuredContext.from_json_str(model_raw)
structured.approved_credentials = list(trusted.approved_credentials)
- structured.signin_email = trusted.signin_email
- structured.signin_email_host = trusted.signin_email_host
structured.carried_trajectory = [dict(entry) for entry in trusted.carried_trajectory]
return structured
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index c33465b87..1af17c240 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -50,7 +50,6 @@ from skyvern.forge.sdk.copilot.secret_redaction import (
contains_email_password_pair,
redact_raw_secrets_for_prompt,
)
-from skyvern.forge.sdk.copilot.signin_email import is_email_address
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
URL_CANDIDATE_RE,
@@ -86,7 +85,6 @@ _CLASSIFICATION_RESPONSE_FIELDS = {
"credential_input_kind",
"credential_refs",
"login_page_urls",
- "login_intent",
"requires_user_clarification",
"completion_contract",
"completion_criteria",
@@ -111,7 +109,6 @@ ClarificationReason = Literal[
"missing_target_context",
"workflow_credential_inputs_unbound",
"login_credentials_unresolved",
- "signin_email_unresolved",
"safety_screen_unavailable",
]
RawSecretHandling = Literal["none", "block", "redacted_draft"]
@@ -347,7 +344,7 @@ _VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_arg
# Only deterministic post-resolution code may mint these; a classifier emission would
# skip the concrete-target and credential-reachability checks the reason stands for.
_DETERMINISTIC_ONLY_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(
- {"login_credentials_unresolved", "signin_email_unresolved", "safety_screen_unavailable"}
+ {"login_credentials_unresolved", "safety_screen_unavailable"}
)
# Concrete credential states for which the combined update-and-run tool persists
# the draft but cannot start a browser run.
@@ -788,13 +785,6 @@ class RequestPolicy:
credential_input_kind: str = "none"
credential_refs: list[str] = field(default_factory=list)
login_page_urls: list[str] = field(default_factory=list)
- login_intent: bool = False
- # Sign-in that identifies the user by email address instead of a stored password, so
- # resolving *which address* is the whole credential question — there is no password to find.
- email_signin_intent: bool = False
- signin_email_candidates: list[str] = field(default_factory=list)
- resolved_signin_email: str | None = None
- resolved_signin_host: str | None = None
requires_user_clarification: bool = False
allow_update_workflow: bool = True
allow_run_blocks: bool = True
@@ -869,9 +859,6 @@ class RequestPolicy:
"testing_intent": self.testing_intent,
"authoring_intent": self.authoring_intent,
"credential_input_kind": self.credential_input_kind,
- "login_intent": self.login_intent,
- "email_signin_intent": self.email_signin_intent,
- "signin_email_resolved": bool(self.resolved_signin_email),
"clarification_reason": self.clarification_reason,
"allow_update_workflow": self.allow_update_workflow,
"allow_run_blocks": self.allow_run_blocks,
@@ -979,8 +966,6 @@ class RequestPolicy:
]
if self.raw_secret_detected:
lines.append(f"raw_secret_detected: {self.raw_secret_detected}")
- if self.resolved_signin_email:
- lines.append(f"resolved_signin_email: {self.resolved_signin_email}")
validation_classification_criteria = [
criterion
for criterion in self.graded_completion_criteria()
@@ -1260,10 +1245,6 @@ def _clean_list(values: list[Any]) -> list[str]:
return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))
-def _clean_email_list(values: list[Any]) -> list[str]:
- return [value for value in _clean_list(values) if is_email_address(value)]
-
-
def _credential_ids(text: str) -> list[str]:
text = text or ""
canonical = _CREDENTIAL_ID_RE.findall(text)
@@ -2603,9 +2584,6 @@ def _classification_from_raw(
credential_input_kind=credential_input_kind if credential_input_kind in _KINDS else "none",
credential_refs=_clean_list(raw.get("credential_refs") or []),
login_page_urls=_clean_list(raw.get("login_page_urls") or []),
- login_intent=bool(raw.get("login_intent")),
- email_signin_intent=bool(raw.get("email_signin_intent")),
- signin_email_candidates=_clean_email_list(raw.get("signin_email_candidates") or []),
requires_user_clarification=bool(raw.get("requires_user_clarification")),
completion_contract=completion_contract or None,
completion_criteria=completion_criteria,
@@ -4008,10 +3986,6 @@ def credential_candidate_label(credential: Credential) -> str:
return f"{credential.name} (`{credential.credential_id}`)"
-def _is_passwordless_email_signin(policy: RequestPolicy) -> bool:
- return policy.login_intent and policy.email_signin_intent
-
-
def _ground_user_provided_sites(
policy: RequestPolicy,
user_message: str,
@@ -4149,21 +4123,6 @@ async def _resolve_live_page_credentials(
"""Ask the live page which saved credentials it vouches for, or None when a precondition declines."""
if not live_page_credentials_admissible(policy) or not page_url:
return None
- if _is_passwordless_email_signin(policy):
- # Turn-start resolution excludes this shape from every URL tier because a site that signs in
- # by emailed link has no password question to put to the user, so admitting one here would
- # fill a password the user said the account does not have.
- if _live_page_log_allowed(policy, page_url, seam, "passwordless_declined"):
- LOG.info(
- "copilot credential live-page admission",
- outcome="passwordless_declined",
- seam=seam,
- organization_id=organization_id,
- credential_id=credential_id,
- page_url=loggable_origin(page_url),
- )
- return None
-
return resolve_by_url(
await (load_org_credentials() if load_org_credentials else _load_credentials(organization_id)),
[page_url],
diff --git a/skyvern/forge/sdk/copilot/signin_email.py b/skyvern/forge/sdk/copilot/signin_email.py
deleted file mode 100644
index 2ddfc0481..000000000
--- a/skyvern/forge/sdk/copilot/signin_email.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""The org's connected Gmail address, as a default sign-in identity."""
-
-from __future__ import annotations
-
-import asyncio
-import re
-
-import httpx
-import structlog
-
-from skyvern.forge.sdk.services import google_oauth_service
-from skyvern.services.email.gmail_client import GMAIL_API_BASE, get_json
-
-LOG = structlog.get_logger()
-
-_GMAIL_PROFILE_TIMEOUT_SECONDS = 10.0
-_EMAIL_ADDRESS_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")
-_MAX_EMAIL_ADDRESS_LENGTH = 254
-# This runs inside the per-turn policy build, so the whole lookup is bounded rather than
-# left to the retrying HTTP client multiplied by however many connections an org has.
-_LOOKUP_DEADLINE_SECONDS = 15.0
-_MAX_CREDENTIALS_TRIED = 3
-
-
-def is_email_address(value: str) -> bool:
- """Shape check for a value that reaches an LLM prompt and a workflow parameter default."""
- return len(value) <= _MAX_EMAIL_ADDRESS_LENGTH and bool(_EMAIL_ADDRESS_RE.fullmatch(value))
-
-
-async def connected_gmail_address(organization_id: str) -> str | None:
- try:
- async with asyncio.timeout(_LOOKUP_DEADLINE_SECONDS):
- return await _connected_gmail_address(organization_id)
- except Exception:
- LOG.warning("copilot signin email: lookup did not resolve an address", exc_info=True)
- return None
-
-
-async def _connected_gmail_address(organization_id: str) -> str | None:
- gmail_scopes = google_oauth_service.scopes_for_profile("gmail")
- try:
- # Active-only: the visible listing also returns ERROR-state credentials, which
- # load_credential_secrets cannot load, so newer broken ones would eat the cap.
- credentials = await google_oauth_service.get_credentials_for_org(organization_id)
- except Exception:
- LOG.warning("copilot signin email: google credential listing failed", exc_info=True)
- return None
-
- # Cap the profile lookups, not the listing: capping first would let a few Sheets-only
- # connections hide a Gmail one further down the list.
- gmail_credentials = [
- credential
- for credential in credentials
- if google_oauth_service.has_required_scopes(credential.scopes_granted, gmail_scopes)
- ]
- for credential in gmail_credentials[:_MAX_CREDENTIALS_TRIED]:
- try:
- secrets = await google_oauth_service.load_credential_secrets(organization_id, credential.id)
- access_token = await google_oauth_service.access_token_from_secrets(secrets, organization_id)
- async with httpx.AsyncClient(timeout=_GMAIL_PROFILE_TIMEOUT_SECONDS) as client:
- profile = await get_json(
- client,
- f"{GMAIL_API_BASE}/users/me/profile",
- access_token=access_token,
- )
- except Exception:
- LOG.warning(
- "copilot signin email: gmail profile lookup failed",
- credential_id=credential.id,
- exc_info=True,
- )
- continue
- address = profile.get("emailAddress")
- if isinstance(address, str) and is_email_address(address.strip()):
- return address.strip()
- return None
diff --git a/skyvern/forge/sdk/copilot/todo_list.py b/skyvern/forge/sdk/copilot/todo_list.py
index 5c005ee9b..e5fe36260 100644
--- a/skyvern/forge/sdk/copilot/todo_list.py
+++ b/skyvern/forge/sdk/copilot/todo_list.py
@@ -4,7 +4,6 @@ from __future__ import annotations
from collections.abc import Sequence
from collections.abc import Set as AbstractSet
-from typing import Any
import structlog
@@ -20,28 +19,8 @@ def _page_key(url: object) -> str | None:
return url.strip().split("#", 1)[0].rstrip("/")
-def _credential_fill_page_keys(ctx: AgentContext) -> set[str] | None:
- """Pages where a credential fill happened, or None when no fill has happened at all."""
- keys: set[str] = set()
- fills = 0
- entries: list[dict[str, Any]] = [dict(interaction) for interaction in ctx.scout_trajectory]
- entries.extend(entry for entry in ctx.prior_carried_trajectory if isinstance(entry, dict))
- for entry in entries:
- if entry.get("tool_name") != "fill_credential_field":
- continue
- fills += 1
- key = _page_key(entry.get("source_url"))
- if key:
- keys.add(key)
- return keys if fills else None
-
-
def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:
- """Pages reached by actually interacting, not just looking — the only evidence that counts as login progress.
-
- Heuristic ceiling: any interaction-reached page off the fill page counts, so a non-submit
- navigation (e.g. a forgot-password link) can suppress the login line without a real login.
- """
+ """Pages reached by actually interacting, not just looking."""
keys: set[str] = set()
for page in ctx.prior_observed_acted_pages:
if isinstance(page, dict) and page.get("reached_via") == "interaction":
@@ -61,18 +40,6 @@ def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:
return keys
-def _login_line(ctx: AgentContext) -> str | None:
- policy = ctx.request_policy
- if not isinstance(policy, RequestPolicy) or not policy.login_intent or not policy.resolved_credentials:
- return None
- fill_pages = _credential_fill_page_keys(ctx)
- if fill_pages is None:
- return "Login: credential resolved but login not yet attempted"
- if _interaction_reached_page_keys(ctx) - fill_pages:
- return None
- return "Login: credential resolved but login not completed (no page reached by interaction yet)"
-
-
def _minted_criteria(ctx: AgentContext) -> list[CompletionCriterion]:
turn_state = ctx.completion_criteria_turn_state
if turn_state is not None and turn_state.decision is not None:
@@ -155,9 +122,10 @@ def unmet_action_deliverable_criteria(ctx: AgentContext) -> list[CompletionCrite
def render_todo_list(ctx: AgentContext) -> str | None:
- lines = [line for line in (_login_line(ctx), _outputs_line(ctx)) if line]
- if not lines:
+ outputs = _outputs_line(ctx)
+ if not outputs:
return None
+ lines = [outputs]
interactions = _interactions_line(ctx)
if interactions:
lines.append(interactions)
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index db88ec948..9c469251a 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -1523,7 +1523,6 @@ def test_credential_pause_resolution_defaults_credential_to_none() -> None:
def _blocked_login_policy() -> RequestPolicy:
"""A policy shaped the way build_request_policy leaves an unresolved login ask."""
return RequestPolicy(
- login_intent=True,
clarification_reason="login_credentials_unresolved",
clarification_question="Connect a saved credential to sign in.",
requires_user_clarification=True,
@@ -1552,7 +1551,7 @@ def test_reason_fires_on_unresolved_login_credentials() -> None:
def test_reason_does_not_fire_for_a_login_ask_that_resolved_to_a_credential() -> None:
ctx = make_copilot_context()
- ctx.request_policy = RequestPolicy(login_intent=True, clarification_reason="none")
+ ctx.request_policy = RequestPolicy(clarification_reason="none")
assert credential_pause_module.credential_pause_reason(ctx) is None
@@ -1566,7 +1565,6 @@ def _card_answerable_policy(
) -> RequestPolicy:
"""A disambiguation ask the credential card can answer: a different reason to the login ask."""
return RequestPolicy(
- login_intent=True,
clarification_reason=clarification_reason,
clarification_question=clarification_question,
requires_user_clarification=True,
diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.py
index 39ffdb48a..ca2b0ca8c 100644
--- a/tests/unit/test_copilot_fill_credential_field.py
+++ b/tests/unit/test_copilot_fill_credential_field.py
@@ -1129,20 +1129,6 @@ class TestCredentialFillLivePageAdmission:
assert password["ok"] is True
assert maximum_active_resolutions == 1
- @pytest.mark.asyncio
- async def test_a_refusable_call_never_reads_the_org_credentials(self) -> None:
- """The refusals that need no lookup must not page the credential table in first."""
- policy = RequestPolicy(login_intent=True, email_signin_intent=True, credential_input_kind="none")
- error, _, load_mock = await self._gate(
- credential_id="cred_analytics",
- page_url="https://analytics.example.com/login",
- org_credentials=[_org_credential("cred_analytics", "analytics", "https://analytics.example.com/login")],
- policy=policy,
- )
-
- assert error is not None
- load_mock.assert_not_awaited()
-
@pytest.mark.asyncio
async def test_a_second_step_on_the_same_site_still_fills(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""Real sign-ins walk email -> password -> one-time code across paths of the same site;
diff --git a/tests/unit/test_copilot_request_policy_signin_intent_absent.py b/tests/unit/test_copilot_request_policy_signin_intent_absent.py
new file mode 100644
index 000000000..0881d7d5a
--- /dev/null
+++ b/tests/unit/test_copilot_request_policy_signin_intent_absent.py
@@ -0,0 +1,53 @@
+"""The trust floor must not report a sign-in fact the production build path cannot supply."""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot.context import StructuredContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, build_request_policy_trust_floor
+from tests.unit.copilot_test_helpers import make_copilot_ctx
+
+SIGNIN_TRACE_KEYS = {"login_intent", "email_signin_intent", "signin_email_resolved"}
+
+
+async def _clean_safety_handler(**_: object) -> dict[str, object]:
+ return {"state": "clean", "citations": []}
+
+
+async def test_trust_floor_trace_data_carries_no_signin_intent_key() -> None:
+ policy = await build_request_policy_trust_floor(
+ user_message="Sign in to the reporting portal and download last week's report.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-replay",
+ handler=_clean_safety_handler,
+ )
+
+ assert SIGNIN_TRACE_KEYS.isdisjoint(policy.to_trace_data())
+
+
+def test_dynamic_system_prompt_never_mentions_a_sign_in_address() -> None:
+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())
+
+ ctx = make_copilot_ctx(request_policy=RequestPolicy())
+
+ prompt = instructions(SimpleNamespace(context=ctx), None)
+
+ assert "signin" not in prompt.lower()
+
+
+def test_stored_context_carrying_a_signin_email_still_loads() -> None:
+ stored = json.dumps(
+ {"user_goal": "sign in", "signin_email": "[email protected]", "signin_email_host": "example.com"}
+ )
+
+ structured = StructuredContext.from_json_str(stored)
+
+ assert structured.user_goal == "sign in"
diff --git a/tests/unit/test_copilot_signin_email.py b/tests/unit/test_copilot_signin_email.py
deleted file mode 100644
index 409b9cd16..000000000
--- a/tests/unit/test_copilot_signin_email.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from types import SimpleNamespace
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-from skyvern.forge.sdk.copilot.signin_email import _MAX_CREDENTIALS_TRIED, connected_gmail_address
-
-GMAIL_SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]
-
-
-def _credential(credential_id: str, scopes: list[str] | None = None) -> SimpleNamespace:
- return SimpleNamespace(id=credential_id, scopes_granted=GMAIL_SCOPES if scopes is None else scopes)
-
-
-def _patches(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> list:
- return [
- patch(
- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.get_credentials_for_org",
- new=AsyncMock(return_value=credentials),
- ),
- patch(
- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.scopes_for_profile",
- return_value=GMAIL_SCOPES,
- ),
- patch(
- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.load_credential_secrets",
- new=AsyncMock(return_value=SimpleNamespace(refresh_token="rt")),
- ),
- patch(
- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.access_token_from_secrets",
- new=AsyncMock(return_value="token"),
- ),
- patch(
- "skyvern.forge.sdk.copilot.signin_email.get_json",
- new=profile or AsyncMock(return_value={"emailAddress": "[email protected]"}),
- ),
- ]
-
-
-async def _resolve(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> str | None:
- patches = _patches(credentials, profile)
- for p in patches:
- p.start()
- try:
- return await connected_gmail_address("o_test")
- finally:
- for p in patches:
- p.stop()
-
-
-async def test_returns_the_address_of_a_gmail_scoped_connection() -> None:
- assert await _resolve([_credential("gc_1")]) == "[email protected]"
-
-
-async def test_skips_a_connection_without_gmail_scope() -> None:
- assert await _resolve([_credential("gc_1", scopes=["https://www.googleapis.com/auth/spreadsheets"])]) is None
-
-
-async def test_a_failing_profile_lookup_resolves_to_no_address_rather_than_raising() -> None:
- # The caller treats None as "no connected account" and falls back to the request;
- # an exception escaping here would abort the whole policy build.
- assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=RuntimeError("boom"))) is None
-
-
-async def test_a_hanging_lookup_gives_up_instead_of_stalling_the_turn() -> None:
- async def _never_returns(*args: object, **kwargs: object) -> dict[str, str]:
- await asyncio.sleep(3600)
- return {}
-
- with patch("skyvern.forge.sdk.copilot.signin_email._LOOKUP_DEADLINE_SECONDS", 0.05):
- assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=_never_returns)) is None
-
-
-async def test_only_a_bounded_number_of_connections_are_tried() -> None:
- profile = AsyncMock(return_value={"emailAddress": ""})
- await _resolve([_credential(f"gc_{n}") for n in range(_MAX_CREDENTIALS_TRIED + 4)], profile=profile)
-
- assert profile.await_count == _MAX_CREDENTIALS_TRIED
-
-
-async def test_a_gmail_connection_is_found_behind_connections_without_gmail_scope() -> None:
- sheets = ["https://www.googleapis.com/auth/spreadsheets"]
- credentials = [_credential(f"gc_sheets_{n}", scopes=sheets) for n in range(_MAX_CREDENTIALS_TRIED + 1)]
- credentials.append(_credential("gc_gmail"))
-
- assert await _resolve(credentials) == "[email protected]"
diff --git a/tests/unit/test_copilot_todo_list.py b/tests/unit/test_copilot_todo_list.py
index 35cb08323..7ab1f42b6 100644
--- a/tests/unit/test_copilot_todo_list.py
+++ b/tests/unit/test_copilot_todo_list.py
@@ -1,4 +1,3 @@
-from datetime import datetime, timezone
from types import SimpleNamespace
import pytest
@@ -7,72 +6,13 @@ from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicy
from skyvern.forge.sdk.copilot.todo_list import render_todo_list, todo_list_prompt
-from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType
from tests.unit.copilot_test_helpers import make_copilot_ctx
-def _login_policy() -> RequestPolicy:
- now = datetime(2026, 1, 1, tzinfo=timezone.utc)
- credential = Credential(
- credential_id="cred-1",
- organization_id="org-1",
- name="Example Login",
- vault_type=CredentialVaultType.SKYVERN,
- item_id="item_1",
- credential_type=CredentialType.PASSWORD,
- username="[email protected]",
- card_last4=None,
- card_brand=None,
- created_at=now,
- modified_at=now,
- )
- return RequestPolicy(login_intent=True, resolved_credentials=[credential])
-
-
def _output_criterion(path: str) -> CompletionCriterion:
return CompletionCriterion(id=f"crit-{path}", outcome=f"observe {path}", output_path=path)
-def test_login_not_attempted() -> None:
- ctx = make_copilot_ctx(request_policy=_login_policy())
- todo = render_todo_list(ctx)
- assert todo is not None
- assert "- Login: credential resolved but login not yet attempted" in todo
- assert "- The site has not been acted on yet (0 interactions recorded)" in todo
-
-
-def test_login_filled_but_no_page_reached_by_interaction() -> None:
- ctx = make_copilot_ctx(request_policy=_login_policy())
- ctx.scout_trajectory = [
- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
- ]
- assert (
- render_todo_list(ctx)
- == "- Login: credential resolved but login not completed (no page reached by interaction yet)"
- )
-
-
-def test_read_only_page_observations_do_not_complete_login() -> None:
- ctx = make_copilot_ctx(request_policy=_login_policy())
- ctx.scout_trajectory = [
- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
- ]
- ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "inspection"}]
- ctx.observed_browser_urls = ["https://example.com/dashboard"]
- todo = render_todo_list(ctx)
- assert todo is not None
- assert "login not completed" in todo
-
-
-def test_login_completed_via_interaction_reached_page() -> None:
- ctx = make_copilot_ctx(request_policy=_login_policy())
- ctx.prior_carried_trajectory = [
- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}
- ]
- ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "interaction"}]
- assert render_todo_list(ctx) is None
-
-
def test_outputs_not_yet_observed() -> None:
policy = RequestPolicy(completion_criteria=[_output_criterion("output.number_of_visitors_last_week")])
ctx = make_copilot_ctx(request_policy=policy)
@@ -80,16 +20,6 @@ def test_outputs_not_yet_observed() -> None:
assert render_todo_list(ctx) == "- Outputs not yet observed: output.number_of_visitors_last_week"
-def test_credential_id_on_non_fill_tool_is_not_a_login_attempt() -> None:
- ctx = make_copilot_ctx(request_policy=_login_policy())
- ctx.scout_trajectory = [
- {"tool_name": "click", "source_url": "https://example.com/login", "credential_id": "cred-1"}
- ]
- todo = render_todo_list(ctx)
- assert todo is not None
- assert "login not yet attempted" in todo
-
-
def test_interaction_reached_page_suppresses_zero_interactions_line() -> None:
policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
ctx = make_copilot_ctx(request_policy=policy)
@@ -160,22 +90,13 @@ def test_all_done_returns_none() -> None:
assert render_todo_list(make_copilot_ctx()) is None
-def test_login_and_outputs_combined() -> None:
- policy = _login_policy()
- policy.completion_criteria = [_output_criterion("output.total")]
- todo = render_todo_list(make_copilot_ctx(request_policy=policy))
- assert todo is not None
- assert "- Login: credential resolved but login not yet attempted" in todo
- assert "- Outputs not yet observed: output.total" in todo
-
-
def test_dynamic_system_prompt_injects_todo_list(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")
instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())
- ctx = make_copilot_ctx(request_policy=_login_policy())
- prompt = instructions(SimpleNamespace(context=ctx), None)
+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])
+ prompt = instructions(SimpleNamespace(context=make_copilot_ctx(request_policy=policy)), None)
assert "TODO — outstanding before you reply:" in prompt
- assert "Login: credential resolved but login not yet attempted" in prompt
+ assert "Outputs not yet observed: output.total" in prompt
def test_dynamic_system_prompt_adds_nothing_when_nothing_pending(monkeypatch: pytest.MonkeyPatch) -> None: