Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Remove three Copilot sign-in fields that were always false (SKY-14054) (#8211)
Andrew Neilson0cbfce0b88fb
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 1507fc27f..c98996633 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -82,7 +82,6 @@ from skyvern.forge.sdk.copilot.context import (finalize_observation_context,parsed_ask_refs,record_approved_credentials_in_global_llm_context,- record_signin_email_in_global_llm_context,sanitize_global_llm_context_for_prompt,)from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason, preflight_credential_pause@@ -1014,10 +1013,7 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -+ "\n```\nThis block contains safety and request facts, not permission or a mandatory next action. "+ "If `raw_secret_handling` is `redacted_draft`, build only from the redacted request, do not run "+ "blocks, and tell the user to store the redacted secret as a saved credential before testing. "- + "If `resolved_credentials` are present, use those `credential_id` values. "- + "If `resolved_signin_email` is present, that address signs in: bind it to the workflow as a "- + "workflow input parameter with that value as the default, so the saved workflow can be re-run "- + "under a different address, and do not ask for a saved password credential for that login."+ + "If `resolved_credentials` are present, use those `credential_id` values."+ _runtime_verification_evidence_prompt(ctx)+ _recorded_build_test_outcome_prompt(ctx)+ _code_authoring_repair_context_prompt(ctx)@@ -1402,12 +1398,7 @@ def _make_agent_result(normal translate-result, missing-SDK fallback, unexpected-error fallback)."""final_context = (- record_signin_email_in_global_llm_context(- ctx,- record_approved_credentials_in_global_llm_context(- ctx, finalize_observation_context(ctx, global_llm_context)- ),- )+ record_approved_credentials_in_global_llm_context(ctx, finalize_observation_context(ctx, global_llm_context))if ctx is not Noneelse global_llm_context)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 4d2353ade..76c6fe5f2 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport jsonimport reimport uuid-from collections.abc import Collection, Mapping, Sequence+from collections.abc import Mapping, Sequencefrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Any, Literal, get_args@@ -251,12 +251,6 @@ class StructuredContext(BaseModel):]entrypoint_url: str | None = None- # Passwordless sign-in identity, carried so a follow-up turn the classifier reads as an- # ordinary login does not fall back to demanding a password the site does not have.- # Scoped by host: reusing it anywhere else would suppress the password ask on a site- # that never established it was passwordless.- signin_email: str = ""- signin_email_host: str = ""def to_json_str(self) -> str:payload = self.model_dump(mode="json")@@ -511,35 +505,13 @@ def record_approved_credentials_in_global_llm_context(ctx: CopilotContext, raw_creturn sc.to_json_str()-def record_signin_email_in_global_llm_context(ctx: CopilotContext, raw_context: str | None) -> str | None:- policy = ctx.request_policy- if policy is None or not policy.resolved_signin_email or not policy.resolved_signin_host:- return raw_context- sc = StructuredContext.from_json_str(raw_context)- if sc.signin_email == policy.resolved_signin_email and sc.signin_email_host == policy.resolved_signin_host:- return raw_context- sc.signin_email = policy.resolved_signin_email- sc.signin_email_host = policy.resolved_signin_host- return sc.to_json_str()---def prior_signin_email_from_context(raw_context: str | None, hosts: Collection[str]) -> str:- """The carried address, only when this turn targets the host it was resolved for."""- sc = StructuredContext.from_json_str(raw_context)- if not sc.signin_email or sc.signin_email_host not in hosts:- return ""- return sc.signin_email--def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) -> StructuredContext:"""Take the model's context but keep the server-owned fields server-owned.Approval is recorded only from server-resolved credentials; an entry the modelsupplied would be promoted into `resolved_credentials` on the next turn and clearthe unapproved-credential gate for a credential the user never named. Membership- of the org is not evidence the user named it. A model-authored `signin_email` is the- same promotion in the other gate: it marks the next turn passwordless and suppresses- the password-credential ask for a site that does need one.+ of the org is not evidence the user named it.`carried_trajectory` is the record of what the browser was observed doing, so anentry the model wrote would enter the factual record as an observation nothing made.@@ -557,8 +529,6 @@ def adopt_model_authored_context(trusted_raw: str | None, model_raw: object) ->elif isinstance(model_raw, str):structured = StructuredContext.from_json_str(model_raw)structured.approved_credentials = list(trusted.approved_credentials)- structured.signin_email = trusted.signin_email- structured.signin_email_host = trusted.signin_email_hoststructured.carried_trajectory = [dict(entry) for entry in trusted.carried_trajectory]return structureddiff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex c33465b87..1af17c240 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -50,7 +50,6 @@ from skyvern.forge.sdk.copilot.secret_redaction import (contains_email_password_pair,redact_raw_secrets_for_prompt,)-from skyvern.forge.sdk.copilot.signin_email import is_email_addressfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.workflow_credential_utils import (URL_CANDIDATE_RE,@@ -86,7 +85,6 @@ _CLASSIFICATION_RESPONSE_FIELDS = {"credential_input_kind","credential_refs","login_page_urls",- "login_intent","requires_user_clarification","completion_contract","completion_criteria",@@ -111,7 +109,6 @@ ClarificationReason = Literal["missing_target_context","workflow_credential_inputs_unbound","login_credentials_unresolved",- "signin_email_unresolved","safety_screen_unavailable",]RawSecretHandling = Literal["none", "block", "redacted_draft"]@@ -347,7 +344,7 @@ _VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_arg# Only deterministic post-resolution code may mint these; a classifier emission would# skip the concrete-target and credential-reachability checks the reason stands for._DETERMINISTIC_ONLY_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(- {"login_credentials_unresolved", "signin_email_unresolved", "safety_screen_unavailable"}+ {"login_credentials_unresolved", "safety_screen_unavailable"})# Concrete credential states for which the combined update-and-run tool persists# the draft but cannot start a browser run.@@ -788,13 +785,6 @@ class RequestPolicy:credential_input_kind: str = "none"credential_refs: list[str] = field(default_factory=list)login_page_urls: list[str] = field(default_factory=list)- login_intent: bool = False- # Sign-in that identifies the user by email address instead of a stored password, so- # resolving *which address* is the whole credential question — there is no password to find.- email_signin_intent: bool = False- signin_email_candidates: list[str] = field(default_factory=list)- resolved_signin_email: str | None = None- resolved_signin_host: str | None = Nonerequires_user_clarification: bool = Falseallow_update_workflow: bool = Trueallow_run_blocks: bool = True@@ -869,9 +859,6 @@ class RequestPolicy:"testing_intent": self.testing_intent,"authoring_intent": self.authoring_intent,"credential_input_kind": self.credential_input_kind,- "login_intent": self.login_intent,- "email_signin_intent": self.email_signin_intent,- "signin_email_resolved": bool(self.resolved_signin_email),"clarification_reason": self.clarification_reason,"allow_update_workflow": self.allow_update_workflow,"allow_run_blocks": self.allow_run_blocks,@@ -979,8 +966,6 @@ class RequestPolicy:]if self.raw_secret_detected:lines.append(f"raw_secret_detected: {self.raw_secret_detected}")- if self.resolved_signin_email:- lines.append(f"resolved_signin_email: {self.resolved_signin_email}")validation_classification_criteria = [criterionfor criterion in self.graded_completion_criteria()@@ -1260,10 +1245,6 @@ def _clean_list(values: list[Any]) -> list[str]:return list(dict.fromkeys(str(value).strip() for value in values if str(value).strip()))-def _clean_email_list(values: list[Any]) -> list[str]:- return [value for value in _clean_list(values) if is_email_address(value)]--def _credential_ids(text: str) -> list[str]:text = text or ""canonical = _CREDENTIAL_ID_RE.findall(text)@@ -2603,9 +2584,6 @@ def _classification_from_raw(credential_input_kind=credential_input_kind if credential_input_kind in _KINDS else "none",credential_refs=_clean_list(raw.get("credential_refs") or []),login_page_urls=_clean_list(raw.get("login_page_urls") or []),- login_intent=bool(raw.get("login_intent")),- email_signin_intent=bool(raw.get("email_signin_intent")),- signin_email_candidates=_clean_email_list(raw.get("signin_email_candidates") or []),requires_user_clarification=bool(raw.get("requires_user_clarification")),completion_contract=completion_contract or None,completion_criteria=completion_criteria,@@ -4008,10 +3986,6 @@ def credential_candidate_label(credential: Credential) -> str:return f"{credential.name} (`{credential.credential_id}`)"-def _is_passwordless_email_signin(policy: RequestPolicy) -> bool:- return policy.login_intent and policy.email_signin_intent--def _ground_user_provided_sites(policy: RequestPolicy,user_message: str,@@ -4149,21 +4123,6 @@ async def _resolve_live_page_credentials("""Ask the live page which saved credentials it vouches for, or None when a precondition declines."""if not live_page_credentials_admissible(policy) or not page_url:return None- if _is_passwordless_email_signin(policy):- # Turn-start resolution excludes this shape from every URL tier because a site that signs in- # by emailed link has no password question to put to the user, so admitting one here would- # fill a password the user said the account does not have.- if _live_page_log_allowed(policy, page_url, seam, "passwordless_declined"):- LOG.info(- "copilot credential live-page admission",- outcome="passwordless_declined",- seam=seam,- organization_id=organization_id,- credential_id=credential_id,- page_url=loggable_origin(page_url),- )- return None-return resolve_by_url(await (load_org_credentials() if load_org_credentials else _load_credentials(organization_id)),[page_url],diff --git a/skyvern/forge/sdk/copilot/signin_email.py b/skyvern/forge/sdk/copilot/signin_email.pydeleted file mode 100644index 2ddfc0481..000000000--- a/skyvern/forge/sdk/copilot/signin_email.py+++ /dev/null@@ -1,76 +0,0 @@-"""The org's connected Gmail address, as a default sign-in identity."""--from __future__ import annotations--import asyncio-import re--import httpx-import structlog--from skyvern.forge.sdk.services import google_oauth_service-from skyvern.services.email.gmail_client import GMAIL_API_BASE, get_json--LOG = structlog.get_logger()--_GMAIL_PROFILE_TIMEOUT_SECONDS = 10.0-_EMAIL_ADDRESS_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")-_MAX_EMAIL_ADDRESS_LENGTH = 254-# This runs inside the per-turn policy build, so the whole lookup is bounded rather than-# left to the retrying HTTP client multiplied by however many connections an org has.-_LOOKUP_DEADLINE_SECONDS = 15.0-_MAX_CREDENTIALS_TRIED = 3---def is_email_address(value: str) -> bool:- """Shape check for a value that reaches an LLM prompt and a workflow parameter default."""- return len(value) <= _MAX_EMAIL_ADDRESS_LENGTH and bool(_EMAIL_ADDRESS_RE.fullmatch(value))---async def connected_gmail_address(organization_id: str) -> str | None:- try:- async with asyncio.timeout(_LOOKUP_DEADLINE_SECONDS):- return await _connected_gmail_address(organization_id)- except Exception:- LOG.warning("copilot signin email: lookup did not resolve an address", exc_info=True)- return None---async def _connected_gmail_address(organization_id: str) -> str | None:- gmail_scopes = google_oauth_service.scopes_for_profile("gmail")- try:- # Active-only: the visible listing also returns ERROR-state credentials, which- # load_credential_secrets cannot load, so newer broken ones would eat the cap.- credentials = await google_oauth_service.get_credentials_for_org(organization_id)- except Exception:- LOG.warning("copilot signin email: google credential listing failed", exc_info=True)- return None-- # Cap the profile lookups, not the listing: capping first would let a few Sheets-only- # connections hide a Gmail one further down the list.- gmail_credentials = [- credential- for credential in credentials- if google_oauth_service.has_required_scopes(credential.scopes_granted, gmail_scopes)- ]- for credential in gmail_credentials[:_MAX_CREDENTIALS_TRIED]:- try:- secrets = await google_oauth_service.load_credential_secrets(organization_id, credential.id)- access_token = await google_oauth_service.access_token_from_secrets(secrets, organization_id)- async with httpx.AsyncClient(timeout=_GMAIL_PROFILE_TIMEOUT_SECONDS) as client:- profile = await get_json(- client,- f"{GMAIL_API_BASE}/users/me/profile",- access_token=access_token,- )- except Exception:- LOG.warning(- "copilot signin email: gmail profile lookup failed",- credential_id=credential.id,- exc_info=True,- )- continue- address = profile.get("emailAddress")- if isinstance(address, str) and is_email_address(address.strip()):- return address.strip()- return Nonediff --git a/skyvern/forge/sdk/copilot/todo_list.py b/skyvern/forge/sdk/copilot/todo_list.pyindex 5c005ee9b..e5fe36260 100644--- a/skyvern/forge/sdk/copilot/todo_list.py+++ b/skyvern/forge/sdk/copilot/todo_list.py@@ -4,7 +4,6 @@ from __future__ import annotationsfrom collections.abc import Sequencefrom collections.abc import Set as AbstractSet-from typing import Anyimport structlog@@ -20,28 +19,8 @@ def _page_key(url: object) -> str | None:return url.strip().split("#", 1)[0].rstrip("/")-def _credential_fill_page_keys(ctx: AgentContext) -> set[str] | None:- """Pages where a credential fill happened, or None when no fill has happened at all."""- keys: set[str] = set()- fills = 0- entries: list[dict[str, Any]] = [dict(interaction) for interaction in ctx.scout_trajectory]- entries.extend(entry for entry in ctx.prior_carried_trajectory if isinstance(entry, dict))- for entry in entries:- if entry.get("tool_name") != "fill_credential_field":- continue- fills += 1- key = _page_key(entry.get("source_url"))- if key:- keys.add(key)- return keys if fills else None--def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:- """Pages reached by actually interacting, not just looking — the only evidence that counts as login progress.-- Heuristic ceiling: any interaction-reached page off the fill page counts, so a non-submit- navigation (e.g. a forgot-password link) can suppress the login line without a real login.- """+ """Pages reached by actually interacting, not just looking."""keys: set[str] = set()for page in ctx.prior_observed_acted_pages:if isinstance(page, dict) and page.get("reached_via") == "interaction":@@ -61,18 +40,6 @@ def _interaction_reached_page_keys(ctx: AgentContext) -> set[str]:return keys-def _login_line(ctx: AgentContext) -> str | None:- policy = ctx.request_policy- if not isinstance(policy, RequestPolicy) or not policy.login_intent or not policy.resolved_credentials:- return None- fill_pages = _credential_fill_page_keys(ctx)- if fill_pages is None:- return "Login: credential resolved but login not yet attempted"- if _interaction_reached_page_keys(ctx) - fill_pages:- return None- return "Login: credential resolved but login not completed (no page reached by interaction yet)"--def _minted_criteria(ctx: AgentContext) -> list[CompletionCriterion]:turn_state = ctx.completion_criteria_turn_stateif turn_state is not None and turn_state.decision is not None:@@ -155,9 +122,10 @@ def unmet_action_deliverable_criteria(ctx: AgentContext) -> list[CompletionCritedef render_todo_list(ctx: AgentContext) -> str | None:- lines = [line for line in (_login_line(ctx), _outputs_line(ctx)) if line]- if not lines:+ outputs = _outputs_line(ctx)+ if not outputs:return None+ lines = [outputs]interactions = _interactions_line(ctx)if interactions:lines.append(interactions)diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex db88ec948..9c469251a 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -1523,7 +1523,6 @@ def test_credential_pause_resolution_defaults_credential_to_none() -> None:def _blocked_login_policy() -> RequestPolicy:"""A policy shaped the way build_request_policy leaves an unresolved login ask."""return RequestPolicy(- login_intent=True,clarification_reason="login_credentials_unresolved",clarification_question="Connect a saved credential to sign in.",requires_user_clarification=True,@@ -1552,7 +1551,7 @@ def test_reason_fires_on_unresolved_login_credentials() -> None:def test_reason_does_not_fire_for_a_login_ask_that_resolved_to_a_credential() -> None:ctx = make_copilot_context()- ctx.request_policy = RequestPolicy(login_intent=True, clarification_reason="none")+ ctx.request_policy = RequestPolicy(clarification_reason="none")assert credential_pause_module.credential_pause_reason(ctx) is None@@ -1566,7 +1565,6 @@ def _card_answerable_policy() -> RequestPolicy:"""A disambiguation ask the credential card can answer: a different reason to the login ask."""return RequestPolicy(- login_intent=True,clarification_reason=clarification_reason,clarification_question=clarification_question,requires_user_clarification=True,diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.pyindex 39ffdb48a..ca2b0ca8c 100644--- a/tests/unit/test_copilot_fill_credential_field.py+++ b/tests/unit/test_copilot_fill_credential_field.py@@ -1129,20 +1129,6 @@ class TestCredentialFillLivePageAdmission:assert password["ok"] is Trueassert maximum_active_resolutions == 1- @pytest.mark.asyncio- async def test_a_refusable_call_never_reads_the_org_credentials(self) -> None:- """The refusals that need no lookup must not page the credential table in first."""- policy = RequestPolicy(login_intent=True, email_signin_intent=True, credential_input_kind="none")- error, _, load_mock = await self._gate(- credential_id="cred_analytics",- page_url="https://analytics.example.com/login",- org_credentials=[_org_credential("cred_analytics", "analytics", "https://analytics.example.com/login")],- policy=policy,- )-- assert error is not None- load_mock.assert_not_awaited()-@pytest.mark.asyncioasync def test_a_second_step_on_the_same_site_still_fills(self, monkeypatch: pytest.MonkeyPatch) -> None:"""Real sign-ins walk email -> password -> one-time code across paths of the same site;diff --git a/tests/unit/test_copilot_request_policy_signin_intent_absent.py b/tests/unit/test_copilot_request_policy_signin_intent_absent.pynew file mode 100644index 000000000..0881d7d5a--- /dev/null+++ b/tests/unit/test_copilot_request_policy_signin_intent_absent.py@@ -0,0 +1,53 @@+"""The trust floor must not report a sign-in fact the production build path cannot supply."""++from __future__ import annotations++import json+from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot.context import StructuredContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy, build_request_policy_trust_floor+from tests.unit.copilot_test_helpers import make_copilot_ctx++SIGNIN_TRACE_KEYS = {"login_intent", "email_signin_intent", "signin_email_resolved"}+++async def _clean_safety_handler(**_: object) -> dict[str, object]:+ return {"state": "clean", "citations": []}+++async def test_trust_floor_trace_data_carries_no_signin_intent_key() -> None:+ policy = await build_request_policy_trust_floor(+ user_message="Sign in to the reporting portal and download last week's report.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-replay",+ handler=_clean_safety_handler,+ )++ assert SIGNIN_TRACE_KEYS.isdisjoint(policy.to_trace_data())+++def test_dynamic_system_prompt_never_mentions_a_sign_in_address() -> None:+ instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())++ ctx = make_copilot_ctx(request_policy=RequestPolicy())++ prompt = instructions(SimpleNamespace(context=ctx), None)++ assert "signin" not in prompt.lower()+++def test_stored_context_carrying_a_signin_email_still_loads() -> None:+ stored = json.dumps(+ {"user_goal": "sign in", "signin_email": "[email protected]", "signin_email_host": "example.com"}+ )++ structured = StructuredContext.from_json_str(stored)++ assert structured.user_goal == "sign in"diff --git a/tests/unit/test_copilot_signin_email.py b/tests/unit/test_copilot_signin_email.pydeleted file mode 100644index 409b9cd16..000000000--- a/tests/unit/test_copilot_signin_email.py+++ /dev/null@@ -1,95 +0,0 @@-from __future__ import annotations--import asyncio-from types import SimpleNamespace-from unittest.mock import AsyncMock, patch--import pytest--from skyvern.forge.sdk.copilot.signin_email import _MAX_CREDENTIALS_TRIED, connected_gmail_address--GMAIL_SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]---def _credential(credential_id: str, scopes: list[str] | None = None) -> SimpleNamespace:- return SimpleNamespace(id=credential_id, scopes_granted=GMAIL_SCOPES if scopes is None else scopes)---def _patches(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> list:- return [- patch(- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.get_credentials_for_org",- new=AsyncMock(return_value=credentials),- ),- patch(- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.scopes_for_profile",- return_value=GMAIL_SCOPES,- ),- patch(- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.load_credential_secrets",- new=AsyncMock(return_value=SimpleNamespace(refresh_token="rt")),- ),- patch(- "skyvern.forge.sdk.copilot.signin_email.google_oauth_service.access_token_from_secrets",- new=AsyncMock(return_value="token"),- ),- patch(- "skyvern.forge.sdk.copilot.signin_email.get_json",- new=profile or AsyncMock(return_value={"emailAddress": "[email protected]"}),- ),- ]---async def _resolve(credentials: list[SimpleNamespace], profile: AsyncMock | None = None) -> str | None:- patches = _patches(credentials, profile)- for p in patches:- p.start()- try:- return await connected_gmail_address("o_test")- finally:- for p in patches:- p.stop()---async def test_returns_the_address_of_a_gmail_scoped_connection() -> None:- assert await _resolve([_credential("gc_1")]) == "[email protected]"---async def test_skips_a_connection_without_gmail_scope() -> None:- assert await _resolve([_credential("gc_1", scopes=["https://www.googleapis.com/auth/spreadsheets"])]) is None---async def test_a_failing_profile_lookup_resolves_to_no_address_rather_than_raising() -> None:- # The caller treats None as "no connected account" and falls back to the request;- # an exception escaping here would abort the whole policy build.- assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=RuntimeError("boom"))) is None---async def test_a_hanging_lookup_gives_up_instead_of_stalling_the_turn() -> None:- async def _never_returns(*args: object, **kwargs: object) -> dict[str, str]:- await asyncio.sleep(3600)- return {}-- with patch("skyvern.forge.sdk.copilot.signin_email._LOOKUP_DEADLINE_SECONDS", 0.05):- assert await _resolve([_credential("gc_1")], profile=AsyncMock(side_effect=_never_returns)) is None---async def test_only_a_bounded_number_of_connections_are_tried() -> None:- profile = AsyncMock(return_value={"emailAddress": ""})- await _resolve([_credential(f"gc_{n}") for n in range(_MAX_CREDENTIALS_TRIED + 4)], profile=profile)-- assert profile.await_count == _MAX_CREDENTIALS_TRIED---async def test_a_gmail_connection_is_found_behind_connections_without_gmail_scope() -> None:- sheets = ["https://www.googleapis.com/auth/spreadsheets"]- credentials = [_credential(f"gc_sheets_{n}", scopes=sheets) for n in range(_MAX_CREDENTIALS_TRIED + 1)]- credentials.append(_credential("gc_gmail"))-- assert await _resolve(credentials) == "[email protected]"diff --git a/tests/unit/test_copilot_todo_list.py b/tests/unit/test_copilot_todo_list.pyindex 35cb08323..7ab1f42b6 100644--- a/tests/unit/test_copilot_todo_list.py+++ b/tests/unit/test_copilot_todo_list.py@@ -1,4 +1,3 @@-from datetime import datetime, timezonefrom types import SimpleNamespaceimport pytest@@ -7,72 +6,13 @@ from skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, RequestPolicyfrom skyvern.forge.sdk.copilot.todo_list import render_todo_list, todo_list_prompt-from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultTypefrom tests.unit.copilot_test_helpers import make_copilot_ctx-def _login_policy() -> RequestPolicy:- now = datetime(2026, 1, 1, tzinfo=timezone.utc)- credential = Credential(- credential_id="cred-1",- organization_id="org-1",- name="Example Login",- vault_type=CredentialVaultType.SKYVERN,- item_id="item_1",- credential_type=CredentialType.PASSWORD,- username="[email protected]",- card_last4=None,- card_brand=None,- created_at=now,- modified_at=now,- )- return RequestPolicy(login_intent=True, resolved_credentials=[credential])--def _output_criterion(path: str) -> CompletionCriterion:return CompletionCriterion(id=f"crit-{path}", outcome=f"observe {path}", output_path=path)-def test_login_not_attempted() -> None:- ctx = make_copilot_ctx(request_policy=_login_policy())- todo = render_todo_list(ctx)- assert todo is not None- assert "- Login: credential resolved but login not yet attempted" in todo- assert "- The site has not been acted on yet (0 interactions recorded)" in todo---def test_login_filled_but_no_page_reached_by_interaction() -> None:- ctx = make_copilot_ctx(request_policy=_login_policy())- ctx.scout_trajectory = [- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}- ]- assert (- render_todo_list(ctx)- == "- Login: credential resolved but login not completed (no page reached by interaction yet)"- )---def test_read_only_page_observations_do_not_complete_login() -> None:- ctx = make_copilot_ctx(request_policy=_login_policy())- ctx.scout_trajectory = [- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}- ]- ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "inspection"}]- ctx.observed_browser_urls = ["https://example.com/dashboard"]- todo = render_todo_list(ctx)- assert todo is not None- assert "login not completed" in todo---def test_login_completed_via_interaction_reached_page() -> None:- ctx = make_copilot_ctx(request_policy=_login_policy())- ctx.prior_carried_trajectory = [- {"tool_name": "fill_credential_field", "source_url": "https://example.com/login", "credential_id": "cred-1"}- ]- ctx.prior_observed_acted_pages = [{"url": "https://example.com/dashboard", "reached_via": "interaction"}]- assert render_todo_list(ctx) is None--def test_outputs_not_yet_observed() -> None:policy = RequestPolicy(completion_criteria=[_output_criterion("output.number_of_visitors_last_week")])ctx = make_copilot_ctx(request_policy=policy)@@ -80,16 +20,6 @@ def test_outputs_not_yet_observed() -> None:assert render_todo_list(ctx) == "- Outputs not yet observed: output.number_of_visitors_last_week"-def test_credential_id_on_non_fill_tool_is_not_a_login_attempt() -> None:- ctx = make_copilot_ctx(request_policy=_login_policy())- ctx.scout_trajectory = [- {"tool_name": "click", "source_url": "https://example.com/login", "credential_id": "cred-1"}- ]- todo = render_todo_list(ctx)- assert todo is not None- assert "login not yet attempted" in todo--def test_interaction_reached_page_suppresses_zero_interactions_line() -> None:policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])ctx = make_copilot_ctx(request_policy=policy)@@ -160,22 +90,13 @@ def test_all_done_returns_none() -> None:assert render_todo_list(make_copilot_ctx()) is None-def test_login_and_outputs_combined() -> None:- policy = _login_policy()- policy.completion_criteria = [_output_criterion("output.total")]- todo = render_todo_list(make_copilot_ctx(request_policy=policy))- assert todo is not None- assert "- Login: credential resolved but login not yet attempted" in todo- assert "- Outputs not yet observed: output.total" in todo--def test_dynamic_system_prompt_injects_todo_list(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(agent_module, "_build_system_prompt", lambda **_: "BASE PROMPT")instructions = agent_module._build_dynamic_system_prompt(tool_usage_guide="", config=agent_module.CopilotConfig())- ctx = make_copilot_ctx(request_policy=_login_policy())- prompt = instructions(SimpleNamespace(context=ctx), None)+ policy = RequestPolicy(completion_criteria=[_output_criterion("output.total")])+ prompt = instructions(SimpleNamespace(context=make_copilot_ctx(request_policy=policy)), None)assert "TODO — outstanding before you reply:" in prompt- assert "Login: credential resolved but login not yet attempted" in prompt+ assert "Outputs not yet observed: output.total" in promptdef test_dynamic_system_prompt_adds_nothing_when_nothing_pending(monkeypatch: pytest.MonkeyPatch) -> None: