Skip to content

Automate browser based workflows with AI

Return a card-connected credential as a typed id and bind its login origin (SKY-13432) (#8013)

Andrew Neilson7abc7256d45f
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
index 8ffdc4bb9..e4ec76662 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.credentialCard.test.tsx
@@ -1,3 +1,4 @@
+import type { ReactNode } from "react";
import {
act,
cleanup,
@@ -114,6 +115,39 @@ vi.mock("@/hooks/useCredentialGetter", () => ({
vi.mock("@/components/ui/use-toast", () => ({ toast: toastFn }));
+const { selectValueChange } = vi.hoisted(() => ({
+ selectValueChange: { current: null as ((value: string) => void) | null },
+}));
+
+vi.mock("@/components/ui/select", () => ({
+ Select: ({
+ children,
+ onValueChange,
+ }: {
+ children?: ReactNode;
+ onValueChange?: (value: string) => void;
+ }) => {
+ selectValueChange.current = onValueChange ?? null;
+ return <div data-testid="mock-select">{children}</div>;
+ },
+ SelectTrigger: ({ children }: { children?: ReactNode }) => (
+ <div>{children}</div>
+ ),
+ SelectValue: ({ placeholder }: { placeholder?: string }) => (
+ <span>{placeholder}</span>
+ ),
+ SelectContent: ({ children }: { children?: ReactNode }) => (
+ <div>{children}</div>
+ ),
+ SelectItem: ({
+ children,
+ value,
+ }: {
+ children?: ReactNode;
+ value: string;
+ }) => <div data-testid={`select-item-${value}`}>{children}</div>,
+}));
+
// The real modal pulls react-query; a stub is enough to prove the connect →
// modal → onCredentialCreated wiring.
vi.mock("@/routes/credentials/CredentialsModal", () => ({
@@ -463,6 +497,38 @@ describe("WorkflowCopilotChat — credential card wiring (flag on)", () => {
action: "connected",
credential_id: "new-cred-1",
});
+ expect(postStreaming).toHaveBeenCalledTimes(1);
+ });
+
+ it("offers the credential_refs candidates even when none has a tested_url", async () => {
+ flagMap.current = { [COPILOT_UX_V1_FLAG]: true };
+ credentialsData.current = [
+ { credential_id: "cred-abc", name: "abc", tested_url: null },
+ { credential_id: "cred-spare", name: "spare-portal", tested_url: null },
+ { credential_id: "cred-other", name: "unrelated", tested_url: null },
+ ];
+ await renderChat();
+ await submit("build me a workflow");
+ await waitFor(() => expect(postStreaming).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ streamCalls[0]!.onMessage(turnStart());
+ streamCalls[0]!.onMessage(
+ credentialFrame({ credential_refs: ["cred-abc", "cred-spare"] }),
+ );
+ });
+ expect(await screen.findByText("Use existing…")).toBeTruthy();
+ expect(screen.getByTestId("select-item-cred-abc")).toBeTruthy();
+ expect(screen.getByTestId("select-item-cred-spare")).toBeTruthy();
+ expect(screen.queryByTestId("select-item-cred-other")).toBeNull();
+ await act(async () => {
+ selectValueChange.current?.("cred-spare");
+ });
+ await waitFor(() => expect(credentialResponsePosts()).toHaveLength(1));
+ expect(credentialResponsePosts()[0]![1]).toMatchObject({
+ action: "connected",
+ credential_id: "cred-spare",
+ });
+ expect(postStreaming).toHaveBeenCalledTimes(1);
});
it("keeps the card actionable, toasts, and never logs the raw error (resume_token leak) when the resume POST fails", async () => {
diff --git a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
index 596ed27c1..f3ffcb7f7 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/WorkflowCopilotChat.tsx
@@ -1058,6 +1058,17 @@ export function WorkflowCopilotChat({
}, [copilotUxV1Enabled, livePauseFrame, credentialsList, credentialGetter]);
const liveMatchingCredentials = useMemo<MatchingCredential[]>(() => {
if (!livePauseFrame) return [];
+ const toMatch = (
+ credential: CredentialApiResponse,
+ ): MatchingCredential => ({
+ credentialId: credential.credential_id,
+ name: credential.name,
+ });
+ // credential_refs may carry names as well as ids; intersecting on id drops the rest.
+ const asked = (credentialsList ?? []).filter((credential) =>
+ livePauseFrame.credential_refs.includes(credential.credential_id),
+ );
+ if (asked.length > 0) return asked.map(toMatch);
const hosts = livePauseFrame.login_page_urls
.map(getHostname)
.filter((host): host is string => host !== null);
@@ -1069,10 +1080,7 @@ export function WorkflowCopilotChat({
: null;
return host !== null && hosts.includes(host);
})
- .map((credential) => ({
- credentialId: credential.credential_id,
- name: credential.name,
- }));
+ .map(toMatch);
}, [livePauseFrame, credentialsList]);
const respondToCredentialPause = useCallback(
async (
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index c0fe03842..b356bda6e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -110,7 +110,7 @@ from skyvern.forge.sdk.copilot.credential_literal_rebind import (
rebind_scouted_credential_literals,
scouted_credential_targets,
)
-from skyvern.forge.sdk.copilot.credential_pause import preflight_credential_pause
+from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason, preflight_credential_pause
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
artifact_health_blocked,
@@ -5019,7 +5019,7 @@ async def _run_copilot_turn_impl(
)
if request_policy is not None and request_policy_guardrail_result.output.tripwire_triggered:
preflight_resolution = None
- if request_policy.clarification_reason == "login_credentials_unresolved":
+ if credential_pause_reason(ctx) is not None:
preflight_resolution = await preflight_credential_pause(ctx, stream, copilot_config)
if preflight_resolution is not None:
await _resume_turn_intent_after_preflight_credential(ctx, request_policy, policy_inputs)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index eb0767ef7..9d4c51083 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -627,7 +627,13 @@ def record_approved_credentials_in_global_llm_context(ctx: CopilotContext, raw_c
sc = StructuredContext.from_json_str(raw_context)
existing_ids = {record.credential_id for record in sc.approved_credentials}
for credential in policy.resolved_credentials:
- if credential.credential_id in existing_ids or credential.credential_id in policy.live_page_admitted_urls:
+ # A credential the user picked from the card is durable approval even though the resume
+ # stamped an origin for it; only page-vouched ids have to be re-earned.
+ stamped_by_page = (
+ credential.credential_id in policy.live_page_admitted_urls
+ and credential.credential_id != ctx.credential_pause_connected_credential_id
+ )
+ if credential.credential_id in existing_ids or stamped_by_page:
continue
sc.approved_credentials.append(ApprovedCredential(credential_id=credential.credential_id))
existing_ids.add(credential.credential_id)
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
index 8394e2e62..34d8ec0be 100644
--- a/skyvern/forge/sdk/copilot/credential_pause.py
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -33,6 +33,7 @@ import structlog
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.credential_resolution import loggable_origin, url_parts
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import DiagnosisFailureType, RepairNextAction
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.schemas.credentials import Credential
@@ -239,7 +240,12 @@ def credential_pause_reason(ctx: Any) -> str | None:
"""
policy = getattr(ctx, "request_policy", None)
skip_test = isinstance(policy, RequestPolicy) and policy.testing_intent == "skip_test"
- if isinstance(policy, RequestPolicy) and policy.clarification_reason == "login_credentials_unresolved":
+ if isinstance(policy, RequestPolicy) and (
+ policy.clarification_reason == "login_credentials_unresolved"
+ # Every ask the credential card can answer resumes through the same typed contract, so
+ # the reason literal no longer distinguishes them -- `card_answerable` does.
+ or (policy.requires_user_clarification and policy.credential_ask_card_answerable)
+ ):
return "login_credentials_unresolved"
if getattr(ctx, "last_run_skipped_unbound_credentials", False) and not skip_test:
@@ -302,6 +308,27 @@ def _defang(text: str) -> str:
return re.sub(r"\s+", " ", text).replace('"', "").strip()[:200]
+def _bind_connected_credential_origin(policy: RequestPolicy, credential: Credential) -> str | None:
+ """Stamp the server-held origin this ask was formed against, so a card-created credential with
+ no ``tested_url`` still has an intended origin at the fill seam. Written directly rather than
+ through ``_record_live_page_admission``, which skips already-resolved ids and would stamp
+ nothing here.
+ """
+ ask_urls = policy.credential_ask_login_page_urls or policy.login_page_urls
+ by_origin: dict[str, str] = {}
+ for url in ask_urls:
+ parts = url_parts(url)
+ if parts is None:
+ continue
+ *_, origin = parts
+ by_origin[origin] = url
+ if len(by_origin) != 1:
+ return None
+ admitted_url = next(iter(by_origin.values()))
+ policy.live_page_admitted_urls[credential.credential_id] = admitted_url
+ return admitted_url
+
+
def _apply_connected_credential_to_policy(ctx: Any, policy: RequestPolicy, credential: Credential) -> None:
"""Grant the resumed loop run authority for ``credential``.
@@ -316,6 +343,13 @@ def _apply_connected_credential_to_policy(ctx: Any, policy: RequestPolicy, crede
"""
ctx.test_after_update_done = False
ctx.credential_pause_connected_credential_id = credential.credential_id
+ admitted_url = _bind_connected_credential_origin(policy, credential)
+ LOG.info(
+ "copilot_credential_pause_connected",
+ card_answerable=policy.credential_ask_card_answerable,
+ credential_id=credential.credential_id,
+ bound_origin=loggable_origin(admitted_url) if admitted_url else None,
+ )
policy.resolved_credentials.append(credential)
policy.allow_run_blocks = True
policy.clarification_reason = "none"
@@ -435,7 +469,9 @@ async def _run_credential_pause(
policy = getattr(ctx, "request_policy", None)
login_page_urls = list(policy.login_page_urls) if isinstance(policy, RequestPolicy) else []
- credential_refs = list(policy.credential_refs) if isinstance(policy, RequestPolicy) else []
+ credential_refs: list[str] = []
+ if isinstance(policy, RequestPolicy):
+ credential_refs = list(policy.credential_ask_candidate_ids or policy.credential_refs)
timeout_seconds = copilot_config.credential_pause_timeout_seconds
now = datetime.now(timezone.utc)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 92e9fd159..dd813f993 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -181,6 +181,9 @@ _STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX = (
)
_STORED_CREDENTIAL_URL_QUESTION = f"{_STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX} {_CREDENTIALS_UI_DIRECTIONS}"
_AMBIGUOUS_URL_CREDENTIAL_QUESTION = "I found multiple stored credentials for that login page. Which one should I use?"
+_AMBIGUOUS_URLLESS_CREDENTIAL_QUESTION = (
+ "I found multiple saved credentials, and none of them is bound to that login page. Which one should I use?"
+)
_SIGNIN_EMAIL_QUESTION = "Which email address should I sign in with?"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
# A credential ID typed with the wrong separator (`cred 530…`, `cred-530…`). The
@@ -610,6 +613,13 @@ class RequestPolicy:
raw_secret_evidence: str | None = None
raw_secret_handling: RawSecretHandling = "none"
clarification_reason: ClarificationReason = "none"
+ # `clarification_reason` cannot say this on its own: credential_name_unresolved is also raised
+ # by raw-secret, invention and invalid-id asks the credential card cannot answer.
+ credential_ask_card_answerable: bool = False
+ # The server-held login page URLs this ask was formed against; the only origin source the
+ # connected resume may bind from.
+ credential_ask_login_page_urls: list[str] = field(default_factory=list)
+ credential_ask_candidate_ids: list[str] = field(default_factory=list)
existing_workflow_credential_ids: list[str] = field(default_factory=list)
# Sorted at the trace/JSON boundary; YAML traversal uses sets.
existing_workflow_credential_origins: dict[str, list[str]] = field(default_factory=dict)
@@ -4542,14 +4552,23 @@ def _block(
candidates: list[Credential] | None = None,
*,
reason: ClarificationReason | None = None,
+ card_answerable: bool = False,
) -> None:
policy.requires_user_clarification = True
policy.user_response_policy = "ask_clarification"
policy.allow_update_workflow = policy.allow_run_blocks = False
if reason is not None:
policy.clarification_reason = reason
+ policy.credential_ask_card_answerable = card_answerable
+ policy.credential_ask_candidate_ids = [candidate.credential_id for candidate in candidates or []]
if candidates:
- question += "\n\nSafe matches:\n" + "\n".join(
+ # "Safe" is a claim about the login page vouching for each candidate, so it only holds
+ # when the ask had a page to match against and every candidate was tested against one.
+ matched_by_url = bool(policy.credential_ask_login_page_urls) and all(
+ candidate.tested_url for candidate in candidates
+ )
+ label = "Safe matches" if matched_by_url else "Saved credentials"
+ question += f"\n\n{label}:\n" + "\n".join(
f"- {credential_candidate_label(candidate)}" for candidate in candidates
)
policy.clarification_question = question
@@ -4933,6 +4952,7 @@ async def _resolve_credentials(
question,
named_matches,
reason="credential_name_unresolved",
+ card_answerable=True,
)
return
if policy.testing_intent == "skip_test" or defer_unresolved_credential_name:
@@ -4953,6 +4973,7 @@ async def _resolve_credentials(
policy,
question,
reason="credential_name_unresolved",
+ card_answerable=True,
)
return
@@ -4979,6 +5000,7 @@ async def _resolve_credentials(
url_candidates = message_url_candidates
if url_candidates:
+ policy.credential_ask_login_page_urls = list(url_candidates)
if credentials is None:
credentials = await _load_credentials(organization_id)
resolution = resolve_by_url(
@@ -4999,9 +5021,12 @@ async def _resolve_credentials(
policy.discovered_credentials = list(resolution.candidates)
_block(
policy,
- _AMBIGUOUS_URL_CREDENTIAL_QUESTION,
+ _AMBIGUOUS_URLLESS_CREDENTIAL_QUESTION
+ if resolution.tier == "urlless_sole"
+ else _AMBIGUOUS_URL_CREDENTIAL_QUESTION,
list(resolution.candidates),
reason="credential_name_unresolved",
+ card_answerable=True,
)
return
@@ -5018,6 +5043,7 @@ async def _resolve_credentials(
policy,
_SAVED_CREDENTIAL_NAME_QUESTION,
reason="credential_name_unresolved",
+ card_answerable=True,
)
return
@@ -5033,6 +5059,7 @@ async def _resolve_credentials(
policy,
"I could not find a stored credential for that login page. Please select a saved credential by exact name or a credential ID beginning with cred_, or create one in the Credentials UI.",
reason="credential_name_unresolved",
+ card_answerable=True,
)
diff --git a/tests/unit/test_copilot_context.py b/tests/unit/test_copilot_context.py
index 4cf182bc8..b35f5d8d4 100644
--- a/tests/unit/test_copilot_context.py
+++ b/tests/unit/test_copilot_context.py
@@ -734,11 +734,12 @@ class TestCopilotContext:
def _policy_ctx(resolved: list[SimpleNamespace], credential_input_kind: str = "credential_name") -> SimpleNamespace:
return SimpleNamespace(
+ credential_pause_connected_credential_id=None,
request_policy=SimpleNamespace(
resolved_credentials=resolved,
credential_input_kind=credential_input_kind,
live_page_admitted_urls={},
- )
+ ),
)
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index 8a8e632be..12098308c 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -39,7 +39,11 @@ from skyvern.forge.sdk.copilot import credential_pause as credential_pause_modul
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import RequestPolicyGuardrailInputs, _derive_turn_intent_on_context
from skyvern.forge.sdk.copilot.config import CopilotConfig
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.context import (
+ CopilotContext,
+ StructuredContext,
+ record_approved_credentials_in_global_llm_context,
+)
from skyvern.forge.sdk.copilot.credential_pause import (
CredentialPauseRejection,
CredentialPauseResolution,
@@ -69,6 +73,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
)
from skyvern.forge.sdk.copilot.request_policy import (
CREDENTIAL_PROMPT_CLARIFICATION_REASONS,
+ ClarificationReason,
RequestPolicy,
credential_prompt_reason,
)
@@ -85,7 +90,10 @@ from skyvern.forge.sdk.routes.workflow_copilot import (
workflow_copilot_credential_response,
)
from skyvern.forge.sdk.schemas.credentials import Credential, CredentialType, CredentialVaultType
-from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotStreamMessageType
+from skyvern.forge.sdk.schemas.workflow_copilot import (
+ WorkflowCopilotCredentialRequiredUpdate,
+ WorkflowCopilotStreamMessageType,
+)
from tests.unit.conftest import make_copilot_context
@@ -1854,6 +1862,109 @@ def test_reason_does_not_fire_for_a_login_ask_that_resolved_to_a_credential() ->
assert credential_pause_module.credential_pause_reason(ctx) is None
+def _card_answerable_policy(
+ *,
+ clarification_reason: ClarificationReason = "credential_name_unresolved",
+ clarification_question: str = "Which one should I use?",
+ card_answerable: bool = True,
+ ask_login_page_urls: list[str] | None = None,
+) -> RequestPolicy:
+ """A disambiguation ask the credential card can answer: a different reason to the login ask."""
+ return RequestPolicy(
+ login_intent=True,
+ clarification_reason=clarification_reason,
+ clarification_question=clarification_question,
+ requires_user_clarification=True,
+ user_response_policy="ask_clarification",
+ allow_update_workflow=False,
+ allow_run_blocks=False,
+ credential_ask_card_answerable=card_answerable,
+ credential_ask_candidate_ids=["cred_1", "cred_2"] if card_answerable else [],
+ credential_ask_login_page_urls=["https://portal.example.com/login"]
+ if ask_login_page_urls is None
+ else ask_login_page_urls,
+ )
+
+
+def test_reason_fires_on_a_card_answerable_ask_under_a_different_reason() -> None:
+ ctx = make_copilot_context()
+ ctx.request_policy = _card_answerable_policy()
+
+ assert credential_pause_module.credential_pause_reason(ctx) == "login_credentials_unresolved"
+
+
+def test_reason_ignores_a_clarification_the_card_cannot_answer() -> None:
+ ctx = make_copilot_context()
+ ctx.request_policy = _card_answerable_policy(clarification_reason="raw_secret", card_answerable=False)
+
+ assert credential_pause_module.credential_pause_reason(ctx) is None
+
+
+def test_widened_reason_still_builds_a_valid_credential_required_frame() -> None:
+ ctx = make_copilot_context()
+ ctx.request_policy = _card_answerable_policy()
+ reason = credential_pause_module.credential_pause_reason(ctx)
+
+ frame = WorkflowCopilotCredentialRequiredUpdate(
+ type=WorkflowCopilotStreamMessageType.CREDENTIAL_REQUIRED,
+ turn_id="turn-1",
+ workflow_copilot_chat_id="chat-1",
+ resume_token="rt",
+ reason=reason or "",
+ message="Which one should I use?",
+ login_page_urls=[],
+ credential_refs=[],
+ timeout_seconds=5,
+ expires_at=datetime.now(timezone.utc),
+ timestamp=datetime.now(timezone.utc),
+ )
+
+ assert frame.reason == "login_credentials_unresolved"
+
+
+def test_connected_resume_stamps_the_single_ask_origin() -> None:
+ policy = _card_answerable_policy()
+ credential = _make_credential()
+
+ credential_pause_module._apply_connected_credential_to_policy(make_copilot_context(), policy, credential)
+
+ assert policy.live_page_admitted_urls == {"cred_1": "https://portal.example.com/login"}
+
+
+def test_connected_resume_stamps_nothing_when_the_ask_spans_several_origins() -> None:
+ policy = _card_answerable_policy(
+ ask_login_page_urls=["https://portal.example.com/login", "https://other.example.com/login"]
+ )
+
+ credential_pause_module._apply_connected_credential_to_policy(make_copilot_context(), policy, _make_credential())
+
+ assert policy.live_page_admitted_urls == {}
+
+
+def test_connected_resume_never_binds_an_origin_from_prose() -> None:
+ policy = _card_answerable_policy(
+ ask_login_page_urls=[],
+ clarification_question="Sign in at https://prose.example.com/login first. Which one should I use?",
+ )
+
+ credential_pause_module._apply_connected_credential_to_policy(make_copilot_context(), policy, _make_credential())
+
+ assert policy.live_page_admitted_urls == {}
+
+
+def test_a_card_connected_credential_keeps_its_durable_cross_turn_approval() -> None:
+ ctx = make_copilot_context()
+ policy = _card_answerable_policy()
+ ctx.request_policy = policy
+
+ credential_pause_module._apply_connected_credential_to_policy(ctx, policy, _make_credential())
+ carried = record_approved_credentials_in_global_llm_context(ctx, None)
+
+ assert [record.credential_id for record in StructuredContext.from_json_str(carried).approved_credentials] == [
+ "cred_1"
+ ]
+
+
@pytest.mark.asyncio
async def test_preflight_connect_binds_the_credential_and_clears_the_clarification_block(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_copilot_fill_credential_field.py b/tests/unit/test_copilot_fill_credential_field.py
index 29ec9d59d..63fb4b685 100644
--- a/tests/unit/test_copilot_fill_credential_field.py
+++ b/tests/unit/test_copilot_fill_credential_field.py
@@ -726,6 +726,48 @@ class TestCredentialFillLivePageAdmission:
assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]
assert [c.credential_id for c in policy.resolved_credentials] == ["cred_analytics"]
+ @pytest.mark.asyncio
+ async def test_a_card_connected_credential_with_no_tested_url_fills(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """The resume stamps the ask's origin, so a credential created from the card can sign in."""
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
+ policy = RequestPolicy(
+ resolved_credentials=[_resolved_credential(tested_url=None)],
+ live_page_admitted_urls={"cred_123": _FIXTURE_LOGIN_URL},
+ )
+ ctx = _ctx(request_policy=policy)
+
+ with patch(
+ "skyvern.forge.app.DATABASE.credentials.get_credentials",
+ new=AsyncMock(side_effect=AssertionError("a stamped credential needs no org scan")),
+ ):
+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+
+ assert result["ok"] is True
+ assert page.fill_calls == [("#passwordInput", _FAKE_PASSWORD)]
+
+ @pytest.mark.asyncio
+ async def test_a_card_connected_credential_still_cannot_follow_a_redirect(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ page = _FakePage()
+ _wire_impl(monkeypatch, page)
+ policy = RequestPolicy(
+ resolved_credentials=[_resolved_credential(tested_url=None)],
+ live_page_admitted_urls={"cred_123": _FIXTURE_LOGIN_URL},
+ )
+ ctx = _ctx(request_policy=policy)
+
+ async def redirect_then_capture(_ctx: Any) -> None:
+ page.url = "https://elsewhere.example.com/collect"
+
+ monkeypatch.setattr(credential_fill_module, "_capture_scout_source_url", redirect_then_capture)
+
+ result = await tools_module._fill_credential_field_impl(ctx, "#passwordInput", "cred_123", "password")
+
+ assert result["ok"] is False
+ assert page.fill_calls == []
+
@pytest.mark.asyncio
async def test_a_redirect_after_admission_stops_the_secret_reaching_the_new_page(
self, monkeypatch: pytest.MonkeyPatch
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 4b0ed7276..38eea24bb 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.credential_pause import credential_pause_reason
from skyvern.forge.sdk.copilot.credential_resolution import resolve_by_url
from skyvern.forge.sdk.copilot.request_policy import (
_AMBIGUOUS_URL_CREDENTIAL_QUESTION,
+ _AMBIGUOUS_URLLESS_CREDENTIAL_QUESTION,
_LOGIN_CREDENTIAL_QUESTION,
_SAVED_CREDENTIAL_NAME_QUESTION,
_SAVED_CREDENTIAL_NAME_QUESTION_STABLE_PREFIX,
@@ -630,9 +631,30 @@ async def test_name_miss_does_not_advance_to_classifier_only_url() -> None:
assert policy.resolved_credentials == []
assert policy.clarification_reason == "credential_name_unresolved"
assert "missing-name" in (policy.clarification_question or "")
+ assert policy.credential_ask_card_answerable is True
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) == "login_credentials_unresolved"
load_mock.assert_awaited_once()
[email protected]("name", ["abc", "spare-portal"], ids=["three_char", "twelve_char"])
+async def test_url_less_ambiguity_is_independent_of_credential_name_length(name: str) -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Log into the portal at https://portal.example.com/login.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="website_stored_credential",
+ login_page_urls=["https://portal.example.com/login"],
+ login_intent=True,
+ classifier_status="success",
+ ),
+ org_credentials=[_cred(name, "cred_one"), _cred("other-login", "cred_two")],
+ )
+
+ assert policy.credential_ask_card_answerable is True
+ assert policy.credential_ask_candidate_ids == ["cred_one", "cred_two"]
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) == "login_credentials_unresolved"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"kind",
@@ -969,6 +991,8 @@ async def test_invalid_explicit_id_does_not_fall_through_to_name() -> None:
assert policy.invalid_credential_ids == ["cred_missing"]
assert policy.clarification_reason == "credential_name_unresolved"
+ assert policy.credential_ask_card_answerable is False
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) is None
load_mock.assert_not_awaited()
@@ -1014,6 +1038,34 @@ async def test_multiple_name_hits_keep_the_existing_picker_and_skip_url() -> Non
load_mock.assert_awaited_once()
+async def test_name_collision_candidates_are_not_labelled_safe_matches() -> None:
+ """A name-formed ask has no login page vouching for its candidates.
+
+ Each candidate carrying some ``tested_url`` says nothing about *this* page, so the picker
+ must not call them safe matches even when every one of them has been tested somewhere.
+ """
+ policy = RequestPolicy(
+ credential_input_kind="website_stored_credential",
+ login_page_urls=["https://portal.example.com/login"],
+ login_intent=True,
+ )
+
+ await _resolve_direct(
+ policy,
+ user_message="Use the credential named shared-login.",
+ org_credentials=[
+ _cred("shared-login", "cred_one", tested_url="https://elsewhere.example.com/login"),
+ _cred("shared-login", "cred_two", tested_url="https://other.example.com/login"),
+ ],
+ )
+
+ assert policy.clarification_question is not None
+ assert "Saved credentials:" in policy.clarification_question
+ assert "Safe matches" not in policy.clarification_question
+ assert policy.credential_ask_login_page_urls == []
+
+
@pytest.mark.asyncio
async def test_url_tier_deduplicates_and_keeps_existing_picker() -> None:
first = _cred("first", "cred_first", tested_url="https://portal.example.com/login")
@@ -1384,12 +1436,18 @@ async def test_website_kind_multiple_url_less_credentials_ask_which_one() -> Non
assert policy.requires_user_clarification is True
assert policy.clarification_question is not None
- assert policy.clarification_question.startswith(_AMBIGUOUS_URL_CREDENTIAL_QUESTION)
+ assert policy.clarification_question.startswith(_AMBIGUOUS_URLLESS_CREDENTIAL_QUESTION)
+ assert "Safe matches" not in policy.clarification_question
+ assert "for that login page" not in policy.clarification_question
assert "first-login" in policy.clarification_question
assert "second-login" in policy.clarification_question
assert policy.clarification_reason == "credential_name_unresolved"
assert sorted(c.credential_id for c in policy.discovered_credentials) == ["cred_one", "cred_two"]
assert not policy.resolved_credentials
+ assert policy.credential_ask_card_answerable is True
+ assert policy.credential_ask_candidate_ids == ["cred_one", "cred_two"]
+ assert policy.credential_ask_login_page_urls == ["https://portal.example.com/login"]
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) == "login_credentials_unresolved"
@pytest.mark.asyncio
@@ -1410,7 +1468,9 @@ async def test_website_kind_multiple_url_matches_keep_disambiguation_reason() ->
assert policy.clarification_question is not None
assert policy.clarification_question.startswith(_AMBIGUOUS_URL_CREDENTIAL_QUESTION)
+ assert "Safe matches:" in policy.clarification_question
assert policy.clarification_reason == "credential_name_unresolved"
+ assert policy.credential_ask_card_answerable is True
assert sorted(c.credential_id for c in policy.discovered_credentials) == ["cred_one", "cred_two"]
@@ -2215,6 +2275,8 @@ async def test_a_pasted_secret_under_login_intent_keeps_the_raw_secret_refusal()
)
assert policy.clarification_reason == "raw_secret"
+ assert policy.credential_ask_card_answerable is False
+ assert credential_pause_reason(SimpleNamespace(request_policy=policy)) is None
def test_login_credentials_unresolved_surfaces_a_prompt_but_grants_no_deferred_draft_authority() -> None:
@@ -3203,7 +3265,9 @@ async def test_a_live_page_admission_does_not_carry_to_the_next_turn() -> None:
page_url=_LIVE_LOGIN_URL,
org_credentials=[_cred("analytics", "cred_analytics", _SAVED_LOGIN_URL)],
)
- carried_context = record_approved_credentials_in_global_llm_context(SimpleNamespace(request_policy=policy), None)
+ carried_context = record_approved_credentials_in_global_llm_context(
+ SimpleNamespace(request_policy=policy, credential_pause_connected_credential_id=None), None
+ )
next_turn = RequestPolicy()
with patch(