Skip to content

Automate browser based workflows with AI

Copilot: use the credential ID someone pasted, however they worded the message (SKY-13552) (#8076)

Andrew Neilsondfe02ce8a482
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index f8341e218..55fe72030 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -1005,11 +1005,11 @@ def _bound_approved_credential_ids(request_policy: RequestPolicy, workflow_yaml:
def _allowed_unresolved_credential_ids(request_policy: RequestPolicy) -> set[str]:
+ # invalid_credential_ids is negation-filtered and affirmative; credential_refs is not — a
+ # model-carried or negated ref must not become draft-bindable (SKY-13552).
if not request_policy.allow_missing_credentials_in_draft:
return set()
- ids = set(request_policy.invalid_credential_ids)
- ids.update(ref for ref in request_policy.credential_refs if isinstance(ref, str) and ref.startswith("cred_"))
- return ids
+ return set(request_policy.invalid_credential_ids)
def _existing_workflow_credential_ids(request_policy: RequestPolicy) -> set[str]:
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index d10a88ecc..3c9cfe63f 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -234,14 +234,6 @@ _POSTFIX_CREDENTIAL_TOKEN_RE = re.compile(
r"\b(?:use|using|with)\s+(?:my\s+|the\s+)?(?:saved\s+)?([A-Za-z0-9_.@:-]{2,100})\s+credential\b",
re.I,
)
-_EXPLICIT_CREDENTIAL_ID_CONTEXT_RE = re.compile(
- r"\b(?:use|using|with|select|choose)\s+"
- r"(?:(?:my|the|a)\s+)?(?:(?:saved|stored)\s+)?(?:credential(?:\s+id)?\s*)?"
- r"(?::\s*)?$",
- re.I,
-)
-_CREDENTIAL_ID_LABEL_CONTEXT_RE = re.compile(r"\bcredential\s+id\s*$", re.I)
-_CREDENTIAL_ID_COORDINATOR_RE = re.compile(r"\s*,?\s*(?:and|or)\s*$", re.I)
_CREDENTIAL_REPLACEMENT_TARGET_RES = (
re.compile(
r"\bswitch(?:ing)?\s+to\b"
@@ -668,6 +660,9 @@ class RequestPolicy:
"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,
"credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly,
"resolved_credential_count": len(self.resolved_credentials),
+ "current_turn_named_credential_count": len(self.current_turn_named_credential_ids),
+ "invalid_credential_id_count": len(self.invalid_credential_ids),
+ "auto_bound_credential_count": len(self.auto_bound_credentials),
"has_completion_contract": bool(self.completion_contract),
"completion_criteria_count": len(self.graded_completion_criteria()),
"completion_criteria_implicit_count": sum(
@@ -4603,8 +4598,12 @@ def _classifier_ref_stated_in_message(policy: RequestPolicy, user_message: str)
def _explicit_credential_ids(user_message: str) -> list[str]:
+ """Credential IDs the user's own message hands to this turn.
+
+ Accepts the canonical ``cred_`` form and ``_MALFORMED_CREDENTIAL_ID_RE``'s ``cred <12+ digits>``
+ normalization; surrounding prose can only withdraw a token, never qualify it.
+ """
text = _credential_authority_text(user_message or "")
- explicit: list[str] = []
matches = sorted(
[(match, match.group(0)) for match in _CREDENTIAL_ID_RE.finditer(text)]
+ [(match, f"cred_{match.group(1)}") for match in _MALFORMED_CREDENTIAL_ID_RE.finditer(text)],
@@ -4615,27 +4614,15 @@ def _explicit_credential_ids(user_message: str) -> list[str]:
for match, credential_id in matches
if _credential_reference_is_negated(text, match.start())
}
- last_explicit_end: int | None = None
- for match, credential_id in matches:
- if last_negated_mention.get(credential_id, -1) > match.start():
- continue
- if _credential_reference_is_negated(text, match.start()) or _credential_reference_is_unrelated_replacement(
- text, match.start()
- ):
- continue
- context = text[max(0, match.start() - 64) : match.start()]
- if (
- _EXPLICIT_CREDENTIAL_ID_CONTEXT_RE.search(context)
- or _CREDENTIAL_ID_LABEL_CONTEXT_RE.search(context)
- or match.group(0).strip() == text.strip()
- or (
- last_explicit_end is not None
- and _CREDENTIAL_ID_COORDINATOR_RE.fullmatch(text[last_explicit_end : match.start()])
- )
- ):
- explicit.append(credential_id)
- last_explicit_end = match.end()
- return _clean_list(explicit)
+ return _clean_list(
+ [
+ credential_id
+ for match, credential_id in matches
+ if last_negated_mention.get(credential_id, -1) <= match.start()
+ and not _credential_reference_is_negated(text, match.start())
+ and not _credential_reference_is_unrelated_replacement(text, match.start())
+ ]
+ )
def credential_candidate_label(credential: Credential) -> str:
@@ -4999,23 +4986,29 @@ async def _resolve_credentials(
# regardless of the classifier label. Do not trust stale/model-authored
# credential_refs that are absent from the latest user message.
ids = _explicit_credential_ids(user_message)
+ explicit_name_candidates = _credential_name_candidates(user_message)
if ids:
existing = await app.DATABASE.credentials.get_credentials_by_ids(ids, organization_id=organization_id)
found = {credential.credential_id for credential in existing}
policy.resolved_credentials = existing
_record_current_turn_named_credentials(policy, existing)
policy.invalid_credential_ids = [credential_id for credential_id in ids if credential_id not in found]
- if policy.invalid_credential_ids and policy.testing_intent != "skip_test":
+ # An unresolvable stated ID asks rather than falling through to the inference tiers below,
+ # but it never blocks another stated reference that did resolve.
+ if not explicit_name_candidates:
+ if existing or not policy.invalid_credential_ids:
+ return
+ if policy.testing_intent == "skip_test":
+ policy.allow_run_blocks = False
+ policy.allow_missing_credentials_in_draft = True
+ return
formatted = ", ".join(f"`{credential_id}`" for credential_id in policy.invalid_credential_ids)
_block(
policy,
f"The credential ID(s) {formatted} were not found in this organization. Please provide a valid saved credential ID or explicitly ask for an unvalidated draft that will not be run yet.",
reason="credential_name_unresolved",
)
- elif policy.invalid_credential_ids:
- policy.allow_run_blocks = False
- policy.allow_missing_credentials_in_draft = True
- return
+ return
# `login_intent` is a classifier-authored hint that misses on runs where the model does not set
# it, so a message asking for a login in its own words must still count. A classifier that failed
@@ -5026,7 +5019,7 @@ async def _resolve_credentials(
or (policy.classifier_status != "fallback" and login_phrase_asserted)
or _classifier_ref_stated_in_message(policy, user_message)
)
- name_candidates = _credential_name_candidates(user_message)
+ name_candidates = explicit_name_candidates
credentials: list[Credential] | None = None
# Only a request that already asserts a login or credential intent may be matched against the
# org's saved names; without that gate an unrelated message would scan the credential list.
@@ -5040,7 +5033,7 @@ async def _resolve_credentials(
[credential for credential in credentials if credential.name in name_candidates]
)
if len(named_matches) == 1:
- policy.resolved_credentials = named_matches
+ policy.resolved_credentials = _deduplicate_credentials(list(policy.resolved_credentials) + named_matches)
_record_current_turn_named_credentials(policy, named_matches)
if (
policy.classifier_status == "fallback"
@@ -5070,6 +5063,9 @@ async def _resolve_credentials(
card_answerable=True,
)
return
+ if policy.resolved_credentials:
+ # A stated ID already resolved; an unmatched name rides along as a finding, not a wall.
+ return
if policy.testing_intent == "skip_test" or defer_unresolved_credential_name:
policy.allow_run_blocks, policy.allow_missing_credentials_in_draft = False, True
if defer_unresolved_credential_name:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index bf2e4027a..7d1d9032b 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -4231,7 +4231,7 @@ class TestRequestPolicyCredentialResolution:
assert policy.completion_contract_status == "present"
@pytest.mark.asyncio
- async def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:
+ async def test_missing_user_supplied_credential_id_rides_along_when_another_resolves(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot import request_policy as policy_module
from skyvern.forge.sdk.copilot.request_policy import build_request_policy
@@ -4254,14 +4254,12 @@ class TestRequestPolicyCredentialResolution:
assert policy.credential_input_kind == "credential_id"
assert policy.credential_refs == ["cred_valid", "cred_missing"]
assert policy.invalid_credential_ids == ["cred_missing"]
- assert policy.user_response_policy == "ask_clarification"
- assert policy.allow_update_workflow is False
- assert policy.allow_run_blocks is False
- assert policy.allow_missing_credentials_in_draft is False
- assert policy.clarification_question
- assert "cred_missing" in policy.clarification_question
- assert "not found in this organization" in policy.clarification_question
- assert "unvalidated draft" in policy.clarification_question
+ # A stated ID that resolved carries the turn; the unresolvable one rides along as a
+ # recorded finding instead of walling the request (SKY-13552).
+ assert [credential.credential_id for credential in policy.resolved_credentials] == ["cred_valid"]
+ assert policy.user_response_policy != "ask_clarification"
+ assert policy.allow_update_workflow is True
+ assert policy.allow_run_blocks is True
get_credentials_by_ids.assert_awaited_once_with(["cred_valid", "cred_missing"], organization_id="org-1")
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 940aabd4b..e7f8cddc3 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -1241,6 +1241,24 @@ def test_allows_explicit_unresolved_credential_id_for_untested_draft() -> None:
assert verdict.allowed
+def test_rejects_credential_ref_the_user_did_not_state_even_for_untested_draft() -> None:
+ """A cred_ ref carried only in credential_refs (model-authored or negated) is not draft-bindable (SKY-13552)."""
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ credential_input_kind="credential_id",
+ credential_refs=["cred_carried"],
+ invalid_credential_ids=[],
+ allow_missing_credentials_in_draft=True,
+ allow_run_blocks=False,
+ ),
+ workflow_yaml=_workflow_yaml().replace("cred_safe", "cred_carried"),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes
+
+
def test_rejects_unrequested_credential_id_even_when_untested_draft_allows_missing_credentials() -> None:
verdict = evaluate_output_policy(
request_policy=_policy(
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 4a8502f21..b0fe8e922 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -684,9 +684,11 @@ async def test_resolver_exact_name_is_independent_of_classifier_label(
@pytest.mark.asyncio
-async def test_explicit_id_wins_and_skips_name_and_url_scan() -> None:
+async def test_explicit_id_and_explicit_name_resolve_together() -> None:
+ """Both stated references resolve; neither preempts the other (SKY-13552)."""
+ named = _cred("named-login", "cred_named")
by_ids = AsyncMock(return_value=[_cred("id-login", "cred_explicit")])
- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an ID hit"))
+ load_mock = AsyncMock(return_value=[named])
policy = RequestPolicy(
credential_input_kind="credential_id",
credential_refs=["cred_explicit", "named-login"],
@@ -696,14 +698,14 @@ async def test_explicit_id_wins_and_skips_name_and_url_scan() -> None:
await _resolve_direct(
policy,
user_message="Use credential ID cred_explicit and the saved credential named named-login.",
- org_credentials=[],
+ org_credentials=[named],
get_credentials=load_mock,
get_credentials_by_ids=by_ids,
)
- assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit"]
+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit", "cred_named"]
+ assert policy.requires_user_clarification is False
by_ids.assert_awaited_once_with(["cred_explicit"], organization_id="o_test")
- load_mock.assert_not_awaited()
@pytest.mark.asyncio
@@ -711,7 +713,6 @@ async def test_explicit_id_wins_for_non_id_label_and_competing_url_match() -> No
explicit = _cred("id-login", "cred_explicit")
competing = _cred("url-login", "cred_url", tested_url="https://portal.example.com/login")
by_ids = AsyncMock(return_value=[explicit])
- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an ID hit"))
policy = RequestPolicy(
credential_input_kind="website_stored_credential",
credential_refs=["cred_explicit"],
@@ -722,13 +723,12 @@ async def test_explicit_id_wins_for_non_id_label_and_competing_url_match() -> No
policy,
user_message="Use credential ID cred_explicit for https://portal.example.com/login.",
org_credentials=[competing],
- get_credentials=load_mock,
get_credentials_by_ids=by_ids,
)
assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit"]
+ assert policy.auto_bound_credentials == []
by_ids.assert_awaited_once_with(["cred_explicit"], organization_id="o_test")
- load_mock.assert_not_awaited()
@pytest.mark.asyncio
@@ -774,6 +774,7 @@ async def test_classifier_id_absent_from_current_message_does_not_resolve() -> N
"Avoid using cred_prod.",
"Exclude credential ID cred_prod.",
"Do not use the old workflow because it still mentions credential ID cred_prod.",
+ "Do not use the old workflow because it mentions cred_prod.",
"Do not use this credential, credential ID cred_prod.",
],
)
@@ -801,12 +802,14 @@ async def test_negated_explicit_id_does_not_resolve(user_message: str) -> None:
[
"Use this as a documentation example: cred_prod.",
"Use the staging workflow; the notes mention cred_prod.",
- "Do not use the old workflow because it mentions cred_prod.",
'Document the string "cred_prod" in the workflow notes.',
+ "cred_prod\n\ncan you use this credential to log into the portal?",
+ "Here's a credential you can use to test: cred_prod",
],
)
-async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str) -> None:
- by_ids = AsyncMock(side_effect=AssertionError("unrelated credential IDs must not be resolved"))
+async def test_stated_explicit_id_resolves_regardless_of_prose_shape(user_message: str) -> None:
+ """A cred_ ID the user typed is supplied, whatever surrounds it; only negation withdraws it (SKY-13552)."""
+ by_ids = AsyncMock(return_value=[_cred("prod-login", "cred_prod")])
policy = RequestPolicy(
credential_input_kind="credential_id",
credential_refs=["cred_prod"],
@@ -819,8 +822,9 @@ async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str)
get_credentials_by_ids=by_ids,
)
- assert policy.resolved_credentials == []
- by_ids.assert_not_awaited()
+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_prod"]
+ assert policy.requires_user_clarification is False
+ by_ids.assert_awaited_once_with(["cred_prod"], organization_id="o_test")
@pytest.mark.asyncio
@@ -979,8 +983,8 @@ async def test_bare_id_appositive_negation_revokes_earlier_authority() -> None:
@pytest.mark.asyncio
-async def test_invalid_explicit_id_does_not_fall_through_to_name() -> None:
- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an explicit ID"))
+async def test_invalid_explicit_id_with_unknown_name_asks_instead_of_inferring() -> None:
+ """Nothing stated resolved, so the turn asks; it never falls through to the inference tiers."""
policy = RequestPolicy(
credential_input_kind="credential_id",
credential_refs=["cred_missing", "named-login"],
@@ -990,14 +994,34 @@ async def test_invalid_explicit_id_does_not_fall_through_to_name() -> None:
policy,
user_message="Use credential ID cred_missing and the saved credential named named-login.",
org_credentials=[],
- get_credentials=load_mock,
)
assert policy.invalid_credential_ids == ["cred_missing"]
+ assert policy.resolved_credentials == []
assert policy.clarification_reason == "credential_name_unresolved"
- assert policy.credential_ask_card_answerable is False
- assert credential_pause_reason(SimpleNamespace(request_policy=policy)) is None
- load_mock.assert_not_awaited()
+ assert policy.requires_user_clarification is True
+
+
+async def test_invalid_explicit_id_rides_along_when_another_stated_reference_resolves() -> None:
+ """An unresolvable stated ID never blocks a stated reference that did resolve (SKY-13552)."""
+ by_ids = AsyncMock(return_value=[_cred("real-login", "cred_real")])
+ policy = RequestPolicy(
+ credential_input_kind="credential_id",
+ credential_refs=["cred_real", "cred_missing"],
+ )
+
+ await _resolve_direct(
+ policy,
+ user_message="Use cred_real and cred_missing to log in.",
+ org_credentials=[],
+ get_credentials_by_ids=by_ids,
+ )
+
+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_real"]
+ assert policy.invalid_credential_ids == ["cred_missing"]
+ assert policy.requires_user_clarification is False
+ assert policy.allow_run_blocks is True
@pytest.mark.asyncio
@@ -1553,7 +1577,7 @@ async def test_website_kind_explicit_name_wins_over_url_matched_card() -> None:
@pytest.mark.asyncio
-async def test_success_credential_id_intent_is_not_overridden_by_name_scan() -> None:
+async def test_success_credential_id_and_stated_name_both_resolve() -> None:
by_ids = AsyncMock(return_value=[_cred("real", "cred_real")])
load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])
policy = await _build_with_forced_classifier(
@@ -1563,14 +1587,13 @@ async def test_success_credential_id_intent_is_not_overridden_by_name_scan() ->
credential_refs=["cred_real"],
classifier_status="success",
),
- org_credentials=[],
+ org_credentials=[_cred("mock-portal-login", "cred_login")],
get_credentials=load_mock,
get_credentials_by_ids=by_ids,
)
assert policy.credential_input_kind == "credential_id"
- assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real"]
- load_mock.assert_not_awaited()
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real", "cred_login"]
@pytest.mark.asyncio