Skip to content

Automate browser based workflows with AI

fix(SKY-11539): resolve exactly-named org credential regardless of classifier status/kind (#7243)

Andrew Neilsonc269e36c162d
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index c23579c81..4b93ae38e 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -153,6 +153,7 @@ _NAMED_CREDENTIAL_TOKEN_RE = re.compile(
r"\b(?:saved\s+credential|credential)\s+(?:named|called)\s+([A-Za-z0-9_.@:-]{2,100})\b",
re.I,
)
+_CREDENTIAL_QUOTE_CONTEXT_RE = re.compile(r"\b(?:credentials?|log[\s-]?in)\b", re.I)
_CODE_BLOCK_AUTHORING_MARKERS = ("code block", "code-block", "codeblock")
_LOGIN_BLOCK_BAN_MARKERS = ("do not create a login block", "don't create a login block", "no login block")
_CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "one-time-code")
@@ -2094,33 +2095,52 @@ async def _load_credentials(organization_id: str) -> list[Credential]:
page += 1
-def _fallback_credential_name_candidates(user_message: str) -> list[str]:
+def _quote_in_credential_context(user_message: str, quote_start: int) -> bool:
+ return bool(_CREDENTIAL_QUOTE_CONTEXT_RE.search(user_message[max(0, quote_start - 48) : quote_start]))
+
+
+def _exact_credential_name_candidates(user_message: str) -> list[str]:
+ text = user_message or ""
candidates: list[str] = []
- for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(user_message or ""):
- value = next((group for group in match.groups() if group), "")
- if value.strip():
- candidates.append(value.strip())
- for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(user_message or ""):
+ for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(text):
+ value = next((group for group in match.groups() if group), "").strip()
+ if value and _quote_in_credential_context(text, match.start()):
+ candidates.append(value)
+ for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(text):
value = match.group(1).strip()
if value:
candidates.append(value)
return _clean_list(candidates)
-async def _apply_fallback_credential_name_scope(
+def _exact_credential_name_scan_eligible(policy: RequestPolicy) -> bool:
+ if policy.raw_secret_detected:
+ return False
+ if policy.clarification_reason in _PRE_RESOLUTION_CLARIFICATION_REASONS:
+ return False
+ if policy.credential_input_kind == "credential_name" and policy.credential_refs:
+ return False
+ return policy.credential_input_kind in ("none", "credential_name", "website_stored_credential")
+
+
+async def _apply_exact_credential_name_scope(
policy: RequestPolicy,
*,
user_message: str,
organization_id: str,
) -> None:
- if policy.classifier_status != "fallback":
- return
- if policy.raw_secret_detected or policy.credential_input_kind not in ("none", "credential_name"):
+ if not _exact_credential_name_scan_eligible(policy):
return
- candidates = _fallback_credential_name_candidates(user_message)
+ candidates = _exact_credential_name_candidates(user_message)
if not candidates:
return
credentials = await _load_credentials(organization_id)
+ if (
+ policy.credential_input_kind == "website_stored_credential"
+ and policy.login_page_urls
+ and _match_by_url(credentials, policy.login_page_urls)
+ ):
+ return
matched_names = _clean_list(
[candidate for candidate in candidates if any(credential.name == candidate for credential in credentials)]
)
@@ -2567,14 +2587,14 @@ async def build_request_policy(
credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()
}
try:
- await _apply_fallback_credential_name_scope(
+ await _apply_exact_credential_name_scope(
policy,
user_message=user_message,
organization_id=organization_id,
)
except Exception:
LOG.warning(
- "request-policy fallback credential-name extraction failed",
+ "request-policy exact credential-name extraction failed",
organization_id=organization_id,
exc_info=True,
)
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 06417464c..e1d5e5e49 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -441,3 +441,243 @@ async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_
assert policy.user_response_policy == "ask_clarification"
assert policy.allow_update_workflow is False
assert policy.allow_run_blocks is False
+
+
+def _cred(name: str, credential_id: str, tested_url: str | None = None) -> SimpleNamespace:
+ return SimpleNamespace(name=name, credential_id=credential_id, tested_url=tested_url)
+
+
+async def _build_with_forced_classifier(
+ *,
+ user_message: str,
+ classifier_policy: RequestPolicy,
+ org_credentials: list[SimpleNamespace],
+ get_credentials: AsyncMock | None = None,
+ get_credentials_by_ids: AsyncMock | None = None,
+) -> RequestPolicy:
+ load_mock = get_credentials or AsyncMock(return_value=org_credentials)
+ by_ids_mock = get_credentials_by_ids or AsyncMock(return_value=[])
+ with (
+ patch(
+ "skyvern.forge.sdk.copilot.request_policy._classify_request",
+ new=AsyncMock(return_value=classifier_policy),
+ ),
+ patch("skyvern.forge.app.DATABASE.credentials.get_credentials", new=load_mock),
+ patch("skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids", new=by_ids_mock),
+ ):
+ return await build_request_policy(
+ user_message=user_message,
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="o_test",
+ handler=None,
+ )
+
+
[email protected]("kind", ["credential_name", "none"])
+async def test_success_empty_refs_resolves_named_credential(kind: str) -> None:
+ credential = _cred("mock-portal-login", "cred_login")
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in with the saved credential named mock-portal-login.",
+ classifier_policy=RequestPolicy(credential_input_kind=kind, classifier_status="success"),
+ org_credentials=[credential, _cred("other", "cred_other")],
+ )
+
+ assert policy.classifier_status == "success"
+ assert policy.credential_input_kind == "credential_name"
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]
+ assert policy.clarification_reason != "credential_name_unresolved"
+
+
+async def test_success_website_kind_with_nonmatching_url_resolves_named_credential() -> None:
+ credential = _cred("mock-portal-login", "cred_login", tested_url="https://saved.example.net/signin")
+ policy = await _build_with_forced_classifier(
+ user_message="Log in with the saved credential named mock-portal-login here.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="website_stored_credential",
+ login_page_urls=["https://unrelated.example.com/login"],
+ classifier_status="success",
+ ),
+ org_credentials=[credential],
+ )
+
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]
+ assert policy.clarification_reason != "credential_name_unresolved"
+ assert policy.clarification_question is None
+
+
+async def test_success_website_kind_with_matching_url_keeps_url_credential() -> None:
+ url_credential = _cred("portal-cred", "cred_url", tested_url="https://portal.example.com/login")
+ named_credential = _cred("other-login", "cred_named", tested_url="https://elsewhere.example.net/login")
+ policy = await _build_with_forced_classifier(
+ user_message="Log in with the saved credential named other-login.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="website_stored_credential",
+ login_page_urls=["https://portal.example.com/login"],
+ classifier_status="success",
+ ),
+ org_credentials=[url_credential, named_credential],
+ )
+
+ assert policy.credential_input_kind == "website_stored_credential"
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_url"]
+
+
+async def test_success_credential_id_intent_is_not_overridden_by_name_scan() -> None:
+ by_ids = AsyncMock(return_value=[_cred("real", "cred_real")])
+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])
+ policy = await _build_with_forced_classifier(
+ user_message="Use cred_real, the saved credential named mock-portal-login.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="credential_id",
+ credential_refs=["cred_real"],
+ classifier_status="success",
+ ),
+ org_credentials=[],
+ get_credentials=load_mock,
+ get_credentials_by_ids=by_ids,
+ )
+
+ assert policy.credential_input_kind == "credential_id"
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real"]
+ load_mock.assert_not_awaited()
+
+
+async def test_success_populated_name_refs_are_not_narrowed_by_scan() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Use the saved credential named alpha.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="credential_name",
+ credential_refs=["alpha", "beta"],
+ classifier_status="success",
+ ),
+ org_credentials=[_cred("alpha", "cred_alpha")],
+ )
+
+ assert policy.credential_refs == ["alpha", "beta"]
+ assert policy.clarification_reason == "credential_name_unresolved"
+
+
+async def test_success_pre_resolution_clarification_is_preserved() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in with the saved credential named mock-portal-login.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="none",
+ requires_user_clarification=True,
+ clarification_reason="invalid_conditional_container",
+ classifier_status="success",
+ ),
+ org_credentials=[_cred("mock-portal-login", "cred_login")],
+ )
+
+ assert policy.clarification_reason == "invalid_conditional_container"
+ assert policy.resolved_credentials == []
+
+
+async def test_success_ambiguous_named_credentials_clarify_for_normal_request() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Use the credential named alpha or the credential named beta to sign in.",
+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),
+ org_credentials=[_cred("alpha", "cred_alpha"), _cred("beta", "cred_beta")],
+ )
+
+ assert policy.requires_user_clarification is True
+ assert policy.clarification_reason == "credential_name_unresolved"
+ assert policy.resolved_credentials == []
+
+
+async def test_success_ambiguous_named_credentials_draft_unbound_for_explicit_draft() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message=(
+ "Build this as a code block using the saved credential named alpha "
+ "or the saved credential named beta. Do not create a login block."
+ ),
+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),
+ org_credentials=[_cred("alpha", "cred_alpha"), _cred("beta", "cred_beta")],
+ )
+
+ assert policy.resolved_credentials == []
+ assert policy.allow_missing_credentials_in_draft is True
+ assert policy.requires_user_clarification is False
+
+
+async def test_success_incidental_quote_out_of_credential_context_is_not_rewritten() -> None:
+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])
+ policy = await _build_with_forced_classifier(
+ user_message='Set the page title to "mock-portal-login" before saving.',
+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),
+ org_credentials=[],
+ get_credentials=load_mock,
+ )
+
+ assert policy.credential_input_kind == "none"
+ assert policy.resolved_credentials == []
+ load_mock.assert_not_awaited()
+
+
+async def test_success_quote_after_word_containing_login_substring_is_not_context() -> None:
+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])
+ policy = await _build_with_forced_classifier(
+ user_message='Publish the blog in the section titled "mock-portal-login".',
+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),
+ org_credentials=[],
+ get_credentials=load_mock,
+ )
+
+ assert policy.credential_input_kind == "none"
+ assert policy.resolved_credentials == []
+ load_mock.assert_not_awaited()
+
+
+async def test_success_quoted_name_in_credential_context_resolves() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message='Sign in using the saved credential "mock-portal-login".',
+ classifier_policy=RequestPolicy(credential_input_kind="credential_name", classifier_status="success"),
+ org_credentials=[_cred("mock-portal-login", "cred_login")],
+ )
+
+ assert policy.credential_input_kind == "credential_name"
+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]
+
+
+async def test_success_zero_name_match_falls_through_to_clarification() -> None:
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in with the saved credential named ghost-cred.",
+ classifier_policy=RequestPolicy(credential_input_kind="credential_name", classifier_status="success"),
+ org_credentials=[_cred("mock-portal-login", "cred_login")],
+ )
+
+ assert policy.resolved_credentials == []
+ assert policy.clarification_reason == "credential_name_unresolved"
+
+
+async def test_success_raw_secret_blocks_name_scan_before_loading_credentials() -> None:
+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])
+ policy = await _build_with_forced_classifier(
+ user_message="Sign in with the saved credential named mock-portal-login.",
+ classifier_policy=RequestPolicy(
+ credential_input_kind="none",
+ raw_secret_detected=True,
+ raw_secret_handling="block",
+ classifier_status="success",
+ ),
+ org_credentials=[],
+ get_credentials=load_mock,
+ )
+
+ assert policy.resolved_credentials == []
+ load_mock.assert_not_awaited()