Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11539): resolve exactly-named org credential regardless of classifier status/kind (#7243)
Andrew Neilsonc269e36c162d
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex c23579c81..4b93ae38e 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -153,6 +153,7 @@ _NAMED_CREDENTIAL_TOKEN_RE = re.compile(r"\b(?:saved\s+credential|credential)\s+(?:named|called)\s+([A-Za-z0-9_.@:-]{2,100})\b",re.I,)+_CREDENTIAL_QUOTE_CONTEXT_RE = re.compile(r"\b(?:credentials?|log[\s-]?in)\b", re.I)_CODE_BLOCK_AUTHORING_MARKERS = ("code block", "code-block", "codeblock")_LOGIN_BLOCK_BAN_MARKERS = ("do not create a login block", "don't create a login block", "no login block")_CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "one-time-code")@@ -2094,33 +2095,52 @@ async def _load_credentials(organization_id: str) -> list[Credential]:page += 1-def _fallback_credential_name_candidates(user_message: str) -> list[str]:+def _quote_in_credential_context(user_message: str, quote_start: int) -> bool:+ return bool(_CREDENTIAL_QUOTE_CONTEXT_RE.search(user_message[max(0, quote_start - 48) : quote_start]))+++def _exact_credential_name_candidates(user_message: str) -> list[str]:+ text = user_message or ""candidates: list[str] = []- for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(user_message or ""):- value = next((group for group in match.groups() if group), "")- if value.strip():- candidates.append(value.strip())- for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(user_message or ""):+ for match in _QUOTED_CREDENTIAL_NAME_RE.finditer(text):+ value = next((group for group in match.groups() if group), "").strip()+ if value and _quote_in_credential_context(text, match.start()):+ candidates.append(value)+ for match in _NAMED_CREDENTIAL_TOKEN_RE.finditer(text):value = match.group(1).strip()if value:candidates.append(value)return _clean_list(candidates)-async def _apply_fallback_credential_name_scope(+def _exact_credential_name_scan_eligible(policy: RequestPolicy) -> bool:+ if policy.raw_secret_detected:+ return False+ if policy.clarification_reason in _PRE_RESOLUTION_CLARIFICATION_REASONS:+ return False+ if policy.credential_input_kind == "credential_name" and policy.credential_refs:+ return False+ return policy.credential_input_kind in ("none", "credential_name", "website_stored_credential")+++async def _apply_exact_credential_name_scope(policy: RequestPolicy,*,user_message: str,organization_id: str,) -> None:- if policy.classifier_status != "fallback":- return- if policy.raw_secret_detected or policy.credential_input_kind not in ("none", "credential_name"):+ if not _exact_credential_name_scan_eligible(policy):return- candidates = _fallback_credential_name_candidates(user_message)+ candidates = _exact_credential_name_candidates(user_message)if not candidates:returncredentials = await _load_credentials(organization_id)+ if (+ policy.credential_input_kind == "website_stored_credential"+ and policy.login_page_urls+ and _match_by_url(credentials, policy.login_page_urls)+ ):+ returnmatched_names = _clean_list([candidate for candidate in candidates if any(credential.name == candidate for credential in credentials)])@@ -2567,14 +2587,14 @@ async def build_request_policy(credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()}try:- await _apply_fallback_credential_name_scope(+ await _apply_exact_credential_name_scope(policy,user_message=user_message,organization_id=organization_id,)except Exception:LOG.warning(- "request-policy fallback credential-name extraction failed",+ "request-policy exact credential-name extraction failed",organization_id=organization_id,exc_info=True,)diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.pyindex 06417464c..e1d5e5e49 100644--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py@@ -441,3 +441,243 @@ async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_assert policy.user_response_policy == "ask_clarification"assert policy.allow_update_workflow is Falseassert policy.allow_run_blocks is False+++def _cred(name: str, credential_id: str, tested_url: str | None = None) -> SimpleNamespace:+ return SimpleNamespace(name=name, credential_id=credential_id, tested_url=tested_url)+++async def _build_with_forced_classifier(+ *,+ user_message: str,+ classifier_policy: RequestPolicy,+ org_credentials: list[SimpleNamespace],+ get_credentials: AsyncMock | None = None,+ get_credentials_by_ids: AsyncMock | None = None,+) -> RequestPolicy:+ load_mock = get_credentials or AsyncMock(return_value=org_credentials)+ by_ids_mock = get_credentials_by_ids or AsyncMock(return_value=[])+ with (+ patch(+ "skyvern.forge.sdk.copilot.request_policy._classify_request",+ new=AsyncMock(return_value=classifier_policy),+ ),+ patch("skyvern.forge.app.DATABASE.credentials.get_credentials", new=load_mock),+ patch("skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids", new=by_ids_mock),+ ):+ return await build_request_policy(+ user_message=user_message,+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="o_test",+ handler=None,+ )++[email protected]("kind", ["credential_name", "none"])+async def test_success_empty_refs_resolves_named_credential(kind: str) -> None:+ credential = _cred("mock-portal-login", "cred_login")+ policy = await _build_with_forced_classifier(+ user_message="Sign in with the saved credential named mock-portal-login.",+ classifier_policy=RequestPolicy(credential_input_kind=kind, classifier_status="success"),+ org_credentials=[credential, _cred("other", "cred_other")],+ )++ assert policy.classifier_status == "success"+ assert policy.credential_input_kind == "credential_name"+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]+ assert policy.clarification_reason != "credential_name_unresolved"+++async def test_success_website_kind_with_nonmatching_url_resolves_named_credential() -> None:+ credential = _cred("mock-portal-login", "cred_login", tested_url="https://saved.example.net/signin")+ policy = await _build_with_forced_classifier(+ user_message="Log in with the saved credential named mock-portal-login here.",+ classifier_policy=RequestPolicy(+ credential_input_kind="website_stored_credential",+ login_page_urls=["https://unrelated.example.com/login"],+ classifier_status="success",+ ),+ org_credentials=[credential],+ )++ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]+ assert policy.clarification_reason != "credential_name_unresolved"+ assert policy.clarification_question is None+++async def test_success_website_kind_with_matching_url_keeps_url_credential() -> None:+ url_credential = _cred("portal-cred", "cred_url", tested_url="https://portal.example.com/login")+ named_credential = _cred("other-login", "cred_named", tested_url="https://elsewhere.example.net/login")+ policy = await _build_with_forced_classifier(+ user_message="Log in with the saved credential named other-login.",+ classifier_policy=RequestPolicy(+ credential_input_kind="website_stored_credential",+ login_page_urls=["https://portal.example.com/login"],+ classifier_status="success",+ ),+ org_credentials=[url_credential, named_credential],+ )++ assert policy.credential_input_kind == "website_stored_credential"+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_url"]+++async def test_success_credential_id_intent_is_not_overridden_by_name_scan() -> None:+ by_ids = AsyncMock(return_value=[_cred("real", "cred_real")])+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])+ policy = await _build_with_forced_classifier(+ user_message="Use cred_real, the saved credential named mock-portal-login.",+ classifier_policy=RequestPolicy(+ credential_input_kind="credential_id",+ credential_refs=["cred_real"],+ classifier_status="success",+ ),+ org_credentials=[],+ get_credentials=load_mock,+ get_credentials_by_ids=by_ids,+ )++ assert policy.credential_input_kind == "credential_id"+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real"]+ load_mock.assert_not_awaited()+++async def test_success_populated_name_refs_are_not_narrowed_by_scan() -> None:+ policy = await _build_with_forced_classifier(+ user_message="Use the saved credential named alpha.",+ classifier_policy=RequestPolicy(+ credential_input_kind="credential_name",+ credential_refs=["alpha", "beta"],+ classifier_status="success",+ ),+ org_credentials=[_cred("alpha", "cred_alpha")],+ )++ assert policy.credential_refs == ["alpha", "beta"]+ assert policy.clarification_reason == "credential_name_unresolved"+++async def test_success_pre_resolution_clarification_is_preserved() -> None:+ policy = await _build_with_forced_classifier(+ user_message="Sign in with the saved credential named mock-portal-login.",+ classifier_policy=RequestPolicy(+ credential_input_kind="none",+ requires_user_clarification=True,+ clarification_reason="invalid_conditional_container",+ classifier_status="success",+ ),+ org_credentials=[_cred("mock-portal-login", "cred_login")],+ )++ assert policy.clarification_reason == "invalid_conditional_container"+ assert policy.resolved_credentials == []+++async def test_success_ambiguous_named_credentials_clarify_for_normal_request() -> None:+ policy = await _build_with_forced_classifier(+ user_message="Use the credential named alpha or the credential named beta to sign in.",+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),+ org_credentials=[_cred("alpha", "cred_alpha"), _cred("beta", "cred_beta")],+ )++ assert policy.requires_user_clarification is True+ assert policy.clarification_reason == "credential_name_unresolved"+ assert policy.resolved_credentials == []+++async def test_success_ambiguous_named_credentials_draft_unbound_for_explicit_draft() -> None:+ policy = await _build_with_forced_classifier(+ user_message=(+ "Build this as a code block using the saved credential named alpha "+ "or the saved credential named beta. Do not create a login block."+ ),+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),+ org_credentials=[_cred("alpha", "cred_alpha"), _cred("beta", "cred_beta")],+ )++ assert policy.resolved_credentials == []+ assert policy.allow_missing_credentials_in_draft is True+ assert policy.requires_user_clarification is False+++async def test_success_incidental_quote_out_of_credential_context_is_not_rewritten() -> None:+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])+ policy = await _build_with_forced_classifier(+ user_message='Set the page title to "mock-portal-login" before saving.',+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),+ org_credentials=[],+ get_credentials=load_mock,+ )++ assert policy.credential_input_kind == "none"+ assert policy.resolved_credentials == []+ load_mock.assert_not_awaited()+++async def test_success_quote_after_word_containing_login_substring_is_not_context() -> None:+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])+ policy = await _build_with_forced_classifier(+ user_message='Publish the blog in the section titled "mock-portal-login".',+ classifier_policy=RequestPolicy(credential_input_kind="none", classifier_status="success"),+ org_credentials=[],+ get_credentials=load_mock,+ )++ assert policy.credential_input_kind == "none"+ assert policy.resolved_credentials == []+ load_mock.assert_not_awaited()+++async def test_success_quoted_name_in_credential_context_resolves() -> None:+ policy = await _build_with_forced_classifier(+ user_message='Sign in using the saved credential "mock-portal-login".',+ classifier_policy=RequestPolicy(credential_input_kind="credential_name", classifier_status="success"),+ org_credentials=[_cred("mock-portal-login", "cred_login")],+ )++ assert policy.credential_input_kind == "credential_name"+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_login"]+++async def test_success_zero_name_match_falls_through_to_clarification() -> None:+ policy = await _build_with_forced_classifier(+ user_message="Sign in with the saved credential named ghost-cred.",+ classifier_policy=RequestPolicy(credential_input_kind="credential_name", classifier_status="success"),+ org_credentials=[_cred("mock-portal-login", "cred_login")],+ )++ assert policy.resolved_credentials == []+ assert policy.clarification_reason == "credential_name_unresolved"+++async def test_success_raw_secret_blocks_name_scan_before_loading_credentials() -> None:+ load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])+ policy = await _build_with_forced_classifier(+ user_message="Sign in with the saved credential named mock-portal-login.",+ classifier_policy=RequestPolicy(+ credential_input_kind="none",+ raw_secret_detected=True,+ raw_secret_handling="block",+ classifier_status="success",+ ),+ org_credentials=[],+ get_credentials=load_mock,+ )++ assert policy.resolved_credentials == []+ load_mock.assert_not_awaited()