Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: use the credential ID someone pasted, however they worded the message (SKY-13552) (#8076)
Andrew Neilsondfe02ce8a482
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex f8341e218..55fe72030 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -1005,11 +1005,11 @@ def _bound_approved_credential_ids(request_policy: RequestPolicy, workflow_yaml:def _allowed_unresolved_credential_ids(request_policy: RequestPolicy) -> set[str]:+ # invalid_credential_ids is negation-filtered and affirmative; credential_refs is not — a+ # model-carried or negated ref must not become draft-bindable (SKY-13552).if not request_policy.allow_missing_credentials_in_draft:return set()- ids = set(request_policy.invalid_credential_ids)- ids.update(ref for ref in request_policy.credential_refs if isinstance(ref, str) and ref.startswith("cred_"))- return ids+ return set(request_policy.invalid_credential_ids)def _existing_workflow_credential_ids(request_policy: RequestPolicy) -> set[str]:diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex d10a88ecc..3c9cfe63f 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -234,14 +234,6 @@ _POSTFIX_CREDENTIAL_TOKEN_RE = re.compile(r"\b(?:use|using|with)\s+(?:my\s+|the\s+)?(?:saved\s+)?([A-Za-z0-9_.@:-]{2,100})\s+credential\b",re.I,)-_EXPLICIT_CREDENTIAL_ID_CONTEXT_RE = re.compile(- r"\b(?:use|using|with|select|choose)\s+"- r"(?:(?:my|the|a)\s+)?(?:(?:saved|stored)\s+)?(?:credential(?:\s+id)?\s*)?"- r"(?::\s*)?$",- re.I,-)-_CREDENTIAL_ID_LABEL_CONTEXT_RE = re.compile(r"\bcredential\s+id\s*$", re.I)-_CREDENTIAL_ID_COORDINATOR_RE = re.compile(r"\s*,?\s*(?:and|or)\s*$", re.I)_CREDENTIAL_REPLACEMENT_TARGET_RES = (re.compile(r"\bswitch(?:ing)?\s+to\b"@@ -668,6 +660,9 @@ class RequestPolicy:"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,"credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly,"resolved_credential_count": len(self.resolved_credentials),+ "current_turn_named_credential_count": len(self.current_turn_named_credential_ids),+ "invalid_credential_id_count": len(self.invalid_credential_ids),+ "auto_bound_credential_count": len(self.auto_bound_credentials),"has_completion_contract": bool(self.completion_contract),"completion_criteria_count": len(self.graded_completion_criteria()),"completion_criteria_implicit_count": sum(@@ -4603,8 +4598,12 @@ def _classifier_ref_stated_in_message(policy: RequestPolicy, user_message: str)def _explicit_credential_ids(user_message: str) -> list[str]:+ """Credential IDs the user's own message hands to this turn.++ Accepts the canonical ``cred_`` form and ``_MALFORMED_CREDENTIAL_ID_RE``'s ``cred <12+ digits>``+ normalization; surrounding prose can only withdraw a token, never qualify it.+ """text = _credential_authority_text(user_message or "")- explicit: list[str] = []matches = sorted([(match, match.group(0)) for match in _CREDENTIAL_ID_RE.finditer(text)]+ [(match, f"cred_{match.group(1)}") for match in _MALFORMED_CREDENTIAL_ID_RE.finditer(text)],@@ -4615,27 +4614,15 @@ def _explicit_credential_ids(user_message: str) -> list[str]:for match, credential_id in matchesif _credential_reference_is_negated(text, match.start())}- last_explicit_end: int | None = None- for match, credential_id in matches:- if last_negated_mention.get(credential_id, -1) > match.start():- continue- if _credential_reference_is_negated(text, match.start()) or _credential_reference_is_unrelated_replacement(- text, match.start()- ):- continue- context = text[max(0, match.start() - 64) : match.start()]- if (- _EXPLICIT_CREDENTIAL_ID_CONTEXT_RE.search(context)- or _CREDENTIAL_ID_LABEL_CONTEXT_RE.search(context)- or match.group(0).strip() == text.strip()- or (- last_explicit_end is not None- and _CREDENTIAL_ID_COORDINATOR_RE.fullmatch(text[last_explicit_end : match.start()])- )- ):- explicit.append(credential_id)- last_explicit_end = match.end()- return _clean_list(explicit)+ return _clean_list(+ [+ credential_id+ for match, credential_id in matches+ if last_negated_mention.get(credential_id, -1) <= match.start()+ and not _credential_reference_is_negated(text, match.start())+ and not _credential_reference_is_unrelated_replacement(text, match.start())+ ]+ )def credential_candidate_label(credential: Credential) -> str:@@ -4999,23 +4986,29 @@ async def _resolve_credentials(# regardless of the classifier label. Do not trust stale/model-authored# credential_refs that are absent from the latest user message.ids = _explicit_credential_ids(user_message)+ explicit_name_candidates = _credential_name_candidates(user_message)if ids:existing = await app.DATABASE.credentials.get_credentials_by_ids(ids, organization_id=organization_id)found = {credential.credential_id for credential in existing}policy.resolved_credentials = existing_record_current_turn_named_credentials(policy, existing)policy.invalid_credential_ids = [credential_id for credential_id in ids if credential_id not in found]- if policy.invalid_credential_ids and policy.testing_intent != "skip_test":+ # An unresolvable stated ID asks rather than falling through to the inference tiers below,+ # but it never blocks another stated reference that did resolve.+ if not explicit_name_candidates:+ if existing or not policy.invalid_credential_ids:+ return+ if policy.testing_intent == "skip_test":+ policy.allow_run_blocks = False+ policy.allow_missing_credentials_in_draft = True+ returnformatted = ", ".join(f"`{credential_id}`" for credential_id in policy.invalid_credential_ids)_block(policy,f"The credential ID(s) {formatted} were not found in this organization. Please provide a valid saved credential ID or explicitly ask for an unvalidated draft that will not be run yet.",reason="credential_name_unresolved",)- elif policy.invalid_credential_ids:- policy.allow_run_blocks = False- policy.allow_missing_credentials_in_draft = True- return+ return# `login_intent` is a classifier-authored hint that misses on runs where the model does not set# it, so a message asking for a login in its own words must still count. A classifier that failed@@ -5026,7 +5019,7 @@ async def _resolve_credentials(or (policy.classifier_status != "fallback" and login_phrase_asserted)or _classifier_ref_stated_in_message(policy, user_message))- name_candidates = _credential_name_candidates(user_message)+ name_candidates = explicit_name_candidatescredentials: list[Credential] | None = None# Only a request that already asserts a login or credential intent may be matched against the# org's saved names; without that gate an unrelated message would scan the credential list.@@ -5040,7 +5033,7 @@ async def _resolve_credentials([credential for credential in credentials if credential.name in name_candidates])if len(named_matches) == 1:- policy.resolved_credentials = named_matches+ policy.resolved_credentials = _deduplicate_credentials(list(policy.resolved_credentials) + named_matches)_record_current_turn_named_credentials(policy, named_matches)if (policy.classifier_status == "fallback"@@ -5070,6 +5063,9 @@ async def _resolve_credentials(card_answerable=True,)return+ if policy.resolved_credentials:+ # A stated ID already resolved; an unmatched name rides along as a finding, not a wall.+ returnif policy.testing_intent == "skip_test" or defer_unresolved_credential_name:policy.allow_run_blocks, policy.allow_missing_credentials_in_draft = False, Trueif defer_unresolved_credential_name:diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex bf2e4027a..7d1d9032b 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -4231,7 +4231,7 @@ class TestRequestPolicyCredentialResolution:assert policy.completion_contract_status == "present"@pytest.mark.asyncio- async def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:+ async def test_missing_user_supplied_credential_id_rides_along_when_another_resolves(self, monkeypatch) -> None:from skyvern.forge.sdk.copilot import request_policy as policy_modulefrom skyvern.forge.sdk.copilot.request_policy import build_request_policy@@ -4254,14 +4254,12 @@ class TestRequestPolicyCredentialResolution:assert policy.credential_input_kind == "credential_id"assert policy.credential_refs == ["cred_valid", "cred_missing"]assert policy.invalid_credential_ids == ["cred_missing"]- assert policy.user_response_policy == "ask_clarification"- assert policy.allow_update_workflow is False- assert policy.allow_run_blocks is False- assert policy.allow_missing_credentials_in_draft is False- assert policy.clarification_question- assert "cred_missing" in policy.clarification_question- assert "not found in this organization" in policy.clarification_question- assert "unvalidated draft" in policy.clarification_question+ # A stated ID that resolved carries the turn; the unresolvable one rides along as a+ # recorded finding instead of walling the request (SKY-13552).+ assert [credential.credential_id for credential in policy.resolved_credentials] == ["cred_valid"]+ assert policy.user_response_policy != "ask_clarification"+ assert policy.allow_update_workflow is True+ assert policy.allow_run_blocks is Trueget_credentials_by_ids.assert_awaited_once_with(["cred_valid", "cred_missing"], organization_id="org-1")@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 940aabd4b..e7f8cddc3 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -1241,6 +1241,24 @@ def test_allows_explicit_unresolved_credential_id_for_untested_draft() -> None:assert verdict.allowed+def test_rejects_credential_ref_the_user_did_not_state_even_for_untested_draft() -> None:+ """A cred_ ref carried only in credential_refs (model-authored or negated) is not draft-bindable (SKY-13552)."""+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ credential_input_kind="credential_id",+ credential_refs=["cred_carried"],+ invalid_credential_ids=[],+ allow_missing_credentials_in_draft=True,+ allow_run_blocks=False,+ ),+ workflow_yaml=_workflow_yaml().replace("cred_safe", "cred_carried"),+ )++ assert not verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes++def test_rejects_unrequested_credential_id_even_when_untested_draft_allows_missing_credentials() -> None:verdict = evaluate_output_policy(request_policy=_policy(diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.pyindex 4a8502f21..b0fe8e922 100644--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py@@ -684,9 +684,11 @@ async def test_resolver_exact_name_is_independent_of_classifier_label(@pytest.mark.asyncio-async def test_explicit_id_wins_and_skips_name_and_url_scan() -> None:+async def test_explicit_id_and_explicit_name_resolve_together() -> None:+ """Both stated references resolve; neither preempts the other (SKY-13552)."""+ named = _cred("named-login", "cred_named")by_ids = AsyncMock(return_value=[_cred("id-login", "cred_explicit")])- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an ID hit"))+ load_mock = AsyncMock(return_value=[named])policy = RequestPolicy(credential_input_kind="credential_id",credential_refs=["cred_explicit", "named-login"],@@ -696,14 +698,14 @@ async def test_explicit_id_wins_and_skips_name_and_url_scan() -> None:await _resolve_direct(policy,user_message="Use credential ID cred_explicit and the saved credential named named-login.",- org_credentials=[],+ org_credentials=[named],get_credentials=load_mock,get_credentials_by_ids=by_ids,)- assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit"]+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit", "cred_named"]+ assert policy.requires_user_clarification is Falseby_ids.assert_awaited_once_with(["cred_explicit"], organization_id="o_test")- load_mock.assert_not_awaited()@pytest.mark.asyncio@@ -711,7 +713,6 @@ async def test_explicit_id_wins_for_non_id_label_and_competing_url_match() -> Noexplicit = _cred("id-login", "cred_explicit")competing = _cred("url-login", "cred_url", tested_url="https://portal.example.com/login")by_ids = AsyncMock(return_value=[explicit])- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an ID hit"))policy = RequestPolicy(credential_input_kind="website_stored_credential",credential_refs=["cred_explicit"],@@ -722,13 +723,12 @@ async def test_explicit_id_wins_for_non_id_label_and_competing_url_match() -> Nopolicy,user_message="Use credential ID cred_explicit for https://portal.example.com/login.",org_credentials=[competing],- get_credentials=load_mock,get_credentials_by_ids=by_ids,)assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_explicit"]+ assert policy.auto_bound_credentials == []by_ids.assert_awaited_once_with(["cred_explicit"], organization_id="o_test")- load_mock.assert_not_awaited()@pytest.mark.asyncio@@ -774,6 +774,7 @@ async def test_classifier_id_absent_from_current_message_does_not_resolve() -> N"Avoid using cred_prod.","Exclude credential ID cred_prod.","Do not use the old workflow because it still mentions credential ID cred_prod.",+ "Do not use the old workflow because it mentions cred_prod.","Do not use this credential, credential ID cred_prod.",],)@@ -801,12 +802,14 @@ async def test_negated_explicit_id_does_not_resolve(user_message: str) -> None:["Use this as a documentation example: cred_prod.","Use the staging workflow; the notes mention cred_prod.",- "Do not use the old workflow because it mentions cred_prod.",'Document the string "cred_prod" in the workflow notes.',+ "cred_prod\n\ncan you use this credential to log into the portal?",+ "Here's a credential you can use to test: cred_prod",],)-async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str) -> None:- by_ids = AsyncMock(side_effect=AssertionError("unrelated credential IDs must not be resolved"))+async def test_stated_explicit_id_resolves_regardless_of_prose_shape(user_message: str) -> None:+ """A cred_ ID the user typed is supplied, whatever surrounds it; only negation withdraws it (SKY-13552)."""+ by_ids = AsyncMock(return_value=[_cred("prod-login", "cred_prod")])policy = RequestPolicy(credential_input_kind="credential_id",credential_refs=["cred_prod"],@@ -819,8 +822,9 @@ async def test_unrelated_explicit_id_mention_does_not_resolve(user_message: str)get_credentials_by_ids=by_ids,)- assert policy.resolved_credentials == []- by_ids.assert_not_awaited()+ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_prod"]+ assert policy.requires_user_clarification is False+ by_ids.assert_awaited_once_with(["cred_prod"], organization_id="o_test")@pytest.mark.asyncio@@ -979,8 +983,8 @@ async def test_bare_id_appositive_negation_revokes_earlier_authority() -> None:@pytest.mark.asyncio-async def test_invalid_explicit_id_does_not_fall_through_to_name() -> None:- load_mock = AsyncMock(side_effect=AssertionError("organization scan should not run after an explicit ID"))+async def test_invalid_explicit_id_with_unknown_name_asks_instead_of_inferring() -> None:+ """Nothing stated resolved, so the turn asks; it never falls through to the inference tiers."""policy = RequestPolicy(credential_input_kind="credential_id",credential_refs=["cred_missing", "named-login"],@@ -990,14 +994,34 @@ async def test_invalid_explicit_id_does_not_fall_through_to_name() -> None:policy,user_message="Use credential ID cred_missing and the saved credential named named-login.",org_credentials=[],- get_credentials=load_mock,)assert policy.invalid_credential_ids == ["cred_missing"]+ assert policy.resolved_credentials == []assert policy.clarification_reason == "credential_name_unresolved"- assert policy.credential_ask_card_answerable is False- assert credential_pause_reason(SimpleNamespace(request_policy=policy)) is None- load_mock.assert_not_awaited()+ assert policy.requires_user_clarification is True+++async def test_invalid_explicit_id_rides_along_when_another_stated_reference_resolves() -> None:+ """An unresolvable stated ID never blocks a stated reference that did resolve (SKY-13552)."""+ by_ids = AsyncMock(return_value=[_cred("real-login", "cred_real")])+ policy = RequestPolicy(+ credential_input_kind="credential_id",+ credential_refs=["cred_real", "cred_missing"],+ )++ await _resolve_direct(+ policy,+ user_message="Use cred_real and cred_missing to log in.",+ org_credentials=[],+ get_credentials_by_ids=by_ids,+ )++ assert [candidate.credential_id for candidate in policy.resolved_credentials] == ["cred_real"]+ assert policy.invalid_credential_ids == ["cred_missing"]+ assert policy.requires_user_clarification is False+ assert policy.allow_run_blocks is True@pytest.mark.asyncio@@ -1553,7 +1577,7 @@ async def test_website_kind_explicit_name_wins_over_url_matched_card() -> None:@pytest.mark.asyncio-async def test_success_credential_id_intent_is_not_overridden_by_name_scan() -> None:+async def test_success_credential_id_and_stated_name_both_resolve() -> None:by_ids = AsyncMock(return_value=[_cred("real", "cred_real")])load_mock = AsyncMock(return_value=[_cred("mock-portal-login", "cred_login")])policy = await _build_with_forced_classifier(@@ -1563,14 +1587,13 @@ async def test_success_credential_id_intent_is_not_overridden_by_name_scan() ->credential_refs=["cred_real"],classifier_status="success",),- org_credentials=[],+ org_credentials=[_cred("mock-portal-login", "cred_login")],get_credentials=load_mock,get_credentials_by_ids=by_ids,)assert policy.credential_input_kind == "credential_id"- assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real"]- load_mock.assert_not_awaited()+ assert [c.credential_id for c in policy.resolved_credentials] == ["cred_real", "cred_login"]@pytest.mark.asyncio