Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-10129): re-validate malformed credential IDs against the org (#6529)
Andrew Neilson2773821fb261
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 8e428ae1f..a10741450 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -77,6 +77,9 @@ _STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX = ()_STORED_CREDENTIAL_URL_QUESTION = f"{_STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX} {_CREDENTIALS_UI_DIRECTIONS}"_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")+# A credential ID typed with the wrong separator (`cred 530…`, `cred-530…`). The+# digit-only body and length floor keep this off prose like `cred and the password`.+_MALFORMED_CREDENTIAL_ID_RE = re.compile(r"\bcred[ \t\-]+([0-9]{12,})\b")_JINJA_TEMPLATE_VAR_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")_CREDENTIAL_PARAM_METADATA_FIELDS = frozenset({@@ -361,7 +364,15 @@ def _clean_list(values: list[Any]) -> list[str]:def _credential_ids(text: str) -> list[str]:- return list(dict.fromkeys(_CREDENTIAL_ID_RE.findall(text or "")))+ text = text or ""+ canonical = _CREDENTIAL_ID_RE.findall(text)+ normalized = [f"cred_{body}" for body in _MALFORMED_CREDENTIAL_ID_RE.findall(text)]+ return list(dict.fromkeys(canonical + normalized))+++def _canonicalize_credential_ref(ref: str) -> str:+ malformed = _MALFORMED_CREDENTIAL_ID_RE.fullmatch(ref.strip())+ return f"cred_{malformed.group(1)}" if malformed else refdef _raw_secret_detected(text: str) -> bool:@@ -604,7 +615,8 @@ async def _classify_request(policy = _classification_from_raw(raw)policy.completion_contract = _ground_completion_contract(user_message, policy.completion_contract)- policy.credential_refs = _clean_list(policy.credential_refs + ids)+ classifier_credential_refs = [_canonicalize_credential_ref(ref) for ref in policy.credential_refs]+ policy.credential_refs = _clean_list(classifier_credential_refs + ids)if raw_secret_present:policy.raw_secret_detected = Trueif policy.raw_secret_handling == "redacted_draft":@@ -617,8 +629,16 @@ async def _classify_request(policy.clarification_reason = structural_reason if structural_reason != "none" else "raw_secret"if policy.testing_intent == "skip_test" and policy.completion_contract:policy.testing_intent = "unspecified"- if ids and policy.credential_input_kind in ("none", "placeholder"):- policy.credential_input_kind = "credential_id"+ if ids and policy.credential_input_kind != "raw_secret":+ # A deterministically-extracted `cred_`-shaped token overrides a non-ID kind, unless the+ # classifier pointed at another resolvable target — a saved name (an ID-shaped ref does+ # not count) or a login-page URL — leaving the cred_ token as contextual.+ classifier_named_a_credential = any(not _credential_ids(ref) for ref in classifier_credential_refs)+ classifier_target_wins = (+ policy.credential_input_kind == "credential_name" and classifier_named_a_credential+ ) or (policy.credential_input_kind == "website_stored_credential" and bool(policy.login_page_urls))+ if not classifier_target_wins:+ policy.credential_input_kind = "credential_id"if (policy.credential_input_kind == "raw_secret"and not raw_secret_presentdiff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex bce793bac..b327c42f0 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -2949,6 +2949,166 @@ workflow_definition:assert agent_result.proposal_disposition == "review_untested"assert "without testing it, as requested" in agent_result.user_response+ @pytest.mark.asyncio+ async def test_malformed_then_corrected_credential_id_resolves_when_valid(self, monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ credential = SimpleNamespace(credential_id="cred_530299673029518520")+ get_credentials_by_ids = AsyncMock(return_value=[credential])+ monkeypatch.setattr(+ policy_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),+ )++ malformed = await build_request_policy(+ user_message="Build it with cred 530299673029518520.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=None,+ )+ assert malformed.credential_input_kind == "credential_id"+ assert malformed.resolved_credentials == [credential]+ assert malformed.invalid_credential_ids == []+ assert malformed.requires_user_clarification is False++ corrected = await build_request_policy(+ user_message="cred_530299673029518520",+ workflow_yaml="",+ chat_history=_history(+ ("user", "Build it with cred 530299673029518520."),+ ("ai", "The credential ID `530299673029518520` appears to be invalid."),+ ),+ global_llm_context="",+ organization_id="org-1",+ handler=None,+ )+ assert corrected.credential_input_kind == "credential_id"+ assert corrected.resolved_credentials == [credential]+ assert corrected.invalid_credential_ids == []+ assert corrected.requires_user_clarification is False++ @pytest.mark.asyncio+ async def test_malformed_then_corrected_credential_id_blocks_authoritatively_when_invalid(+ self, monkeypatch+ ) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ get_credentials_by_ids = AsyncMock(return_value=[])+ monkeypatch.setattr(+ policy_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),+ )++ for user_message, history in (+ ("Build it with cred 530299673029518520.", []),+ (+ "cred_530299673029518520",+ _history(+ ("user", "Build it with cred 530299673029518520."),+ ("ai", "The credential ID `530299673029518520` appears to be invalid."),+ ),+ ),+ ):+ policy = await build_request_policy(+ user_message=user_message,+ workflow_yaml="",+ chat_history=history,+ global_llm_context="",+ organization_id="org-1",+ handler=None,+ )+ assert policy.credential_input_kind == "credential_id"+ assert policy.invalid_credential_ids == ["cred_530299673029518520"]+ assert "not found in this organization" in (policy.clarification_question or "")+ assert "previously identified" not in (policy.clarification_question or "")++ @pytest.mark.asyncio+ async def test_malformed_id_promotes_over_non_id_kind_without_competing_scope(self, monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ credential = SimpleNamespace(credential_id="cred_530299673029518520")++ for classifier_body in (+ {"credential_input_kind": "credential_name", "credential_refs": []},+ {"credential_input_kind": "website_stored_credential", "login_page_urls": []},+ ):+ get_credentials_by_ids = AsyncMock(return_value=[credential])+ monkeypatch.setattr(+ policy_module.app,+ "DATABASE",+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),+ )++ async def handler(*, prompt: str, prompt_name: str, _body=classifier_body) -> dict:+ return {"testing_intent": "unspecified", **_body}++ policy = await build_request_policy(+ user_message="Build it with cred 530299673029518520.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=handler,+ )+ assert policy.credential_input_kind == "credential_id", classifier_body+ assert policy.resolved_credentials == [credential], classifier_body+ get_credentials_by_ids.assert_awaited_once_with(["cred_530299673029518520"], organization_id="org-1")++ @pytest.mark.asyncio+ async def test_contextual_credential_id_does_not_override_classifier_target(self, monkeypatch) -> None:+ from skyvern.forge.sdk.copilot import request_policy as policy_module+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy++ bank = SimpleNamespace(credential_id="cred_bank", name="Bank", tested_url="https://bank.example/login")+ monkeypatch.setattr(+ policy_module.app,+ "DATABASE",+ SimpleNamespace(+ credentials=SimpleNamespace(+ get_credentials=AsyncMock(return_value=[bank]),+ get_credentials_by_ids=AsyncMock(side_effect=AssertionError("contextual id must not be resolved")),+ )+ ),+ )++ async def name_handler(*, prompt: str, prompt_name: str) -> dict:+ return {"credential_input_kind": "credential_name", "credential_refs": ["Bank"]}++ name_policy = await build_request_policy(+ user_message="Replace cred_530299673029518520 with my saved credential named Bank.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=name_handler,+ )+ assert name_policy.credential_input_kind == "credential_name"+ assert name_policy.resolved_credentials == [bank]++ async def url_handler(*, prompt: str, prompt_name: str) -> dict:+ return {+ "credential_input_kind": "website_stored_credential",+ "login_page_urls": ["https://bank.example/login"],+ }++ url_policy = await build_request_policy(+ user_message="Use the saved login for https://bank.example/login instead of cred_530299673029518520.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="org-1",+ handler=url_handler,+ )+ assert url_policy.credential_input_kind == "website_stored_credential"+ assert url_policy.resolved_credentials == [bank]+class TestNativeToolCredentialIdValidation:def test_extracts_credential_ids_from_nested_tool_values(self) -> None:diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 8924d9017..1679a2795 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -10,6 +10,7 @@ from skyvern.forge.sdk.copilot.request_policy import (PROMPT_NAME,RAW_SECRET_REFUSAL_SENTINEL,_classify_request,+ _credential_ids,_raw_secret_detected,contains_email_password_pair,redact_raw_secrets_for_prompt,@@ -266,3 +267,60 @@ class TestRawSecretEvidenceContract:)assert policy.credential_input_kind == "credential_id"assert policy.credential_refs == ["cred_payroll_42"]+++class TestMalformedCredentialIdExtraction:+ def test_space_separator_is_normalized(self) -> None:+ assert _credential_ids("use cred 530299673029518520 to log in") == ["cred_530299673029518520"]++ def test_hyphen_separator_is_normalized(self) -> None:+ assert _credential_ids("cred-530299673029518520") == ["cred_530299673029518520"]++ def test_canonical_id_is_unchanged_and_not_double_counted(self) -> None:+ assert _credential_ids("cred_530299673029518520") == ["cred_530299673029518520"]++ def test_prose_after_cred_word_is_not_matched(self) -> None:+ assert _credential_ids("set up the cred and the password later") == []++ def test_short_number_is_not_matched(self) -> None:+ assert _credential_ids("cred 530299") == []++ @pytest.mark.asyncio+ async def test_classify_promotes_malformed_id_over_credential_name_without_competing_scope(self) -> None:+ policy = await _classify(+ "use cred 530299673029518520 for the login",+ kind="credential_name",+ reason="credential_name_unresolved",+ )+ assert policy.credential_input_kind == "credential_id"+ assert "cred_530299673029518520" in policy.credential_refs++ @pytest.mark.asyncio+ async def test_classify_promotes_malformed_id_over_website_stored_credential_without_url(self) -> None:+ policy = await _classify(+ "use cred 530299673029518520 for the login",+ kind="website_stored_credential",+ reason="none",+ )+ assert policy.credential_input_kind == "credential_id"+ assert "cred_530299673029518520" in policy.credential_refs++ @pytest.mark.asyncio+ async def test_classify_keeps_credential_name_when_classifier_surfaced_a_name(self) -> None:+ policy = await _classify(+ "replace cred_530299673029518520 with my saved credential named Bank",+ kind="credential_name",+ refs=["Bank"],+ reason="credential_name_unresolved",+ )+ assert policy.credential_input_kind == "credential_name"++ @pytest.mark.asyncio+ async def test_classify_promotes_when_classifier_mislabeled_ids_as_name(self) -> None:+ policy = await _classify(+ "use cred_alpha and cred_beta for the two logins",+ kind="credential_name",+ refs=["cred_alpha", "cred_beta"],+ reason="none",+ )+ assert policy.credential_input_kind == "credential_id"