Skip to content

Automate browser based workflows with AI

fix(SKY-10129): re-validate malformed credential IDs against the org (#6529)

Andrew Neilson2773821fb261
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 8e428ae1f..a10741450 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -77,6 +77,9 @@ _STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX = (
)
_STORED_CREDENTIAL_URL_QUESTION = f"{_STORED_CREDENTIAL_URL_QUESTION_STABLE_PREFIX} {_CREDENTIALS_UI_DIRECTIONS}"
_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
+# A credential ID typed with the wrong separator (`cred 530…`, `cred-530…`). The
+# digit-only body and length floor keep this off prose like `cred and the password`.
+_MALFORMED_CREDENTIAL_ID_RE = re.compile(r"\bcred[ \t\-]+([0-9]{12,})\b")
_JINJA_TEMPLATE_VAR_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")
_CREDENTIAL_PARAM_METADATA_FIELDS = frozenset(
{
@@ -361,7 +364,15 @@ def _clean_list(values: list[Any]) -> list[str]:
def _credential_ids(text: str) -> list[str]:
- return list(dict.fromkeys(_CREDENTIAL_ID_RE.findall(text or "")))
+ text = text or ""
+ canonical = _CREDENTIAL_ID_RE.findall(text)
+ normalized = [f"cred_{body}" for body in _MALFORMED_CREDENTIAL_ID_RE.findall(text)]
+ return list(dict.fromkeys(canonical + normalized))
+
+
+def _canonicalize_credential_ref(ref: str) -> str:
+ malformed = _MALFORMED_CREDENTIAL_ID_RE.fullmatch(ref.strip())
+ return f"cred_{malformed.group(1)}" if malformed else ref
def _raw_secret_detected(text: str) -> bool:
@@ -604,7 +615,8 @@ async def _classify_request(
policy = _classification_from_raw(raw)
policy.completion_contract = _ground_completion_contract(user_message, policy.completion_contract)
- policy.credential_refs = _clean_list(policy.credential_refs + ids)
+ classifier_credential_refs = [_canonicalize_credential_ref(ref) for ref in policy.credential_refs]
+ policy.credential_refs = _clean_list(classifier_credential_refs + ids)
if raw_secret_present:
policy.raw_secret_detected = True
if policy.raw_secret_handling == "redacted_draft":
@@ -617,8 +629,16 @@ async def _classify_request(
policy.clarification_reason = structural_reason if structural_reason != "none" else "raw_secret"
if policy.testing_intent == "skip_test" and policy.completion_contract:
policy.testing_intent = "unspecified"
- if ids and policy.credential_input_kind in ("none", "placeholder"):
- policy.credential_input_kind = "credential_id"
+ if ids and policy.credential_input_kind != "raw_secret":
+ # A deterministically-extracted `cred_`-shaped token overrides a non-ID kind, unless the
+ # classifier pointed at another resolvable target — a saved name (an ID-shaped ref does
+ # not count) or a login-page URL — leaving the cred_ token as contextual.
+ classifier_named_a_credential = any(not _credential_ids(ref) for ref in classifier_credential_refs)
+ classifier_target_wins = (
+ policy.credential_input_kind == "credential_name" and classifier_named_a_credential
+ ) or (policy.credential_input_kind == "website_stored_credential" and bool(policy.login_page_urls))
+ if not classifier_target_wins:
+ policy.credential_input_kind = "credential_id"
if (
policy.credential_input_kind == "raw_secret"
and not raw_secret_present
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index bce793bac..b327c42f0 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -2949,6 +2949,166 @@ workflow_definition:
assert agent_result.proposal_disposition == "review_untested"
assert "without testing it, as requested" in agent_result.user_response
+ @pytest.mark.asyncio
+ async def test_malformed_then_corrected_credential_id_resolves_when_valid(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ credential = SimpleNamespace(credential_id="cred_530299673029518520")
+ get_credentials_by_ids = AsyncMock(return_value=[credential])
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
+ )
+
+ malformed = await build_request_policy(
+ user_message="Build it with cred 530299673029518520.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=None,
+ )
+ assert malformed.credential_input_kind == "credential_id"
+ assert malformed.resolved_credentials == [credential]
+ assert malformed.invalid_credential_ids == []
+ assert malformed.requires_user_clarification is False
+
+ corrected = await build_request_policy(
+ user_message="cred_530299673029518520",
+ workflow_yaml="",
+ chat_history=_history(
+ ("user", "Build it with cred 530299673029518520."),
+ ("ai", "The credential ID `530299673029518520` appears to be invalid."),
+ ),
+ global_llm_context="",
+ organization_id="org-1",
+ handler=None,
+ )
+ assert corrected.credential_input_kind == "credential_id"
+ assert corrected.resolved_credentials == [credential]
+ assert corrected.invalid_credential_ids == []
+ assert corrected.requires_user_clarification is False
+
+ @pytest.mark.asyncio
+ async def test_malformed_then_corrected_credential_id_blocks_authoritatively_when_invalid(
+ self, monkeypatch
+ ) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ get_credentials_by_ids = AsyncMock(return_value=[])
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
+ )
+
+ for user_message, history in (
+ ("Build it with cred 530299673029518520.", []),
+ (
+ "cred_530299673029518520",
+ _history(
+ ("user", "Build it with cred 530299673029518520."),
+ ("ai", "The credential ID `530299673029518520` appears to be invalid."),
+ ),
+ ),
+ ):
+ policy = await build_request_policy(
+ user_message=user_message,
+ workflow_yaml="",
+ chat_history=history,
+ global_llm_context="",
+ organization_id="org-1",
+ handler=None,
+ )
+ assert policy.credential_input_kind == "credential_id"
+ assert policy.invalid_credential_ids == ["cred_530299673029518520"]
+ assert "not found in this organization" in (policy.clarification_question or "")
+ assert "previously identified" not in (policy.clarification_question or "")
+
+ @pytest.mark.asyncio
+ async def test_malformed_id_promotes_over_non_id_kind_without_competing_scope(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ credential = SimpleNamespace(credential_id="cred_530299673029518520")
+
+ for classifier_body in (
+ {"credential_input_kind": "credential_name", "credential_refs": []},
+ {"credential_input_kind": "website_stored_credential", "login_page_urls": []},
+ ):
+ get_credentials_by_ids = AsyncMock(return_value=[credential])
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
+ )
+
+ async def handler(*, prompt: str, prompt_name: str, _body=classifier_body) -> dict:
+ return {"testing_intent": "unspecified", **_body}
+
+ policy = await build_request_policy(
+ user_message="Build it with cred 530299673029518520.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=handler,
+ )
+ assert policy.credential_input_kind == "credential_id", classifier_body
+ assert policy.resolved_credentials == [credential], classifier_body
+ get_credentials_by_ids.assert_awaited_once_with(["cred_530299673029518520"], organization_id="org-1")
+
+ @pytest.mark.asyncio
+ async def test_contextual_credential_id_does_not_override_classifier_target(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ bank = SimpleNamespace(credential_id="cred_bank", name="Bank", tested_url="https://bank.example/login")
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(
+ credentials=SimpleNamespace(
+ get_credentials=AsyncMock(return_value=[bank]),
+ get_credentials_by_ids=AsyncMock(side_effect=AssertionError("contextual id must not be resolved")),
+ )
+ ),
+ )
+
+ async def name_handler(*, prompt: str, prompt_name: str) -> dict:
+ return {"credential_input_kind": "credential_name", "credential_refs": ["Bank"]}
+
+ name_policy = await build_request_policy(
+ user_message="Replace cred_530299673029518520 with my saved credential named Bank.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=name_handler,
+ )
+ assert name_policy.credential_input_kind == "credential_name"
+ assert name_policy.resolved_credentials == [bank]
+
+ async def url_handler(*, prompt: str, prompt_name: str) -> dict:
+ return {
+ "credential_input_kind": "website_stored_credential",
+ "login_page_urls": ["https://bank.example/login"],
+ }
+
+ url_policy = await build_request_policy(
+ user_message="Use the saved login for https://bank.example/login instead of cred_530299673029518520.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="org-1",
+ handler=url_handler,
+ )
+ assert url_policy.credential_input_kind == "website_stored_credential"
+ assert url_policy.resolved_credentials == [bank]
+
class TestNativeToolCredentialIdValidation:
def test_extracts_credential_ids_from_nested_tool_values(self) -> None:
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 8924d9017..1679a2795 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -10,6 +10,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
PROMPT_NAME,
RAW_SECRET_REFUSAL_SENTINEL,
_classify_request,
+ _credential_ids,
_raw_secret_detected,
contains_email_password_pair,
redact_raw_secrets_for_prompt,
@@ -266,3 +267,60 @@ class TestRawSecretEvidenceContract:
)
assert policy.credential_input_kind == "credential_id"
assert policy.credential_refs == ["cred_payroll_42"]
+
+
+class TestMalformedCredentialIdExtraction:
+ def test_space_separator_is_normalized(self) -> None:
+ assert _credential_ids("use cred 530299673029518520 to log in") == ["cred_530299673029518520"]
+
+ def test_hyphen_separator_is_normalized(self) -> None:
+ assert _credential_ids("cred-530299673029518520") == ["cred_530299673029518520"]
+
+ def test_canonical_id_is_unchanged_and_not_double_counted(self) -> None:
+ assert _credential_ids("cred_530299673029518520") == ["cred_530299673029518520"]
+
+ def test_prose_after_cred_word_is_not_matched(self) -> None:
+ assert _credential_ids("set up the cred and the password later") == []
+
+ def test_short_number_is_not_matched(self) -> None:
+ assert _credential_ids("cred 530299") == []
+
+ @pytest.mark.asyncio
+ async def test_classify_promotes_malformed_id_over_credential_name_without_competing_scope(self) -> None:
+ policy = await _classify(
+ "use cred 530299673029518520 for the login",
+ kind="credential_name",
+ reason="credential_name_unresolved",
+ )
+ assert policy.credential_input_kind == "credential_id"
+ assert "cred_530299673029518520" in policy.credential_refs
+
+ @pytest.mark.asyncio
+ async def test_classify_promotes_malformed_id_over_website_stored_credential_without_url(self) -> None:
+ policy = await _classify(
+ "use cred 530299673029518520 for the login",
+ kind="website_stored_credential",
+ reason="none",
+ )
+ assert policy.credential_input_kind == "credential_id"
+ assert "cred_530299673029518520" in policy.credential_refs
+
+ @pytest.mark.asyncio
+ async def test_classify_keeps_credential_name_when_classifier_surfaced_a_name(self) -> None:
+ policy = await _classify(
+ "replace cred_530299673029518520 with my saved credential named Bank",
+ kind="credential_name",
+ refs=["Bank"],
+ reason="credential_name_unresolved",
+ )
+ assert policy.credential_input_kind == "credential_name"
+
+ @pytest.mark.asyncio
+ async def test_classify_promotes_when_classifier_mislabeled_ids_as_name(self) -> None:
+ policy = await _classify(
+ "use cred_alpha and cred_beta for the two logins",
+ kind="credential_name",
+ refs=["cred_alpha", "cred_beta"],
+ reason="none",
+ )
+ assert policy.credential_input_kind == "credential_id"