Skip to content

Automate browser based workflows with AI

SKY-9719 Remove superseded Copilot intent gates (#5922)

Andrew Neilsonb9c25db630cf
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index ba004c48a..3ed9cb100 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -50,21 +50,6 @@ WORKFLOW_KNOWLEDGE_BASE_PATH = (
Path(__file__).resolve().parents[2] / "prompts" / "skyvern" / "workflow_knowledge_base.txt"
)
-_USER_SUPPLIED_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")
-_UNTESTED_DRAFT_REQUEST_RE = re.compile(
- r"\b(?:"
- r"without\s+testing|"
- r"without\s+(?:a\s+)?test(?:\s+run)?|"
- r"do\s+not\s+test|"
- r"don['’]?t\s+test|"
- r"skip\s+(?:the\s+)?test(?:ing)?|"
- r"no\s+test(?:ing)?|"
- r"untested\s+draft|"
- r"draft\s+only"
- r")\b",
- re.IGNORECASE,
-)
-
def _resolve_request_policy_handler(fallback_handler: Any) -> Any:
with contextlib.suppress(RuntimeError, AttributeError):
@@ -225,95 +210,6 @@ def _build_user_context(
)
-def _extract_user_supplied_credential_ids(user_message: str) -> list[str]:
- return list(dict.fromkeys(_USER_SUPPLIED_CREDENTIAL_ID_RE.findall(user_message or "")))
-
-
-def _user_requests_untested_workflow_draft(user_message: str) -> bool:
- return bool(_UNTESTED_DRAFT_REQUEST_RE.search(user_message or ""))
-
-
-def _missing_credential_ids_result(
- missing_credential_ids: list[str],
- global_llm_context: str | None,
-) -> AgentResult:
- formatted_ids = ", ".join(f"`{credential_id}`" for credential_id in missing_credential_ids)
- id_word = "ID" if len(missing_credential_ids) == 1 else "IDs"
- was_word = "was" if len(missing_credential_ids) == 1 else "were"
- structured = StructuredContext.from_json_str(global_llm_context)
- structured.decisions_made.append(
- f"credential reference validation blocked missing ids: {', '.join(missing_credential_ids)}"
- )
- return AgentResult(
- user_response=(
- f"The credential {id_word} {formatted_ids} {was_word} not found in this organization. "
- "Please choose one of these paths: provide/select a valid credential ID, create the credential "
- "in the Credentials UI and return with its ID, or explicitly tell me to continue with an "
- "unvalidated draft workflow that will not be run until credentials are available."
- ),
- updated_workflow=None,
- global_llm_context=structured.to_json_str(),
- response_type="ASK_QUESTION",
- workflow_yaml=None,
- workflow_was_persisted=False,
- clear_proposed_workflow=True,
- )
-
-
-def _credential_lookup_failed_result(global_llm_context: str | None) -> AgentResult:
- structured = StructuredContext.from_json_str(global_llm_context)
- structured.decisions_made.append("credential reference validation failed before workflow build")
- return AgentResult(
- user_response=(
- "I couldn't verify the supplied credential ID against this organization, so I can't build or run "
- "the workflow with it yet. Please provide/select a valid credential ID, create the credential in "
- "the Credentials UI and return with its ID, or explicitly tell me to continue with an unvalidated "
- "draft workflow that will not be run until credentials are available."
- ),
- updated_workflow=None,
- global_llm_context=structured.to_json_str(),
- response_type="ASK_QUESTION",
- workflow_yaml=None,
- workflow_was_persisted=False,
- clear_proposed_workflow=True,
- )
-
-
-async def _credential_validation_result_for_user_message(
- *,
- user_message: str,
- organization_id: str,
- global_llm_context: str | None,
- allow_untested_workflow_draft: bool = False,
-) -> AgentResult | None:
- if allow_untested_workflow_draft:
- return None
-
- credential_ids = _extract_user_supplied_credential_ids(user_message)
- if not credential_ids:
- return None
-
- try:
- existing_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
- credential_ids,
- organization_id=organization_id,
- )
- except Exception:
- LOG.warning(
- "Copilot failed to validate user-supplied credential IDs",
- organization_id=organization_id,
- credential_ids=credential_ids,
- exc_info=True,
- )
- return _credential_lookup_failed_result(global_llm_context)
-
- found_ids = {credential.credential_id for credential in existing_credentials}
- missing_ids = [credential_id for credential_id in credential_ids if credential_id not in found_ids]
- if not missing_ids:
- return None
- return _missing_credential_ids_result(missing_ids, global_llm_context)
-
-
def _truncate_summary_text(value: Any, max_chars: int = 240) -> str:
text = str(value)
if len(text) <= max_chars:
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 5a752507b..19fdfe80d 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -194,19 +194,11 @@ class TestVerifiedWorkflowOrNone:
assert _verified_workflow_or_none(ctx) == (None, None)
-class TestUntestedDraftRequest:
- def test_detects_explicit_skip_testing_request(self) -> None:
- from skyvern.forge.sdk.copilot.agent import _user_requests_untested_workflow_draft
-
- assert _user_requests_untested_workflow_draft("Can you just make a workflow without testing it?")
- assert _user_requests_untested_workflow_draft("Draft only please; do not test.")
- assert _user_requests_untested_workflow_draft("Skip testing and create the workflow.")
-
- def test_does_not_detect_routine_testing_mentions(self) -> None:
- from skyvern.forge.sdk.copilot.agent import _user_requests_untested_workflow_draft
-
- assert not _user_requests_untested_workflow_draft("Build and test this workflow.")
- assert not _user_requests_untested_workflow_draft("Test the login step before you finish.")
+class TestSupersededAgentIntentGates:
+ def test_agent_no_longer_owns_request_policy_classification(self) -> None:
+ assert not hasattr(agent_module, "_user_requests_untested_workflow_draft")
+ assert not hasattr(agent_module, "_extract_user_supplied_credential_ids")
+ assert not hasattr(agent_module, "_credential_validation_result_for_user_message")
class TestShouldRestorePersistedWorkflow:
@@ -891,84 +883,137 @@ class TestCredentialRefusalReachesAgent:
assert cross_ref.search(desc), f"{tool.name} missing refusal cross-reference"
-class TestUserSuppliedCredentialIdValidation:
- def test_extracts_unique_credential_ids_from_latest_message(self) -> None:
- ids = agent_module._extract_user_supplied_credential_ids(
- "Use cred_123456789, then cred_abc-DEF_42. Also repeat cred_123456789."
- )
-
- assert ids == ["cred_123456789", "cred_abc-DEF_42"]
-
- def test_ignores_non_credential_secret_text(self) -> None:
- ids = agent_module._extract_user_supplied_credential_ids(
- "My username is [email protected] and my password is hunter2."
- )
-
- assert ids == []
-
+class TestRequestPolicyCredentialResolution:
@pytest.mark.asyncio
- async def test_missing_user_supplied_credential_ids_return_ask_question(self, monkeypatch) -> None:
+ async def test_missing_user_supplied_credential_ids_ask_for_clarification(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
get_credentials_by_ids = AsyncMock(return_value=[SimpleNamespace(credential_id="cred_valid")])
monkeypatch.setattr(
- agent_module.app,
+ policy_module.app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
)
- result = await agent_module._credential_validation_result_for_user_message(
+ policy = await build_request_policy(
user_message="Please build it with cred_valid and cred_missing.",
+ workflow_yaml="",
+ chat_history="",
+ global_llm_context="",
organization_id="org-1",
- global_llm_context=None,
- )
-
- assert result is not None
- assert result.response_type == "ASK_QUESTION"
- assert result.updated_workflow is None
- assert result.workflow_yaml is None
- assert result.workflow_was_persisted is False
- assert result.clear_proposed_workflow is True
- assert "cred_missing" in result.user_response
- assert "not found in this organization" in result.user_response
- assert "unvalidated draft workflow" in result.user_response
+ handler=None,
+ )
+
+ assert policy.credential_input_kind == "credential_id"
+ assert policy.credential_refs == ["cred_valid", "cred_missing"]
+ assert policy.invalid_credential_ids == ["cred_missing"]
+ assert policy.user_response_policy == "ask_clarification"
+ assert policy.allow_update_workflow is False
+ assert policy.allow_run_blocks is False
+ assert policy.allow_missing_credentials_in_draft is False
+ assert policy.clarification_question
+ assert "cred_missing" in policy.clarification_question
+ assert "not found in this organization" in policy.clarification_question
+ assert "unvalidated draft" in policy.clarification_question
get_credentials_by_ids.assert_awaited_once_with(["cred_valid", "cred_missing"], organization_id="org-1")
@pytest.mark.asyncio
- async def test_missing_user_supplied_credential_ids_continue_for_explicit_untested_draft(self, monkeypatch) -> None:
+ async def test_skip_test_allows_missing_user_supplied_credential_ids_in_draft(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
get_credentials_by_ids = AsyncMock(return_value=[])
monkeypatch.setattr(
- agent_module.app,
+ policy_module.app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
)
- result = await agent_module._credential_validation_result_for_user_message(
+ async def handler(**kwargs):
+ return {
+ "testing_intent": "skip_test",
+ "credential_input_kind": "credential_id",
+ "credential_refs": ["cred_missing"],
+ }
+
+ policy = await build_request_policy(
user_message="Build an untested draft with cred_missing.",
+ workflow_yaml="",
+ chat_history="",
+ global_llm_context="",
organization_id="org-1",
- global_llm_context=None,
- allow_untested_workflow_draft=True,
+ handler=handler,
)
- assert result is None
- get_credentials_by_ids.assert_not_called()
+ assert policy.testing_intent == "skip_test"
+ assert policy.invalid_credential_ids == ["cred_missing"]
+ assert policy.user_response_policy == "proceed"
+ assert policy.allow_update_workflow is True
+ assert policy.allow_run_blocks is False
+ assert policy.allow_missing_credentials_in_draft is True
+ get_credentials_by_ids.assert_awaited_once_with(["cred_missing"], organization_id="org-1")
@pytest.mark.asyncio
async def test_valid_user_supplied_credential_ids_continue_normally(self, monkeypatch) -> None:
- get_credentials_by_ids = AsyncMock(return_value=[SimpleNamespace(credential_id="cred_valid")])
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ credential = SimpleNamespace(credential_id="cred_valid")
+ get_credentials_by_ids = AsyncMock(return_value=[credential])
monkeypatch.setattr(
- agent_module.app,
+ policy_module.app,
"DATABASE",
SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
)
- result = await agent_module._credential_validation_result_for_user_message(
+ policy = await build_request_policy(
user_message="Please build it with cred_valid.",
+ workflow_yaml="",
+ chat_history="",
+ global_llm_context="",
organization_id="org-1",
- global_llm_context=None,
+ handler=None,
)
- assert result is None
+ assert policy.credential_refs == ["cred_valid"]
+ assert policy.resolved_credentials == [credential]
+ assert policy.invalid_credential_ids == []
+ assert policy.user_response_policy == "proceed"
+ assert policy.allow_run_blocks is True
get_credentials_by_ids.assert_awaited_once_with(["cred_valid"], organization_id="org-1")
+ @pytest.mark.asyncio
+ async def test_raw_inline_secret_refuses_before_model_classification(self, monkeypatch) -> None:
+ from skyvern.forge.sdk.copilot import request_policy as policy_module
+ from skyvern.forge.sdk.copilot.request_policy import build_request_policy
+
+ get_credentials_by_ids = AsyncMock()
+ monkeypatch.setattr(
+ policy_module.app,
+ "DATABASE",
+ SimpleNamespace(credentials=SimpleNamespace(get_credentials_by_ids=get_credentials_by_ids)),
+ )
+ handler = AsyncMock()
+
+ policy = await build_request_policy(
+ user_message="Use username [email protected] and password=hunter2 to log in.",
+ workflow_yaml="",
+ chat_history="",
+ global_llm_context="",
+ organization_id="org-1",
+ handler=handler,
+ )
+
+ assert policy.raw_secret_detected is True
+ assert policy.credential_input_kind == "raw_secret"
+ assert policy.user_response_policy == "ask_clarification"
+ assert policy.allow_update_workflow is False
+ assert policy.allow_run_blocks is False
+ assert "DO NOT PROVIDE RAW LOGIN/PASSWORD" in (policy.clarification_question or "")
+ handler.assert_not_awaited()
+ get_credentials_by_ids.assert_not_called()
+
@pytest.mark.asyncio
async def test_request_policy_url_matching_and_skip_draft(self, monkeypatch) -> None:
from skyvern.forge.sdk.copilot import request_policy as policy_module
@@ -982,6 +1027,22 @@ class TestUserSuppliedCredentialIdValidation:
return handler.response
args = dict(workflow_yaml="", global_llm_context="", organization_id="org-1", handler=handler)
+ handler.response = {
+ "credential_input_kind": "credential_name",
+ "credential_refs": ["Bank"],
+ }
+ name_policy = await build_request_policy(user_message="use my saved Bank credential", chat_history="", **args)
+ assert name_policy.user_response_policy == "proceed"
+ assert name_policy.resolved_credentials == [credential]
+
+ handler.response = {
+ "credential_input_kind": "website_stored_credential",
+ "login_page_urls": ["https://bank.co.uk/login"],
+ }
+ site_policy = await build_request_policy(user_message="use the saved login", chat_history="", **args)
+ assert site_policy.user_response_policy == "proceed"
+ assert site_policy.resolved_credentials == [credential]
+
handler.response = {
"credential_input_kind": "website_stored_credential",
"login_page_urls": ["https://evil.co.uk/login"],