Skip to content

Automate browser based workflows with AI

SKY-9933 Avoid repeated credential clarification loop (#6027)

Andrew Neilsoneec704c4fa85
diff --git a/docs/cloud/managing-credentials/totp-setup.mdx b/docs/cloud/managing-credentials/totp-setup.mdx
index 3d1cf31e5..8a2ecb947 100644
--- a/docs/cloud/managing-credentials/totp-setup.mdx
+++ b/docs/cloud/managing-credentials/totp-setup.mdx
@@ -173,7 +173,7 @@ This turns email-based 2FA into something nearly as automated as an authenticato
Instead of pushing codes to Skyvern, you can have Skyvern pull them from your server. Pass `totp_url` when running a [task](/cloud/getting-started/run-from-code) or [workflow](/cloud/building-workflows/build-a-workflow), and Skyvern will call your endpoint when it needs a verification code.
-Your endpoint must accept POST requests from Skyvern. For workflow runs, use the `workflow_run_id` in the signed request body to find the matching code, ignore any additional metadata fields, then return:
+Your endpoint must accept a POST request with `task_id`, `workflow_run_id`, and `workflow_permanent_id` in the body, and return:
```json
{
diff --git a/docs/developers/credentials/handle-2fa.mdx b/docs/developers/credentials/handle-2fa.mdx
index f406e6551..b64874e7b 100644
--- a/docs/developers/credentials/handle-2fa.mdx
+++ b/docs/developers/credentials/handle-2fa.mdx
@@ -239,30 +239,28 @@ Use a virtual phone number service like [Twilio](https://www.twilio.com/en-us/do
</Accordion>
</AccordionGroup>
-### Step 2: Start the workflow run with a TOTP identifier
+### Step 2: Start the run with a TOTP identifier
-When running the workflow that performs the login, include a `totp_identifier` that matches what you'll use when pushing codes:
+When running your login, include a `totp_identifier` that matches what you'll use when pushing codes:
<CodeGroup>
```python Python
-result = await client.run_workflow(
- workflow_id="wpid_abc123",
- parameters={
- "login_credential": "cred_xyz789",
- },
+result = await client.login(
+ credential_type="skyvern",
+ url="https://portal.example.com/login",
+ credential_id="cred_xyz789",
totp_identifier="[email protected]", # Must match when pushing code
)
```
```bash cURL
-curl -X POST "https://api.skyvern.com/v1/run/workflows" \
+curl -X POST "https://api.skyvern.com/v1/run/tasks/login" \
-H "x-api-key: $SKYVERN_API_KEY" \
-H "Content-Type: application/json" \
-d '{
- "workflow_id": "wpid_abc123",
- "parameters": {
- "login_credential": "cred_xyz789"
- },
+ "credential_type": "skyvern",
+ "url": "https://portal.example.com/login",
+ "credential_id": "cred_xyz789",
"totp_identifier": "[email protected]"
}'
```
@@ -316,7 +314,7 @@ Instead of pushing codes to Skyvern, you implement an endpoint that Skyvern poll
### Step 1: Implement the endpoint
-Your endpoint must accept POST requests and return the verification code. For workflow runs, use the `workflow_run_id` in the request body to find the matching code in your system, and ignore any additional metadata fields Skyvern includes:
+Your endpoint must accept POST requests and return the verification code:
**Request from Skyvern:**
```http
@@ -325,7 +323,7 @@ x-skyvern-signature: <hmac-sha256-signature>
Content-Type: application/json
{
- "workflow_run_id": "wr_123456"
+ "task_id": "tsk_123456"
}
```
@@ -357,30 +355,28 @@ def verify_skyvern_request(request, api_key: str) -> bool:
return hmac.compare_digest(signature, expected)
```
-### Step 3: Configure the workflow run
+### Step 3: Configure the login
-Pass `totp_url` when running the workflow that performs the login:
+Pass `totp_url` when running your login:
<CodeGroup>
```python Python
-result = await client.run_workflow(
- workflow_id="wpid_abc123",
- parameters={
- "login_credential": "cred_xyz789",
- },
+result = await client.login(
+ credential_type="skyvern",
+ url="https://portal.example.com/login",
+ credential_id="cred_xyz789",
totp_url="https://your-server.com/totp-webhook",
)
```
```bash cURL
-curl -X POST "https://api.skyvern.com/v1/run/workflows" \
+curl -X POST "https://api.skyvern.com/v1/run/tasks/login" \
-H "x-api-key: $SKYVERN_API_KEY" \
-H "Content-Type: application/json" \
-d '{
- "workflow_id": "wpid_abc123",
- "parameters": {
- "login_credential": "cred_xyz789"
- },
+ "credential_type": "skyvern",
+ "url": "https://portal.example.com/login",
+ "credential_id": "cred_xyz789",
"totp_url": "https://your-server.com/totp-webhook"
}'
```
diff --git a/docs/developers/credentials/troubleshooting-login.mdx b/docs/developers/credentials/troubleshooting-login.mdx
index 54ac99db4..37b25b35b 100644
--- a/docs/developers/credentials/troubleshooting-login.mdx
+++ b/docs/developers/credentials/troubleshooting-login.mdx
@@ -144,10 +144,11 @@ curl -X GET "https://api.skyvern.com/v1/credentials/totp?totp_identifier=user@ex
-H "x-api-key: $SKYVERN_API_KEY"
```
-4. **For pull-based 2FA**: Check your endpoint uses the incoming `workflow_run_id` to find the code and returns the correct format:
+4. **For pull-based 2FA**: Check your endpoint returns the correct format:
```json
{
+ "task_id": "tsk_123456",
"verification_code": "123456"
}
```
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 9a496f64e..b86e0437a 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -498,6 +498,15 @@ def _previous_credential_clarification_was_asked(global_llm_context: str) -> boo
)
+def _last_assistant_message_was_saved_credential_question(
+ chat_history: list[WorkflowCopilotChatHistoryMessage],
+) -> bool:
+ for message in reversed(chat_history):
+ if message.sender == WorkflowCopilotChatSender.AI:
+ return _SAVED_CREDENTIAL_NAME_QUESTION in message.content
+ return False
+
+
def _can_defer_unresolved_credential_name_for_draft(
policy: RequestPolicy,
*,
@@ -512,6 +521,19 @@ def _can_defer_unresolved_credential_name_for_draft(
return False
+def _should_defer_repeated_unresolved_credential_question(
+ policy: RequestPolicy,
+ *,
+ chat_history: list[WorkflowCopilotChatHistoryMessage],
+) -> bool:
+ return (
+ policy.credential_input_kind in ("none", "credential_name")
+ and policy.clarification_reason == "credential_name_unresolved"
+ and not _has_resolvable_credential_scope(policy)
+ and _last_assistant_message_was_saved_credential_question(chat_history)
+ )
+
+
async def _resolve_credentials(policy: RequestPolicy, organization_id: str) -> None:
if policy.credential_input_kind == "credential_id":
ids = _clean_list([ref for ref in policy.credential_refs if ref.startswith("cred_")])
@@ -534,7 +556,7 @@ async def _resolve_credentials(policy: RequestPolicy, organization_id: str) -> N
return
if policy.credential_input_kind == "credential_name" and not policy.credential_refs:
- if policy.testing_intent == "skip_test" and policy.allow_missing_credentials_in_draft:
+ if policy.allow_missing_credentials_in_draft:
policy.allow_run_blocks = False
return
_block(
@@ -734,6 +756,14 @@ async def build_request_policy(
policy.requires_user_clarification = False
policy.allow_missing_credentials_in_draft = True
+ if _should_defer_repeated_unresolved_credential_question(
+ policy,
+ chat_history=chat_history,
+ ):
+ policy.requires_user_clarification = False
+ policy.allow_run_blocks = False
+ policy.allow_missing_credentials_in_draft = True
+
if policy.raw_secret_detected:
_block(
policy,
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index cd4695152..90d8d2859 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -1384,6 +1384,10 @@ class TestRequestPolicyCredentialResolution:
prior_clarification_context = (
'{"decisions_made":["request-policy clarification required: credential_name/credential_name_unresolved"]}'
)
+ saved_credential_question = (
+ "Which saved credential should I use? "
+ "Please provide the exact credential name or a credential ID beginning with cred_."
+ )
history_refs_from_context = await build_request_policy(
user_message="Just draft a workflow without testing it.",
workflow_yaml="",
@@ -1427,6 +1431,75 @@ class TestRequestPolicyCredentialResolution:
assert not follow_up_missing_name_policy.allow_update_workflow
assert not follow_up_missing_name_policy.allow_run_blocks
+ handler.response = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": True,
+ "clarification_reason": "credential_name_unresolved",
+ }
+ prior_clarification_follow_up_policy = await build_request_policy(
+ user_message="let me help logging in",
+ workflow_yaml="",
+ chat_history=_history(
+ ("user", "log in via eduID"),
+ ("ai", saved_credential_question),
+ ),
+ global_llm_context=prior_clarification_context,
+ organization_id="org-1",
+ handler=handler,
+ )
+ assert prior_clarification_follow_up_policy.user_response_policy == "proceed"
+ assert prior_clarification_follow_up_policy.allow_update_workflow
+ assert not prior_clarification_follow_up_policy.allow_run_blocks
+ assert prior_clarification_follow_up_policy.allow_missing_credentials_in_draft
+ assert prior_clarification_follow_up_policy.clarification_question is None
+
+ handler.response = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "credential_name",
+ "requires_user_clarification": True,
+ "clarification_reason": "credential_name_unresolved",
+ }
+ prior_clarification_name_policy = await build_request_policy(
+ user_message="let me help logging in",
+ workflow_yaml="",
+ chat_history=_history(
+ ("user", "log in via eduID"),
+ ("ai", saved_credential_question),
+ ),
+ global_llm_context=prior_clarification_context,
+ organization_id="org-1",
+ handler=handler,
+ )
+ assert prior_clarification_name_policy.user_response_policy == "proceed"
+ assert prior_clarification_name_policy.allow_update_workflow
+ assert not prior_clarification_name_policy.allow_run_blocks
+ assert prior_clarification_name_policy.allow_missing_credentials_in_draft
+ assert prior_clarification_name_policy.clarification_question is None
+
+ handler.response = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": True,
+ "clarification_reason": "credential_name_unresolved",
+ }
+ stale_clarification_policy = await build_request_policy(
+ user_message="log into this other site",
+ workflow_yaml="",
+ chat_history=_history(
+ ("user", "log in via eduID"),
+ ("ai", saved_credential_question),
+ ("user", "never mind"),
+ ("ai", "Which page or URL should the workflow go to?"),
+ ),
+ global_llm_context=prior_clarification_context,
+ organization_id="org-1",
+ handler=handler,
+ )
+ assert stale_clarification_policy.user_response_policy == "ask_clarification"
+ assert not stale_clarification_policy.allow_update_workflow
+ assert not stale_clarification_policy.allow_run_blocks
+
handler.response = {
"testing_intent": "skip_test",
"credential_input_kind": "credential_name",