Skip to content

Automate browser based workflows with AI

feat(SKY-11988): add credentialPrompt signal to copilot narrative payload (#7200)

Celal Zamanoğlu9949fb0dedb7
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index f22d6da7d..9ff53af97 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -136,6 +136,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
build_request_policy,
+ credential_prompt_reason,
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason
@@ -1524,6 +1525,11 @@ def _make_agent_result(
payload_updates["proposalDisposition"] = proposal_disposition
if turn_outcome is not None and "responseKind" not in narrative_payload:
payload_updates["responseKind"] = turn_outcome.response_kind.value
+ if "credentialPrompt" not in narrative_payload:
+ policy = ctx.request_policy if ctx is not None else None
+ reason = credential_prompt_reason(policy, kwargs.get("user_response"))
+ if reason:
+ payload_updates["credentialPrompt"] = {"reason": reason}
if ctx is not None and "verifiedSuccess" not in narrative_payload:
payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx))
if ctx is not None and "outcomeAdjudication" not in narrative_payload:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 36f4566b6..ddf120e06 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -82,6 +82,8 @@ class TurnNarrativePayload(TypedDict):
verifiedSuccess: NotRequired[bool]
# Verdict-state summary from the turn's latest evaluated adjudication.
outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]
+ # {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.
+ credentialPrompt: NotRequired[dict[str, str]]
designStarted: bool
designEnded: bool
draft: NarrativeDraft | None
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 38daae9d7..6fee80510 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -78,9 +78,14 @@ ClarificationReason = Literal[
]
RawSecretHandling = Literal["none", "block", "redacted_draft"]
_VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason))
+# Gates guardrails.py's deferred-draft tool authority — narrower than the prompt set below.
CREDENTIAL_DEFERRED_DRAFT_REASONS: frozenset[ClarificationReason] = frozenset(
{"workflow_credential_inputs_unbound", "credential_name_unresolved"}
)
+# Broader: any reason credential_prompt_reason() should surface an add-credential CTA for.
+CREDENTIAL_PROMPT_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(
+ {"raw_secret", "credential_name_unresolved", "credential_invention_requested", "workflow_credential_inputs_unbound"}
+)
_PRE_RESOLUTION_CLARIFICATION_REASONS = {
"credential_invention_requested",
"ambiguous_loop_edit",
@@ -97,6 +102,9 @@ _REASONS_OVERRIDDEN_BY_CREDENTIAL_REFS = {
_CREDENTIALS_UI_DIRECTIONS = (
f"You can find or add saved credentials at {settings.SKYVERN_APP_URL.rstrip('/')}/credentials."
)
+# Matches any final reply containing these substrings, not just credential-blocking
+# ones; safe today because every such emitter routes through _CREDENTIALS_UI_DIRECTIONS.
+_CREDENTIAL_PROMPT_TEXT_MARKERS = ("/credentials", "credentials ui")
# Stable tail of every raw-secret refusal; transcript redaction keys off it, so all refusal emitters must keep it verbatim.
RAW_SECRET_REFUSAL_SENTINEL = "DO NOT PROVIDE RAW LOGIN/PASSWORD"
_RAW_SECRET_QUESTION = (
@@ -280,6 +288,10 @@ class RequestPolicy:
allow_update_workflow: bool = True
allow_run_blocks: bool = True
allow_missing_credentials_in_draft: bool = False
+ # Narrower than the flag above: True only when a credential-specific path (an explicit
+ # code-block credential draft, or a redacted raw secret) set it, not the generic
+ # skip_test fallthrough that fires for any untested draft regardless of credentials.
+ credential_draft_deferred_explicitly: bool = False
user_response_policy: str = "proceed"
completion_contract: str | None = None
completion_criteria: list[CompletionCriterion] = field(default_factory=list)
@@ -313,6 +325,7 @@ class RequestPolicy:
"allow_update_workflow": self.allow_update_workflow,
"allow_run_blocks": self.allow_run_blocks,
"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,
+ "credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly,
"resolved_credential_count": len(self.resolved_credentials),
"has_completion_contract": bool(self.completion_contract),
"completion_criteria_count": len(self.graded_completion_criteria()),
@@ -405,6 +418,21 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy
return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
+def credential_prompt_reason(policy: RequestPolicy | None, final_text: str | None) -> str | None:
+ # Typed clarification_reason wins, then the explicit-defer flag — narrowly, since
+ # allow_missing_credentials_in_draft alone also covers the generic skip_test
+ # fallthrough with no credential involvement — then a text marker.
+ if isinstance(policy, RequestPolicy):
+ if policy.clarification_reason in CREDENTIAL_PROMPT_CLARIFICATION_REASONS:
+ return policy.clarification_reason
+ if policy.credential_draft_deferred_explicitly:
+ return "credential_deferred_draft"
+ normalized = " ".join((final_text or "").lower().split())
+ if any(marker in normalized for marker in _CREDENTIAL_PROMPT_TEXT_MARKERS):
+ return "assistant_directed"
+ return None
+
+
def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:
return (
isinstance(criterion.expected_output_value, bool) or criterion.expected_output_shape == "goal_judgment_boolean"
@@ -1793,6 +1821,7 @@ def _apply_explicit_code_block_credential_draft_policy(policy: RequestPolicy, us
policy.allow_update_workflow = True
policy.allow_run_blocks = False
policy.allow_missing_credentials_in_draft = True
+ policy.credential_draft_deferred_explicitly = True
policy.requires_user_clarification = False
policy.user_response_policy = "proceed"
policy.clarification_reason = "none"
@@ -2485,6 +2514,7 @@ async def build_request_policy(
policy.allow_update_workflow = True
policy.allow_run_blocks = False
policy.allow_missing_credentials_in_draft = True
+ policy.credential_draft_deferred_explicitly = True
policy.clarification_reason = "none"
policy.clarification_question = None
diff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.py
index 9dcedbdc6..813a8c731 100644
--- a/tests/unit/test_copilot_make_agent_result_backfill.py
+++ b/tests/unit/test_copilot_make_agent_result_backfill.py
@@ -6,8 +6,10 @@ from __future__ import annotations
import pytest
-from skyvern.forge.sdk.copilot.agent import _make_agent_result
-from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_result
+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from tests.unit.copilot_test_helpers import make_copilot_ctx as _ctx
from tests.unit.copilot_test_helpers import make_verified_goal_contract
@@ -117,3 +119,44 @@ def test_backfill_tolerates_missing_payload() -> None:
def test_missing_payload_is_allowed_without_ctx() -> None:
result = _result(None, turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)
assert result.narrative_payload is None
+
+
+def test_backfill_adds_credential_prompt_for_typed_clarification_reason() -> None:
+ ctx = _ctx(request_policy=RequestPolicy(clarification_reason="credential_name_unresolved"))
+ result = _result(ctx, turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["credentialPrompt"] == {"reason": "credential_name_unresolved"}
+
+
+def test_blocker_override_path_adds_credential_prompt_from_request_policy() -> None:
+ ctx = _ctx(request_policy=RequestPolicy(clarification_reason="workflow_credential_inputs_unbound"))
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="authority_denied",
+ agent_steering_text="Reply to the user without updating the workflow.",
+ user_facing_reason="I couldn't find the required credentials for the existing workflow.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code="turn_intent_no_mutation_run_blocked",
+ blocked_tool="update_workflow",
+ )
+ pre_override = AgentResult(user_response="agent reply", updated_workflow=None, global_llm_context=None)
+
+ overridden = _finalize_result_with_blocker_override(ctx, pre_override)
+
+ assert overridden.narrative_payload is not None
+ assert overridden.narrative_payload["credentialPrompt"] == {"reason": "workflow_credential_inputs_unbound"}
+
+
+def test_backfill_adds_credential_prompt_from_text_marker_when_no_policy_signal() -> None:
+ result = _result(
+ _ctx(),
+ user_response="You can add one at https://app.skyvern.com/credentials.",
+ narrative_payload=_payload(),
+ )
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["credentialPrompt"] == {"reason": "assistant_directed"}
+
+
+def test_backfill_omits_credential_prompt_when_no_signal_present() -> None:
+ result = _result(_ctx(), user_response="Done, the workflow is ready.", narrative_payload=_payload())
+ assert result.narrative_payload is not None
+ assert "credentialPrompt" not in result.narrative_payload
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 8a2deba98..06417464c 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -8,7 +8,14 @@ import pytest
from skyvern.config import settings
from skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext
-from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policy
+from skyvern.forge.sdk.copilot.request_policy import (
+ CREDENTIAL_DEFERRED_DRAFT_REASONS,
+ CREDENTIAL_PROMPT_CLARIFICATION_REASONS,
+ RequestPolicy,
+ _workflow_credential_inputs_unbound,
+ build_request_policy,
+ credential_prompt_reason,
+)
def _yaml(body: str) -> str:
@@ -339,6 +346,81 @@ async def test_fallback_code_block_generic_one_time_code_does_not_skip_run() ->
assert policy.testing_intent == "unspecified"
+def test_credential_prompt_clarification_reasons_membership() -> None:
+ assert CREDENTIAL_PROMPT_CLARIFICATION_REASONS == {
+ "raw_secret",
+ "credential_name_unresolved",
+ "credential_invention_requested",
+ "workflow_credential_inputs_unbound",
+ }
+
+
+def test_credential_deferred_draft_reasons_is_narrower_than_prompt_reasons() -> None:
+ assert CREDENTIAL_DEFERRED_DRAFT_REASONS < CREDENTIAL_PROMPT_CLARIFICATION_REASONS
+
+
+def test_credential_prompt_reason_typed_reason_wins_over_deferred_draft_flag() -> None:
+ policy = RequestPolicy(clarification_reason="credential_name_unresolved", allow_missing_credentials_in_draft=True)
+ assert credential_prompt_reason(policy, None) == "credential_name_unresolved"
+
+
+def test_credential_prompt_reason_deferred_draft_when_reason_cleared_but_flag_set() -> None:
+ # Mirrors the explicit-defer path (_apply_explicit_code_block_credential_draft_policy),
+ # which clears clarification_reason to "none" while leaving both flags set.
+ policy = RequestPolicy(
+ clarification_reason="none",
+ allow_missing_credentials_in_draft=True,
+ credential_draft_deferred_explicitly=True,
+ )
+ assert credential_prompt_reason(policy, None) == "credential_deferred_draft"
+
+
+def test_credential_prompt_reason_ignores_generic_skip_test_with_no_credential_signal() -> None:
+ # allow_missing_credentials_in_draft alone also fires for the generic skip_test
+ # fallthrough (any "draft only, don't test" turn), independent of credentials;
+ # only credential_draft_deferred_explicitly means a credential was really deferred.
+ policy = RequestPolicy(
+ testing_intent="skip_test",
+ clarification_reason="none",
+ allow_missing_credentials_in_draft=True,
+ credential_draft_deferred_explicitly=False,
+ )
+ assert credential_prompt_reason(policy, None) is None
+
+
+async def test_explicit_code_block_login_block_ban_surfaces_credential_prompt_end_to_end() -> None:
+ policy = await build_request_policy(
+ user_message="Write this as a code block. Do not create a login block for this part.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="o_test",
+ handler=None,
+ )
+ assert policy.credential_input_kind == "none"
+ assert policy.testing_intent == "skip_test"
+ assert credential_prompt_reason(policy, None) == "credential_deferred_draft"
+
+
+ "text",
+ [
+ "Add it at https://app.skyvern.com/credentials.",
+ "ADD IT AT HTTPS://APP.SKYVERN.COM/CREDENTIALS.",
+ "Store it in the Credentials UI first.",
+ "store it in the CREDENTIALS UI first.",
+ ],
+)
+def test_credential_prompt_reason_marker_fallback_is_case_insensitive(text: str) -> None:
+ assert credential_prompt_reason(None, text) == "assistant_directed"
+
+
+def test_credential_prompt_reason_policy_none_is_safe() -> None:
+ assert credential_prompt_reason(None, None) is None
+ assert credential_prompt_reason(None, "Everything is set, no action needed.") is None
+
+
@pytest.mark.asyncio
async def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_log_only(
monkeypatch: pytest.MonkeyPatch,