Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11988): add credentialPrompt signal to copilot narrative payload (#7200)
Celal Zamanoğlu9949fb0dedb7
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex f22d6da7d..9ff53af97 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -136,6 +136,7 @@ from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,RequestPolicy,build_request_policy,+ credential_prompt_reason,redact_raw_secrets_for_prompt,)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, run_outcome_display_reason@@ -1524,6 +1525,11 @@ def _make_agent_result(payload_updates["proposalDisposition"] = proposal_dispositionif turn_outcome is not None and "responseKind" not in narrative_payload:payload_updates["responseKind"] = turn_outcome.response_kind.value+ if "credentialPrompt" not in narrative_payload:+ policy = ctx.request_policy if ctx is not None else None+ reason = credential_prompt_reason(policy, kwargs.get("user_response"))+ if reason:+ payload_updates["credentialPrompt"] = {"reason": reason}if ctx is not None and "verifiedSuccess" not in narrative_payload:payload_updates["verifiedSuccess"] = bool(verified_goal_claim_authorized(ctx))if ctx is not None and "outcomeAdjudication" not in narrative_payload:diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 36f4566b6..ddf120e06 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -82,6 +82,8 @@ class TurnNarrativePayload(TypedDict):verifiedSuccess: NotRequired[bool]# Verdict-state summary from the turn's latest evaluated adjudication.outcomeAdjudication: NotRequired[NarrativeOutcomeAdjudication]+ # {"reason": <credential_prompt_reason() token>}, set when this turn surfaces a credential need.+ credentialPrompt: NotRequired[dict[str, str]]designStarted: booldesignEnded: booldraft: NarrativeDraft | Nonediff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 38daae9d7..6fee80510 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -78,9 +78,14 @@ ClarificationReason = Literal[]RawSecretHandling = Literal["none", "block", "redacted_draft"]_VALID_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(get_args(ClarificationReason))+# Gates guardrails.py's deferred-draft tool authority — narrower than the prompt set below.CREDENTIAL_DEFERRED_DRAFT_REASONS: frozenset[ClarificationReason] = frozenset({"workflow_credential_inputs_unbound", "credential_name_unresolved"})+# Broader: any reason credential_prompt_reason() should surface an add-credential CTA for.+CREDENTIAL_PROMPT_CLARIFICATION_REASONS: frozenset[ClarificationReason] = frozenset(+ {"raw_secret", "credential_name_unresolved", "credential_invention_requested", "workflow_credential_inputs_unbound"}+)_PRE_RESOLUTION_CLARIFICATION_REASONS = {"credential_invention_requested","ambiguous_loop_edit",@@ -97,6 +102,9 @@ _REASONS_OVERRIDDEN_BY_CREDENTIAL_REFS = {_CREDENTIALS_UI_DIRECTIONS = (f"You can find or add saved credentials at {settings.SKYVERN_APP_URL.rstrip('/')}/credentials.")+# Matches any final reply containing these substrings, not just credential-blocking+# ones; safe today because every such emitter routes through _CREDENTIALS_UI_DIRECTIONS.+_CREDENTIAL_PROMPT_TEXT_MARKERS = ("/credentials", "credentials ui")# Stable tail of every raw-secret refusal; transcript redaction keys off it, so all refusal emitters must keep it verbatim.RAW_SECRET_REFUSAL_SENTINEL = "DO NOT PROVIDE RAW LOGIN/PASSWORD"_RAW_SECRET_QUESTION = (@@ -280,6 +288,10 @@ class RequestPolicy:allow_update_workflow: bool = Trueallow_run_blocks: bool = Trueallow_missing_credentials_in_draft: bool = False+ # Narrower than the flag above: True only when a credential-specific path (an explicit+ # code-block credential draft, or a redacted raw secret) set it, not the generic+ # skip_test fallthrough that fires for any untested draft regardless of credentials.+ credential_draft_deferred_explicitly: bool = Falseuser_response_policy: str = "proceed"completion_contract: str | None = Nonecompletion_criteria: list[CompletionCriterion] = field(default_factory=list)@@ -313,6 +325,7 @@ class RequestPolicy:"allow_update_workflow": self.allow_update_workflow,"allow_run_blocks": self.allow_run_blocks,"allow_missing_credentials_in_draft": self.allow_missing_credentials_in_draft,+ "credential_draft_deferred_explicitly": self.credential_draft_deferred_explicitly,"resolved_credential_count": len(self.resolved_credentials),"has_completion_contract": bool(self.completion_contract),"completion_criteria_count": len(self.graded_completion_criteria()),@@ -405,6 +418,21 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicyreturn request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)+def credential_prompt_reason(policy: RequestPolicy | None, final_text: str | None) -> str | None:+ # Typed clarification_reason wins, then the explicit-defer flag — narrowly, since+ # allow_missing_credentials_in_draft alone also covers the generic skip_test+ # fallthrough with no credential involvement — then a text marker.+ if isinstance(policy, RequestPolicy):+ if policy.clarification_reason in CREDENTIAL_PROMPT_CLARIFICATION_REASONS:+ return policy.clarification_reason+ if policy.credential_draft_deferred_explicitly:+ return "credential_deferred_draft"+ normalized = " ".join((final_text or "").lower().split())+ if any(marker in normalized for marker in _CREDENTIAL_PROMPT_TEXT_MARKERS):+ return "assistant_directed"+ return None++def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:return (isinstance(criterion.expected_output_value, bool) or criterion.expected_output_shape == "goal_judgment_boolean"@@ -1793,6 +1821,7 @@ def _apply_explicit_code_block_credential_draft_policy(policy: RequestPolicy, uspolicy.allow_update_workflow = Truepolicy.allow_run_blocks = Falsepolicy.allow_missing_credentials_in_draft = True+ policy.credential_draft_deferred_explicitly = Truepolicy.requires_user_clarification = Falsepolicy.user_response_policy = "proceed"policy.clarification_reason = "none"@@ -2485,6 +2514,7 @@ async def build_request_policy(policy.allow_update_workflow = Truepolicy.allow_run_blocks = Falsepolicy.allow_missing_credentials_in_draft = True+ policy.credential_draft_deferred_explicitly = Truepolicy.clarification_reason = "none"policy.clarification_question = Nonediff --git a/tests/unit/test_copilot_make_agent_result_backfill.py b/tests/unit/test_copilot_make_agent_result_backfill.pyindex 9dcedbdc6..813a8c731 100644--- a/tests/unit/test_copilot_make_agent_result_backfill.py+++ b/tests/unit/test_copilot_make_agent_result_backfill.py@@ -6,8 +6,10 @@ from __future__ import annotationsimport pytest-from skyvern.forge.sdk.copilot.agent import _make_agent_result-from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.agent import _finalize_result_with_blocker_override, _make_agent_result+from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal+from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext+from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcomefrom tests.unit.copilot_test_helpers import make_copilot_ctx as _ctxfrom tests.unit.copilot_test_helpers import make_verified_goal_contract@@ -117,3 +119,44 @@ def test_backfill_tolerates_missing_payload() -> None:def test_missing_payload_is_allowed_without_ctx() -> None:result = _result(None, turn_outcome=_outcome(ResponseKind.BUILD), narrative_payload=None)assert result.narrative_payload is None+++def test_backfill_adds_credential_prompt_for_typed_clarification_reason() -> None:+ ctx = _ctx(request_policy=RequestPolicy(clarification_reason="credential_name_unresolved"))+ result = _result(ctx, turn_outcome=_outcome(ResponseKind.CLARIFY), narrative_payload=_payload())+ assert result.narrative_payload is not None+ assert result.narrative_payload["credentialPrompt"] == {"reason": "credential_name_unresolved"}+++def test_blocker_override_path_adds_credential_prompt_from_request_policy() -> None:+ ctx = _ctx(request_policy=RequestPolicy(clarification_reason="workflow_credential_inputs_unbound"))+ ctx.blocker_signal = CopilotToolBlockerSignal(+ blocker_kind="authority_denied",+ agent_steering_text="Reply to the user without updating the workflow.",+ user_facing_reason="I couldn't find the required credentials for the existing workflow.",+ recovery_hint="report_blocker_to_user",+ internal_reason_code="turn_intent_no_mutation_run_blocked",+ blocked_tool="update_workflow",+ )+ pre_override = AgentResult(user_response="agent reply", updated_workflow=None, global_llm_context=None)++ overridden = _finalize_result_with_blocker_override(ctx, pre_override)++ assert overridden.narrative_payload is not None+ assert overridden.narrative_payload["credentialPrompt"] == {"reason": "workflow_credential_inputs_unbound"}+++def test_backfill_adds_credential_prompt_from_text_marker_when_no_policy_signal() -> None:+ result = _result(+ _ctx(),+ user_response="You can add one at https://app.skyvern.com/credentials.",+ narrative_payload=_payload(),+ )+ assert result.narrative_payload is not None+ assert result.narrative_payload["credentialPrompt"] == {"reason": "assistant_directed"}+++def test_backfill_omits_credential_prompt_when_no_signal_present() -> None:+ result = _result(_ctx(), user_response="Done, the workflow is ready.", narrative_payload=_payload())+ assert result.narrative_payload is not None+ assert "credentialPrompt" not in result.narrative_payloaddiff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.pyindex 8a2deba98..06417464c 100644--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py@@ -8,7 +8,14 @@ import pytestfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext-from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policy+from skyvern.forge.sdk.copilot.request_policy import (+ CREDENTIAL_DEFERRED_DRAFT_REASONS,+ CREDENTIAL_PROMPT_CLARIFICATION_REASONS,+ RequestPolicy,+ _workflow_credential_inputs_unbound,+ build_request_policy,+ credential_prompt_reason,+)def _yaml(body: str) -> str:@@ -339,6 +346,81 @@ async def test_fallback_code_block_generic_one_time_code_does_not_skip_run() ->assert policy.testing_intent == "unspecified"+def test_credential_prompt_clarification_reasons_membership() -> None:+ assert CREDENTIAL_PROMPT_CLARIFICATION_REASONS == {+ "raw_secret",+ "credential_name_unresolved",+ "credential_invention_requested",+ "workflow_credential_inputs_unbound",+ }+++def test_credential_deferred_draft_reasons_is_narrower_than_prompt_reasons() -> None:+ assert CREDENTIAL_DEFERRED_DRAFT_REASONS < CREDENTIAL_PROMPT_CLARIFICATION_REASONS+++def test_credential_prompt_reason_typed_reason_wins_over_deferred_draft_flag() -> None:+ policy = RequestPolicy(clarification_reason="credential_name_unresolved", allow_missing_credentials_in_draft=True)+ assert credential_prompt_reason(policy, None) == "credential_name_unresolved"+++def test_credential_prompt_reason_deferred_draft_when_reason_cleared_but_flag_set() -> None:+ # Mirrors the explicit-defer path (_apply_explicit_code_block_credential_draft_policy),+ # which clears clarification_reason to "none" while leaving both flags set.+ policy = RequestPolicy(+ clarification_reason="none",+ allow_missing_credentials_in_draft=True,+ credential_draft_deferred_explicitly=True,+ )+ assert credential_prompt_reason(policy, None) == "credential_deferred_draft"+++def test_credential_prompt_reason_ignores_generic_skip_test_with_no_credential_signal() -> None:+ # allow_missing_credentials_in_draft alone also fires for the generic skip_test+ # fallthrough (any "draft only, don't test" turn), independent of credentials;+ # only credential_draft_deferred_explicitly means a credential was really deferred.+ policy = RequestPolicy(+ testing_intent="skip_test",+ clarification_reason="none",+ allow_missing_credentials_in_draft=True,+ credential_draft_deferred_explicitly=False,+ )+ assert credential_prompt_reason(policy, None) is None+++async def test_explicit_code_block_login_block_ban_surfaces_credential_prompt_end_to_end() -> None:+ policy = await build_request_policy(+ user_message="Write this as a code block. Do not create a login block for this part.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="o_test",+ handler=None,+ )+ assert policy.credential_input_kind == "none"+ assert policy.testing_intent == "skip_test"+ assert credential_prompt_reason(policy, None) == "credential_deferred_draft"+++ "text",+ [+ "Add it at https://app.skyvern.com/credentials.",+ "ADD IT AT HTTPS://APP.SKYVERN.COM/CREDENTIALS.",+ "Store it in the Credentials UI first.",+ "store it in the CREDENTIALS UI first.",+ ],+)+def test_credential_prompt_reason_marker_fallback_is_case_insensitive(text: str) -> None:+ assert credential_prompt_reason(None, text) == "assistant_directed"+++def test_credential_prompt_reason_policy_none_is_safe() -> None:+ assert credential_prompt_reason(None, None) is None+ assert credential_prompt_reason(None, "Everything is set, no action needed.") is None++@pytest.mark.asyncioasync def test_request_policy_resolver_still_blocks_raw_secret_with_author_time_log_only(monkeypatch: pytest.MonkeyPatch,