Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-15418 Make output policy block reasons countable (#8403)
Andrew Neilson146e654a068b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex fec767934..2d6176ea1 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -3478,6 +3478,7 @@ async def _translate_to_agent_result(prior_workflow_yaml=chat_request.workflow_yaml,output_policy_diagnostics=inline_diagnostics,require_full_workflow_test=chat_request.product_action == "test_end_to_end",+ evaluated_reason_codes=list(inline_raw_verdict.reason_codes),)# REPLACE_WORKFLOW bypasses the update_workflow tool guardrail, so# policy and post-emission rejects run here before YAML processing.@@ -3745,6 +3746,7 @@ async def _translate_to_agent_result(prior_workflow_yaml=chat_request.workflow_yaml,output_policy_diagnostics=output_policy_diagnostics,require_full_workflow_test=direct_test_handoff,+ evaluated_reason_codes=list(raw_output_policy_verdict.reason_codes),)final_user_response = str(user_response)@@ -4248,6 +4250,19 @@ def _output_policy_diagnostics_from_guardrail_exception(exc: BaseException) -> dreturn {key: data[key] for key in keys if key in data}+def _output_policy_reason_codes_from_guardrail_exception(exc: BaseException) -> list[OutputPolicyReason]:+ diagnostics = _output_policy_diagnostics_from_guardrail_exception(exc)+ if diagnostics is None:+ return list(_output_policy_verdict_from_guardrail_exception(exc).reason_codes)+ reason_codes: list[OutputPolicyReason] = []+ for raw_reason in diagnostics.get("raw_reason_codes") or []:+ try:+ reason_codes.append(OutputPolicyReason(str(raw_reason)))+ except ValueError:+ continue+ return reason_codes or list(_output_policy_verdict_from_guardrail_exception(exc).reason_codes)++def _unapproved_credential_reference_reply() -> str:# "Credentials UI" is a credential_prompt_reason() text marker the FE credential card keys off, so# this reply must keep it verbatim. One sentence, no candidate enumeration: the card renders the@@ -4272,6 +4287,7 @@ def _build_output_policy_blocked_result(prior_workflow_yaml: str | None,output_policy_diagnostics: dict[str, Any] | None = None,require_full_workflow_test: bool = False,+ evaluated_reason_codes: list[OutputPolicyReason] | None = None,) -> AgentResult:# A blocker turn whose signal owns final rendering never ships a proposal;# steering-only blockers should still flow through normal output-policy@@ -4282,9 +4298,15 @@ def _build_output_policy_blocked_result(ctx.last_workflow if ctx.last_workflow is not None and ctx.last_workflow_yaml and not blocker_active else None)preserved_workflow_yaml = ctx.last_workflow_yaml if preserved_workflow is not None else None+ output_policy_reasons = list(verdict.reason_codes if evaluated_reason_codes is None else evaluated_reason_codes)structured = StructuredContext.from_json_str(prior_global_llm_context)structured.decisions_made.append(- "output-policy blocked final output: " + ", ".join(reason.value for reason in verdict.reason_codes)+ "output-policy blocked final output: " + ", ".join(reason.value for reason in output_policy_reasons)+ )+ LOG.warning(+ "copilot output policy blocked final output",+ log_code="copilot_output_policy_block",+ **{"copilot.output_policy_reasons": [reason.value for reason in output_policy_reasons]},)add_saved_draft_copy = Falsefallback_user_response: str | None = None@@ -4390,6 +4412,7 @@ def _build_output_policy_blocked_result(reason_code="output_policy_block",terminal_reason="output_policy_block",)+ output_policy_outcome = output_policy_outcome.model_copy(update={"output_policy_reasons": output_policy_reasons})proposal_tested = ctx.last_full_workflow_test_ok if require_full_workflow_test else ctx.last_test_okreturn _make_agent_result(ctx,@@ -5081,6 +5104,7 @@ async def _run_copilot_turn_impl(prior_workflow_yaml=chat_request.workflow_yaml,output_policy_diagnostics=_output_policy_diagnostics_from_guardrail_exception(exc),require_full_workflow_test=chat_request.product_action == "test_end_to_end",+ evaluated_reason_codes=_output_policy_reason_codes_from_guardrail_exception(exc),)except CopilotTurnHalt as exc:LOG.info(diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex e2c3843ea..8e7e2f29c 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -30,6 +30,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (workflow_credential_ids_from_parsed,workflow_credential_origins_from_parsed,)+from skyvern.forge.sdk.schemas.copilot_turn_outcome import OutputPolicyReasonWORKFLOW_PRESENT_SENTINEL = object()_CREDENTIAL_ID_RE = re.compile(r"\bcred_[A-Za-z0-9][A-Za-z0-9_-]*\b")@@ -174,20 +175,6 @@ class CopilotOutputKind(StrEnum):WORKFLOW_RUN_RESULT = "workflow_run_result"-class OutputPolicyReason(StrEnum):- RAW_SECRET_LEAK = "raw_secret_leak"- UNAPPROVED_CREDENTIAL_REFERENCE = "unapproved_credential_reference"- CREDENTIAL_SCOPE_BROADENED = "credential_scope_broadened"- UNBACKED_WORKFLOW_DELIVERY_CLAIM = "unbacked_workflow_delivery_claim"- MISSING_PROPOSAL_STATE = "missing_proposal_state"- PERSISTENCE_STATE_MISMATCH = "persistence_state_mismatch"- OUTPUT_POLICY_CONTEXT_MISSING = "output_policy_context_missing"- INTERNAL_BLOCK_TAXONOMY_LEAK = "internal_block_taxonomy_leak"- INTERNAL_CLASSIFIER_VOCAB_LEAK = "internal_classifier_vocab_leak"- SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"- WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"--@dataclassclass OutputPolicyVerdict:allowed: bool = Truediff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.pyindex 8db842510..69ca35bef 100644--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py@@ -23,6 +23,20 @@ class ResponseKind(StrEnum):RECOVER = "recover"+class OutputPolicyReason(StrEnum):+ RAW_SECRET_LEAK = "raw_secret_leak"+ UNAPPROVED_CREDENTIAL_REFERENCE = "unapproved_credential_reference"+ CREDENTIAL_SCOPE_BROADENED = "credential_scope_broadened"+ UNBACKED_WORKFLOW_DELIVERY_CLAIM = "unbacked_workflow_delivery_claim"+ MISSING_PROPOSAL_STATE = "missing_proposal_state"+ PERSISTENCE_STATE_MISMATCH = "persistence_state_mismatch"+ OUTPUT_POLICY_CONTEXT_MISSING = "output_policy_context_missing"+ INTERNAL_BLOCK_TAXONOMY_LEAK = "internal_block_taxonomy_leak"+ INTERNAL_CLASSIFIER_VOCAB_LEAK = "internal_classifier_vocab_leak"+ SELF_PRESCRIPTIVE_PHRASE_LEAK = "self_prescriptive_phrase_leak"+ WORKFLOW_YAML_IN_REPLY = "workflow_yaml_in_reply"++_UNSAFE_IDENTIFIER_RE = re.compile(r"[^A-Za-z0-9_\- ]")@@ -65,6 +79,7 @@ class TurnOutcome(BaseModel):response_kind: ResponseKindreason_code: str = ""+ output_policy_reasons: list[OutputPolicyReason] = Field(default_factory=list)actuation_obligation_key: str = ""normalized_reply_signature: str = ""tool_calls: list[str] = Field(default_factory=list)@@ -78,3 +93,16 @@ class TurnOutcome(BaseModel):idempotency_digest: str | None = Noneunresolved_runtime_failure: UnresolvedRuntimeFailure | None = Noneconnected_account_choices: list[ConnectedAccountChoice] | None = None++ @field_validator("output_policy_reasons", mode="before")+ @classmethod+ def _drop_unknown_output_policy_reasons(cls, value: object) -> object:+ if not isinstance(value, list):+ return value+ known_reasons: list[OutputPolicyReason] = []+ for raw_reason in value:+ try:+ known_reasons.append(OutputPolicyReason(raw_reason))+ except (TypeError, ValueError):+ continue+ return known_reasonsdiff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 1f9f68a8f..91448347a 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -1609,6 +1609,79 @@ def test_output_policy_credential_block_asks_for_credential_confirmation(assert "I could not safely return" not in result.user_response+def test_output_policy_block_turn_outcome_persists_and_logs_every_reason(monkeypatch: pytest.MonkeyPatch) -> None:+ log = MagicMock()+ monkeypatch.setattr(agent_module, "LOG", log)+ reasons = [+ OutputPolicyReason.PERSISTENCE_STATE_MISMATCH,+ OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING,+ ]++ result = agent_module._build_output_policy_blocked_result(+ _ctx(),+ OutputPolicyVerdict(reason_codes=reasons),+ prior_global_llm_context="{}",+ prior_workflow_yaml="title: Prior",+ )++ assert result.turn_outcome is not None+ assert result.turn_outcome.output_policy_reasons == reasons+ log.warning.assert_any_call(+ "copilot output policy blocked final output",+ log_code="copilot_output_policy_block",+ **{"copilot.output_policy_reasons": [reason.value for reason in reasons]},+ )+++def test_unblocked_turn_outcome_has_empty_output_policy_reasons() -> None:+ outcome = TurnOutcome(response_kind=ResponseKind.ANSWER)++ assert outcome.output_policy_reasons == []+ assert outcome.model_dump(mode="json")["output_policy_reasons"] == []+++def test_turn_outcome_drops_unknown_historical_output_policy_reasons() -> None:+ outcome = TurnOutcome.model_validate(+ {+ "response_kind": "clarify",+ "reason_code": "output_policy_block",+ "terminal_reason": "output_policy_block",+ "output_policy_reasons": [+ "internal_tool_instruction_leak",+ OutputPolicyReason.RAW_SECRET_LEAK.value,+ ],+ }+ )++ assert outcome.response_kind is ResponseKind.CLARIFY+ assert outcome.terminal_reason == "output_policy_block"+ assert outcome.output_policy_reasons == [OutputPolicyReason.RAW_SECRET_LEAK]+++def test_output_policy_fallback_replacement_keeps_turn_outcome_reasons(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ctx = _ctx()+ ctx.last_test_anti_bot = "challenge"+ reasons = [OutputPolicyReason.PERSISTENCE_STATE_MISMATCH]+ monkeypatch.setattr(+ agent_module,+ "evaluate_output_policy",+ lambda **_kwargs: OutputPolicyVerdict(reason_codes=[OutputPolicyReason.RAW_SECRET_LEAK]),+ )++ result = agent_module._build_output_policy_blocked_result(+ ctx,+ OutputPolicyVerdict(reason_codes=reasons),+ prior_global_llm_context="{}",+ prior_workflow_yaml="title: Prior",+ )++ assert result.turn_outcome is not None+ assert result.turn_outcome.output_policy_reasons == reasons+ assert result.turn_outcome.model_dump(mode="json")["output_policy_reasons"] == [reason.value for reason in reasons]++def test_output_policy_block_preserves_already_gated_workflow_proposal() -> None:ctx = _ctx()ctx.last_workflow = SimpleNamespace(name="draft")@@ -1919,13 +1992,13 @@ workflow_definition:assert ctx.consecutive_tool_tracker == ["update_workflow", "update_workflow"]-def test_inline_replace_workflow_rejects_raw_secret_before_processing(monkeypatch) -> None:+def test_inline_replace_output_policy_turn_outcome_keeps_raw_reasons_before_processing(monkeypatch) -> None:process_mock = MagicMock()monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.workflow_update._process_workflow_yaml", process_mock)result = _fake_run_result({"type": "REPLACE_WORKFLOW",- "user_response": "Here is the workflow.",+ "user_response": "Here is the workflow. Saw [copilot:nudge] in the trace.","workflow_yaml": """workflow_definition:blocks:@@ -1950,6 +2023,11 @@ workflow_definition:assert agent_result.updated_workflow is Noneassert agent_result.clear_proposed_workflow is Falseassert agent_result.proposal_disposition == "no_proposal"+ assert agent_result.turn_outcome is not None+ assert agent_result.turn_outcome.output_policy_reasons == [+ OutputPolicyReason.RAW_SECRET_LEAK,+ OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK,+ ]process_mock.assert_not_called()@@ -2179,6 +2257,59 @@ def test_sdk_output_guardrail_records_raw_soft_reason_alongside_hard_block() ->assert "internal_block_taxonomy_leak" in raw_reason_codes+def test_translate_output_policy_block_turn_outcome_keeps_raw_soft_reason(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ log = MagicMock()+ monkeypatch.setattr(agent_module, "LOG", log)+ leak = f"Use the key {_FAKE_OPENAI_KEY} to authenticate. Saw [copilot:nudge] in the trace."++ agent_result = asyncio.run(+ agent_module._translate_to_agent_result(+ _fake_run_result({"type": "REPLY", "user_response": leak}),+ _ctx(),+ global_llm_context=None,+ chat_request=_chat_request(),+ organization_id="org-1",+ )+ )++ assert agent_result.turn_outcome is not None+ assert agent_result.turn_outcome.output_policy_reasons == [+ OutputPolicyReason.RAW_SECRET_LEAK,+ OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK,+ ]+ log.warning.assert_any_call(+ "copilot output policy blocked final output",+ log_code="copilot_output_policy_block",+ **{+ "copilot.output_policy_reasons": [+ OutputPolicyReason.RAW_SECRET_LEAK.value,+ OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK.value,+ ]+ },+ )+++def test_output_policy_guardrail_turn_outcome_recovers_every_raw_reason() -> None:+ output_info = {+ "allowed": False,+ "reason_codes": [OutputPolicyReason.RAW_SECRET_LEAK.value],+ "raw_reason_codes": [+ OutputPolicyReason.RAW_SECRET_LEAK.value,+ OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK.value,+ ],+ }+ exc = SimpleNamespace(guardrail_result=SimpleNamespace(output=SimpleNamespace(output_info=output_info)))++ reasons = agent_module._output_policy_reason_codes_from_guardrail_exception(exc)++ assert reasons == [+ OutputPolicyReason.RAW_SECRET_LEAK,+ OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK,+ ]++def test_evaluate_output_policy_skips_new_detectors_on_ask_question() -> None:classifier_leak = "RequestPolicy blocked this turn, safe_reason_code=request_policy_clarification."self_prescriptive_leak = "Please send 'continue debugging' next and I'll keep going."