Skip to content

Automate browser based workflows with AI

SKY-9947 track copilot output policy diagnostics (#6073)

Andrew Neilsona9bb0aa6b18a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 4a46129aa..7484e80ce 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -40,6 +40,7 @@ from skyvern.forge.sdk.copilot.output_policy import (
CopilotOutputKind,
OutputPolicyReason,
OutputPolicyVerdict,
+ build_output_policy_diagnostics,
derive_output_kind,
evaluate_output_policy,
hard_block_output_policy_verdict,
@@ -952,7 +953,7 @@ def _translate_to_agent_result(
unvalidated=unvalidated,
)
- output_policy_verdict = evaluate_output_policy(
+ raw_output_policy_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type=resp_type,
user_response=str(user_response),
@@ -964,6 +965,7 @@ def _translate_to_agent_result(
unvalidated=unvalidated,
output_kind=output_kind,
)
+ output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict)
soft_rewrite_reasons: list[OutputPolicyReason] = []
if OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in output_policy_verdict.reason_codes:
user_response = _INTERNAL_BLOCK_TAXONOMY_REPLY
@@ -982,12 +984,24 @@ def _translate_to_agent_result(
user_response = _ensure_unvalidated_proposal_affordance(str(user_response))
soft_rewrite_reasons.append(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
output_policy_verdict.remove(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)
+ final_output_kind = (
+ _blocked_final_output_kind(output_policy_verdict)
+ if not output_policy_verdict.allowed
+ else output_policy_verdict.output_kind
+ )
+ output_policy_diagnostics = build_output_policy_diagnostics(
+ raw_verdict=raw_output_policy_verdict,
+ final_verdict=output_policy_verdict,
+ final_output_kind=final_output_kind,
+ hard_block_reason_codes=list(output_policy_verdict.reason_codes),
+ soft_rewrite_reason_codes=soft_rewrite_reasons,
+ )
trace_data = output_policy_verdict_to_trace_data(
output_policy_verdict,
surface="final_translation",
response_type=resp_type,
)
- trace_data["soft_rewrite_reason_codes"] = [reason.value for reason in soft_rewrite_reasons]
+ trace_data.update(output_policy_diagnostics)
LOG.info(
"copilot output policy final verdict",
**trace_data,
@@ -998,6 +1012,7 @@ def _translate_to_agent_result(
output_policy_verdict,
prior_global_llm_context=global_llm_context,
prior_workflow_yaml=chat_request.workflow_yaml,
+ output_policy_diagnostics=output_policy_diagnostics,
)
return AgentResult(
@@ -1010,6 +1025,7 @@ def _translate_to_agent_result(
total_tokens=ctx.total_tokens_used,
clear_proposed_workflow=resp_type == "ASK_QUESTION",
unvalidated=unvalidated,
+ output_policy_diagnostics=output_policy_diagnostics,
)
@@ -1149,10 +1165,29 @@ def _should_surface_untested_draft_despite_question(ctx: CopilotContext, respons
)
+def _copy_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict:
+ return OutputPolicyVerdict(
+ allowed=verdict.allowed,
+ output_kind=verdict.output_kind,
+ reason_codes=list(verdict.reason_codes),
+ )
+
+
+def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKind:
+ clarification_reasons = {
+ OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
+ OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,
+ OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,
+ }
+ if any(reason in clarification_reasons for reason in verdict.reason_codes):
+ return CopilotOutputKind.CLARIFICATION_REQUEST
+ return CopilotOutputKind.REFUSAL
+
+
def _evaluate_copilot_final_output_policy(
ctx: CopilotContext,
agent_output: Any,
-) -> tuple[OutputPolicyVerdict, str]:
+) -> tuple[OutputPolicyVerdict, str, dict[str, Any]]:
text = _agent_output_to_text(agent_output)
action_data = parse_final_response(text)
response_type = action_data.get("type", "REPLY")
@@ -1192,7 +1227,7 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted=workflow_attempted,
unvalidated=False,
)
- verdict = evaluate_output_policy(
+ raw_verdict = evaluate_output_policy(
request_policy=ctx.request_policy,
response_type=policy_response_type,
user_response=policy_user_response,
@@ -1204,7 +1239,17 @@ def _evaluate_copilot_final_output_policy(
workflow_attempted=workflow_attempted,
output_kind=output_kind,
)
- return hard_block_output_policy_verdict(verdict), response_type
+ hard_verdict = hard_block_output_policy_verdict(raw_verdict)
+ diagnostics = build_output_policy_diagnostics(
+ raw_verdict=raw_verdict,
+ final_verdict=hard_verdict,
+ final_output_kind=_blocked_final_output_kind(hard_verdict)
+ if not hard_verdict.allowed
+ else hard_verdict.output_kind,
+ hard_block_reason_codes=list(hard_verdict.reason_codes),
+ soft_rewrite_reason_codes=[],
+ )
+ return hard_verdict, response_type, diagnostics
def _build_copilot_input_guardrails(
@@ -1268,13 +1313,21 @@ def _build_copilot_output_guardrails(
reason_codes=[OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING],
)
response_type = "REPLY"
+ diagnostics = build_output_policy_diagnostics(
+ raw_verdict=verdict,
+ final_verdict=verdict,
+ final_output_kind=_blocked_final_output_kind(verdict),
+ hard_block_reason_codes=list(verdict.reason_codes),
+ soft_rewrite_reason_codes=[],
+ )
else:
- verdict, response_type = _evaluate_copilot_final_output_policy(ctx, agent_output)
+ verdict, response_type, diagnostics = _evaluate_copilot_final_output_policy(ctx, agent_output)
trace_data = output_policy_verdict_to_trace_data(
verdict,
surface="agent_output",
response_type=response_type,
)
+ trace_data.update(diagnostics)
LOG.info("copilot output policy guardrail verdict", **trace_data)
return GuardrailFunctionOutputCls(output_info=trace_data, tripwire_triggered=not verdict.allowed)
@@ -1292,11 +1345,30 @@ def _output_policy_verdict_from_guardrail_exception(exc: BaseException) -> Outpu
return output_policy_verdict_from_trace_data(getattr(guardrail_output, "output_info", None))
+def _output_policy_diagnostics_from_guardrail_exception(exc: BaseException) -> dict[str, Any] | None:
+ guardrail_result = getattr(exc, "guardrail_result", None)
+ guardrail_output = getattr(guardrail_result, "output", None)
+ data = getattr(guardrail_output, "output_info", None)
+ if not isinstance(data, dict):
+ return None
+ keys = {
+ "raw_output_kind",
+ "final_output_kind",
+ "hard_block_reason_codes",
+ "soft_rewrite_reason_codes",
+ "raw_would_have_failed",
+ "contained_failure",
+ "final_output_policy_allowed",
+ }
+ return {key: data[key] for key in keys if key in data}
+
+
def _build_output_policy_blocked_result(
ctx: CopilotContext,
verdict: OutputPolicyVerdict,
prior_global_llm_context: str | None,
prior_workflow_yaml: str | None,
+ output_policy_diagnostics: dict[str, Any] | None = None,
) -> AgentResult:
structured = StructuredContext.from_json_str(prior_global_llm_context)
structured.decisions_made.append(
@@ -1340,6 +1412,14 @@ def _build_output_policy_blocked_result(
# Policy blocks invalidate any currently staged proposal. The prior
# YAML is retained for context, but the accept UI should not stay armed.
clear_proposed_workflow=True,
+ output_policy_diagnostics=output_policy_diagnostics
+ or build_output_policy_diagnostics(
+ raw_verdict=verdict,
+ final_verdict=verdict,
+ final_output_kind=_blocked_final_output_kind(verdict),
+ hard_block_reason_codes=list(verdict.reason_codes),
+ soft_rewrite_reason_codes=[],
+ ),
)
@@ -1690,6 +1770,7 @@ async def _run_copilot_turn_impl(
_output_policy_verdict_from_guardrail_exception(exc),
prior_global_llm_context=global_llm_context,
prior_workflow_yaml=chat_request.workflow_yaml,
+ output_policy_diagnostics=_output_policy_diagnostics_from_guardrail_exception(exc),
)
except MaxTurnsExceeded:
return _build_max_turns_exit_result(ctx, global_llm_context)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 1f6bdb626..1d1802992 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -133,6 +133,7 @@ class AgentResult:
cancelled: bool = False
# The route forces Accept/Reject regardless of ``auto_accept`` when this is True.
unvalidated: bool = False
+ output_policy_diagnostics: dict[str, Any] | None = None
@dataclass
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 4ec22aabb..3c4ffc322 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -233,6 +233,27 @@ def output_policy_verdict_to_trace_data(
return data
+def build_output_policy_diagnostics(
+ *,
+ raw_verdict: OutputPolicyVerdict,
+ final_verdict: OutputPolicyVerdict,
+ final_output_kind: CopilotOutputKind,
+ hard_block_reason_codes: list[OutputPolicyReason],
+ soft_rewrite_reason_codes: list[OutputPolicyReason],
+) -> dict[str, Any]:
+ raw_would_have_failed = bool(raw_verdict.reason_codes)
+ contained_failure = bool(hard_block_reason_codes or soft_rewrite_reason_codes)
+ return {
+ "raw_output_kind": raw_verdict.output_kind.value,
+ "final_output_kind": final_output_kind.value,
+ "hard_block_reason_codes": [reason.value for reason in hard_block_reason_codes],
+ "soft_rewrite_reason_codes": [reason.value for reason in soft_rewrite_reason_codes],
+ "raw_would_have_failed": raw_would_have_failed,
+ "contained_failure": raw_would_have_failed and contained_failure,
+ "final_output_policy_allowed": final_verdict.allowed,
+ }
+
+
def output_policy_verdict_from_trace_data(data: Any) -> OutputPolicyVerdict:
if not isinstance(data, dict):
return OutputPolicyVerdict(
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index fb881554b..67f712a02 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -332,6 +332,7 @@ async def _persist_cancel_turn(
total_tokens = None
response_type = "REPLY"
unvalidated = False
+ output_policy_diagnostics = None
if chat.proposed_workflow is not None:
await asyncio.shield(_clear_proposed_workflow(chat))
else:
@@ -348,6 +349,7 @@ async def _persist_cancel_turn(
total_tokens = getattr(agent_result, "total_tokens", None)
response_type = getattr(agent_result, "response_type", "REPLY")
unvalidated = bool(getattr(agent_result, "unvalidated", False))
+ output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)
await asyncio.shield(
app.DATABASE.workflow_params.create_workflow_copilot_chat_message(
@@ -379,6 +381,7 @@ async def _persist_cancel_turn(
response_type=response_type,
unvalidated=unvalidated,
cancelled=True,
+ output_policy_diagnostics=output_policy_diagnostics,
)
)
)
@@ -451,6 +454,7 @@ async def _finalise_normal_turn(
total_tokens=getattr(agent_result, "total_tokens", None),
response_type=getattr(agent_result, "response_type", "REPLY"),
unvalidated=bool(getattr(agent_result, "unvalidated", False)),
+ output_policy_diagnostics=getattr(agent_result, "output_policy_diagnostics", None),
)
)
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 4f4771dc9..37f38a37a 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -1,5 +1,6 @@
from datetime import datetime
from enum import StrEnum
+from typing import Any
from pydantic import BaseModel, ConfigDict, Field
@@ -125,6 +126,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):
False,
description="When true, this RESPONSE was emitted by a user cancel; clients must not auto-apply.",
)
+ output_policy_diagnostics: dict[str, Any] | None = Field(
+ None,
+ description="Diagnostic output-policy labels for raw-vs-final quality reporting.",
+ )
class WorkflowCopilotStreamErrorUpdate(BaseModel):
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 1a6a651bb..5ad4b81b2 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -652,7 +652,7 @@ async def test_sdk_request_policy_guardrail_trips_on_required_clarification() ->
def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
_ctx(),
{"type": "REPLY", "user_response": "I used password: hunter2."},
)
@@ -660,6 +660,15 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:
assert response_type == "REPLY"
assert not verdict.allowed
assert verdict.reason_codes == [OutputPolicyReason.RAW_SECRET_LEAK]
+ assert diagnostics == {
+ "raw_output_kind": "informational_answer",
+ "final_output_kind": "refusal",
+ "hard_block_reason_codes": ["raw_secret_leak"],
+ "soft_rewrite_reason_codes": [],
+ "raw_would_have_failed": True,
+ "contained_failure": True,
+ "final_output_policy_allowed": False,
+ }
@pytest.mark.parametrize(
@@ -759,7 +768,7 @@ workflow_definition:
- azure_credentials
"""
- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
ctx,
{
"type": "REPLY",
@@ -771,6 +780,8 @@ workflow_definition:
assert response_type == "REPLY"
assert verdict.allowed
assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes
+ assert diagnostics["raw_would_have_failed"] is False
+ assert diagnostics["contained_failure"] is False
def test_sdk_output_guardrail_defers_raw_question_after_untested_draft() -> None:
@@ -801,7 +812,7 @@ workflow_definition:
- azure_credentials
"""
- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(
+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(
ctx,
{
"type": "ASK_QUESTION",
@@ -812,6 +823,8 @@ workflow_definition:
assert response_type == "ASK_QUESTION"
assert verdict.allowed
assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes
+ assert diagnostics["raw_output_kind"] == "workflow_draft_proposal"
+ assert diagnostics["final_output_kind"] == "workflow_draft_proposal"
def test_translation_surfaces_untested_draft_when_agent_asks_after_drafting() -> None:
@@ -1026,7 +1039,21 @@ def test_translate_to_agent_result_rewrites_deprecated_block_taxonomy() -> None:
final_log = next(call for call in log_info.call_args_list if call.args[0] == "copilot output policy final verdict")
assert final_log.kwargs["allowed"] is True
assert final_log.kwargs["reason_codes"] == []
+ assert final_log.kwargs["raw_output_kind"] == "informational_answer"
+ assert final_log.kwargs["final_output_kind"] == "informational_answer"
+ assert final_log.kwargs["hard_block_reason_codes"] == []
assert final_log.kwargs["soft_rewrite_reason_codes"] == ["internal_block_taxonomy_leak"]
+ assert final_log.kwargs["raw_would_have_failed"] is True
+ assert final_log.kwargs["contained_failure"] is True
+ assert agent_result.output_policy_diagnostics == {
+ "raw_output_kind": "informational_answer",
+ "final_output_kind": "informational_answer",
+ "hard_block_reason_codes": [],
+ "soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],
+ "raw_would_have_failed": True,
+ "contained_failure": True,
+ "final_output_policy_allowed": True,
+ }
def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 97164d08c..e60bc1abe 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -408,6 +408,14 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest
unvalidated=True,
total_tokens=42,
response_type="REPLY",
+ output_policy_diagnostics={
+ "raw_output_kind": "informational_answer",
+ "final_output_kind": "informational_answer",
+ "hard_block_reason_codes": [],
+ "soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],
+ "raw_would_have_failed": True,
+ "contained_failure": True,
+ },
)
restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)
@@ -448,6 +456,7 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest
)
assert response_frame is not None
assert getattr(response_frame, "unvalidated", False) is True
+ assert response_frame.output_policy_diagnostics == agent_result.output_policy_diagnostics
assert not [f for f in sent_frames if isinstance(f, WorkflowCopilotStreamErrorUpdate)]