Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-9947 track copilot output policy diagnostics (#6073)
Andrew Neilsona9bb0aa6b18a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 4a46129aa..7484e80ce 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -40,6 +40,7 @@ from skyvern.forge.sdk.copilot.output_policy import (CopilotOutputKind,OutputPolicyReason,OutputPolicyVerdict,+ build_output_policy_diagnostics,derive_output_kind,evaluate_output_policy,hard_block_output_policy_verdict,@@ -952,7 +953,7 @@ def _translate_to_agent_result(unvalidated=unvalidated,)- output_policy_verdict = evaluate_output_policy(+ raw_output_policy_verdict = evaluate_output_policy(request_policy=ctx.request_policy,response_type=resp_type,user_response=str(user_response),@@ -964,6 +965,7 @@ def _translate_to_agent_result(unvalidated=unvalidated,output_kind=output_kind,)+ output_policy_verdict = _copy_output_policy_verdict(raw_output_policy_verdict)soft_rewrite_reasons: list[OutputPolicyReason] = []if OutputPolicyReason.INTERNAL_BLOCK_TAXONOMY_LEAK in output_policy_verdict.reason_codes:user_response = _INTERNAL_BLOCK_TAXONOMY_REPLY@@ -982,12 +984,24 @@ def _translate_to_agent_result(user_response = _ensure_unvalidated_proposal_affordance(str(user_response))soft_rewrite_reasons.append(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)output_policy_verdict.remove(OutputPolicyReason.MISSING_UNVALIDATED_PROPOSAL_AFFORDANCE)+ final_output_kind = (+ _blocked_final_output_kind(output_policy_verdict)+ if not output_policy_verdict.allowed+ else output_policy_verdict.output_kind+ )+ output_policy_diagnostics = build_output_policy_diagnostics(+ raw_verdict=raw_output_policy_verdict,+ final_verdict=output_policy_verdict,+ final_output_kind=final_output_kind,+ hard_block_reason_codes=list(output_policy_verdict.reason_codes),+ soft_rewrite_reason_codes=soft_rewrite_reasons,+ )trace_data = output_policy_verdict_to_trace_data(output_policy_verdict,surface="final_translation",response_type=resp_type,)- trace_data["soft_rewrite_reason_codes"] = [reason.value for reason in soft_rewrite_reasons]+ trace_data.update(output_policy_diagnostics)LOG.info("copilot output policy final verdict",**trace_data,@@ -998,6 +1012,7 @@ def _translate_to_agent_result(output_policy_verdict,prior_global_llm_context=global_llm_context,prior_workflow_yaml=chat_request.workflow_yaml,+ output_policy_diagnostics=output_policy_diagnostics,)return AgentResult(@@ -1010,6 +1025,7 @@ def _translate_to_agent_result(total_tokens=ctx.total_tokens_used,clear_proposed_workflow=resp_type == "ASK_QUESTION",unvalidated=unvalidated,+ output_policy_diagnostics=output_policy_diagnostics,)@@ -1149,10 +1165,29 @@ def _should_surface_untested_draft_despite_question(ctx: CopilotContext, respons)+def _copy_output_policy_verdict(verdict: OutputPolicyVerdict) -> OutputPolicyVerdict:+ return OutputPolicyVerdict(+ allowed=verdict.allowed,+ output_kind=verdict.output_kind,+ reason_codes=list(verdict.reason_codes),+ )+++def _blocked_final_output_kind(verdict: OutputPolicyVerdict) -> CopilotOutputKind:+ clarification_reasons = {+ OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,+ OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE,+ OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED,+ }+ if any(reason in clarification_reasons for reason in verdict.reason_codes):+ return CopilotOutputKind.CLARIFICATION_REQUEST+ return CopilotOutputKind.REFUSAL++def _evaluate_copilot_final_output_policy(ctx: CopilotContext,agent_output: Any,-) -> tuple[OutputPolicyVerdict, str]:+) -> tuple[OutputPolicyVerdict, str, dict[str, Any]]:text = _agent_output_to_text(agent_output)action_data = parse_final_response(text)response_type = action_data.get("type", "REPLY")@@ -1192,7 +1227,7 @@ def _evaluate_copilot_final_output_policy(workflow_attempted=workflow_attempted,unvalidated=False,)- verdict = evaluate_output_policy(+ raw_verdict = evaluate_output_policy(request_policy=ctx.request_policy,response_type=policy_response_type,user_response=policy_user_response,@@ -1204,7 +1239,17 @@ def _evaluate_copilot_final_output_policy(workflow_attempted=workflow_attempted,output_kind=output_kind,)- return hard_block_output_policy_verdict(verdict), response_type+ hard_verdict = hard_block_output_policy_verdict(raw_verdict)+ diagnostics = build_output_policy_diagnostics(+ raw_verdict=raw_verdict,+ final_verdict=hard_verdict,+ final_output_kind=_blocked_final_output_kind(hard_verdict)+ if not hard_verdict.allowed+ else hard_verdict.output_kind,+ hard_block_reason_codes=list(hard_verdict.reason_codes),+ soft_rewrite_reason_codes=[],+ )+ return hard_verdict, response_type, diagnosticsdef _build_copilot_input_guardrails(@@ -1268,13 +1313,21 @@ def _build_copilot_output_guardrails(reason_codes=[OutputPolicyReason.OUTPUT_POLICY_CONTEXT_MISSING],)response_type = "REPLY"+ diagnostics = build_output_policy_diagnostics(+ raw_verdict=verdict,+ final_verdict=verdict,+ final_output_kind=_blocked_final_output_kind(verdict),+ hard_block_reason_codes=list(verdict.reason_codes),+ soft_rewrite_reason_codes=[],+ )else:- verdict, response_type = _evaluate_copilot_final_output_policy(ctx, agent_output)+ verdict, response_type, diagnostics = _evaluate_copilot_final_output_policy(ctx, agent_output)trace_data = output_policy_verdict_to_trace_data(verdict,surface="agent_output",response_type=response_type,)+ trace_data.update(diagnostics)LOG.info("copilot output policy guardrail verdict", **trace_data)return GuardrailFunctionOutputCls(output_info=trace_data, tripwire_triggered=not verdict.allowed)@@ -1292,11 +1345,30 @@ def _output_policy_verdict_from_guardrail_exception(exc: BaseException) -> Outpureturn output_policy_verdict_from_trace_data(getattr(guardrail_output, "output_info", None))+def _output_policy_diagnostics_from_guardrail_exception(exc: BaseException) -> dict[str, Any] | None:+ guardrail_result = getattr(exc, "guardrail_result", None)+ guardrail_output = getattr(guardrail_result, "output", None)+ data = getattr(guardrail_output, "output_info", None)+ if not isinstance(data, dict):+ return None+ keys = {+ "raw_output_kind",+ "final_output_kind",+ "hard_block_reason_codes",+ "soft_rewrite_reason_codes",+ "raw_would_have_failed",+ "contained_failure",+ "final_output_policy_allowed",+ }+ return {key: data[key] for key in keys if key in data}++def _build_output_policy_blocked_result(ctx: CopilotContext,verdict: OutputPolicyVerdict,prior_global_llm_context: str | None,prior_workflow_yaml: str | None,+ output_policy_diagnostics: dict[str, Any] | None = None,) -> AgentResult:structured = StructuredContext.from_json_str(prior_global_llm_context)structured.decisions_made.append(@@ -1340,6 +1412,14 @@ def _build_output_policy_blocked_result(# Policy blocks invalidate any currently staged proposal. The prior# YAML is retained for context, but the accept UI should not stay armed.clear_proposed_workflow=True,+ output_policy_diagnostics=output_policy_diagnostics+ or build_output_policy_diagnostics(+ raw_verdict=verdict,+ final_verdict=verdict,+ final_output_kind=_blocked_final_output_kind(verdict),+ hard_block_reason_codes=list(verdict.reason_codes),+ soft_rewrite_reason_codes=[],+ ),)@@ -1690,6 +1770,7 @@ async def _run_copilot_turn_impl(_output_policy_verdict_from_guardrail_exception(exc),prior_global_llm_context=global_llm_context,prior_workflow_yaml=chat_request.workflow_yaml,+ output_policy_diagnostics=_output_policy_diagnostics_from_guardrail_exception(exc),)except MaxTurnsExceeded:return _build_max_turns_exit_result(ctx, global_llm_context)diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 1f6bdb626..1d1802992 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -133,6 +133,7 @@ class AgentResult:cancelled: bool = False# The route forces Accept/Reject regardless of ``auto_accept`` when this is True.unvalidated: bool = False+ output_policy_diagnostics: dict[str, Any] | None = None@dataclassdiff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex 4ec22aabb..3c4ffc322 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -233,6 +233,27 @@ def output_policy_verdict_to_trace_data(return data+def build_output_policy_diagnostics(+ *,+ raw_verdict: OutputPolicyVerdict,+ final_verdict: OutputPolicyVerdict,+ final_output_kind: CopilotOutputKind,+ hard_block_reason_codes: list[OutputPolicyReason],+ soft_rewrite_reason_codes: list[OutputPolicyReason],+) -> dict[str, Any]:+ raw_would_have_failed = bool(raw_verdict.reason_codes)+ contained_failure = bool(hard_block_reason_codes or soft_rewrite_reason_codes)+ return {+ "raw_output_kind": raw_verdict.output_kind.value,+ "final_output_kind": final_output_kind.value,+ "hard_block_reason_codes": [reason.value for reason in hard_block_reason_codes],+ "soft_rewrite_reason_codes": [reason.value for reason in soft_rewrite_reason_codes],+ "raw_would_have_failed": raw_would_have_failed,+ "contained_failure": raw_would_have_failed and contained_failure,+ "final_output_policy_allowed": final_verdict.allowed,+ }++def output_policy_verdict_from_trace_data(data: Any) -> OutputPolicyVerdict:if not isinstance(data, dict):return OutputPolicyVerdict(diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.pyindex fb881554b..67f712a02 100644--- a/skyvern/forge/sdk/routes/workflow_copilot.py+++ b/skyvern/forge/sdk/routes/workflow_copilot.py@@ -332,6 +332,7 @@ async def _persist_cancel_turn(total_tokens = Noneresponse_type = "REPLY"unvalidated = False+ output_policy_diagnostics = Noneif chat.proposed_workflow is not None:await asyncio.shield(_clear_proposed_workflow(chat))else:@@ -348,6 +349,7 @@ async def _persist_cancel_turn(total_tokens = getattr(agent_result, "total_tokens", None)response_type = getattr(agent_result, "response_type", "REPLY")unvalidated = bool(getattr(agent_result, "unvalidated", False))+ output_policy_diagnostics = getattr(agent_result, "output_policy_diagnostics", None)await asyncio.shield(app.DATABASE.workflow_params.create_workflow_copilot_chat_message(@@ -379,6 +381,7 @@ async def _persist_cancel_turn(response_type=response_type,unvalidated=unvalidated,cancelled=True,+ output_policy_diagnostics=output_policy_diagnostics,)))@@ -451,6 +454,7 @@ async def _finalise_normal_turn(total_tokens=getattr(agent_result, "total_tokens", None),response_type=getattr(agent_result, "response_type", "REPLY"),unvalidated=bool(getattr(agent_result, "unvalidated", False)),+ output_policy_diagnostics=getattr(agent_result, "output_policy_diagnostics", None),))diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 4f4771dc9..37f38a37a 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -1,5 +1,6 @@from datetime import datetimefrom enum import StrEnum+from typing import Anyfrom pydantic import BaseModel, ConfigDict, Field@@ -125,6 +126,10 @@ class WorkflowCopilotStreamResponseUpdate(BaseModel):False,description="When true, this RESPONSE was emitted by a user cancel; clients must not auto-apply.",)+ output_policy_diagnostics: dict[str, Any] | None = Field(+ None,+ description="Diagnostic output-policy labels for raw-vs-final quality reporting.",+ )class WorkflowCopilotStreamErrorUpdate(BaseModel):diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 1a6a651bb..5ad4b81b2 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -652,7 +652,7 @@ async def test_sdk_request_policy_guardrail_trips_on_required_clarification() ->def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(_ctx(),{"type": "REPLY", "user_response": "I used password: hunter2."},)@@ -660,6 +660,15 @@ def test_sdk_output_guardrail_hard_blocks_raw_secret_final_text() -> None:assert response_type == "REPLY"assert not verdict.allowedassert verdict.reason_codes == [OutputPolicyReason.RAW_SECRET_LEAK]+ assert diagnostics == {+ "raw_output_kind": "informational_answer",+ "final_output_kind": "refusal",+ "hard_block_reason_codes": ["raw_secret_leak"],+ "soft_rewrite_reason_codes": [],+ "raw_would_have_failed": True,+ "contained_failure": True,+ "final_output_policy_allowed": False,+ }@pytest.mark.parametrize(@@ -759,7 +768,7 @@ workflow_definition:- azure_credentials"""- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(ctx,{"type": "REPLY",@@ -771,6 +780,8 @@ workflow_definition:assert response_type == "REPLY"assert verdict.allowedassert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes+ assert diagnostics["raw_would_have_failed"] is False+ assert diagnostics["contained_failure"] is Falsedef test_sdk_output_guardrail_defers_raw_question_after_untested_draft() -> None:@@ -801,7 +812,7 @@ workflow_definition:- azure_credentials"""- verdict, response_type = agent_module._evaluate_copilot_final_output_policy(+ verdict, response_type, diagnostics = agent_module._evaluate_copilot_final_output_policy(ctx,{"type": "ASK_QUESTION",@@ -812,6 +823,8 @@ workflow_definition:assert response_type == "ASK_QUESTION"assert verdict.allowedassert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes+ assert diagnostics["raw_output_kind"] == "workflow_draft_proposal"+ assert diagnostics["final_output_kind"] == "workflow_draft_proposal"def test_translation_surfaces_untested_draft_when_agent_asks_after_drafting() -> None:@@ -1026,7 +1039,21 @@ def test_translate_to_agent_result_rewrites_deprecated_block_taxonomy() -> None:final_log = next(call for call in log_info.call_args_list if call.args[0] == "copilot output policy final verdict")assert final_log.kwargs["allowed"] is Trueassert final_log.kwargs["reason_codes"] == []+ assert final_log.kwargs["raw_output_kind"] == "informational_answer"+ assert final_log.kwargs["final_output_kind"] == "informational_answer"+ assert final_log.kwargs["hard_block_reason_codes"] == []assert final_log.kwargs["soft_rewrite_reason_codes"] == ["internal_block_taxonomy_leak"]+ assert final_log.kwargs["raw_would_have_failed"] is True+ assert final_log.kwargs["contained_failure"] is True+ assert agent_result.output_policy_diagnostics == {+ "raw_output_kind": "informational_answer",+ "final_output_kind": "informational_answer",+ "hard_block_reason_codes": [],+ "soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],+ "raw_would_have_failed": True,+ "contained_failure": True,+ "final_output_policy_allowed": True,+ }def test_translate_to_agent_result_prioritizes_unbacked_workflow_claim_over_taxonomy_rewrite() -> None:diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 97164d08c..e60bc1abe 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -408,6 +408,14 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytestunvalidated=True,total_tokens=42,response_type="REPLY",+ output_policy_diagnostics={+ "raw_output_kind": "informational_answer",+ "final_output_kind": "informational_answer",+ "hard_block_reason_codes": [],+ "soft_rewrite_reason_codes": ["internal_block_taxonomy_leak"],+ "raw_would_have_failed": True,+ "contained_failure": True,+ },)restore_mock = _setup_new_copilot_mocks(monkeypatch, chat, original_workflow, agent_result)@@ -448,6 +456,7 @@ async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest)assert response_frame is not Noneassert getattr(response_frame, "unvalidated", False) is True+ assert response_frame.output_policy_diagnostics == agent_result.output_policy_diagnosticsassert not [f for f in sent_frames if isinstance(f, WorkflowCopilotStreamErrorUpdate)]