Skip to content

Automate browser based workflows with AI

SKY-10910 Preserve terminal evidence in Copilot hard blocks (#6579)

Andrew Neilson338c91b61183
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index c14631657..8e5fec877 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -36,7 +36,12 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
+ clear_terminal_evidence_on_workflow_edit,
+ compose_terminal_evidence_user_facing_reason,
contains_internal_machinery_leak,
+ refresh_held_loop_blocker_evidence,
+ terminal_evidence_from_ctx,
+ terminal_evidence_has_recorded_state,
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
@@ -1241,6 +1246,9 @@ def _build_turn_halt_exit_result(
halt: TurnHalt,
) -> AgentResult:
signal = halt.blocker_signal
+ if isinstance(signal, CopilotToolBlockerSignal) and signal.blocker_kind == "loop_detected":
+ refresh_held_loop_blocker_evidence(ctx)
+ signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else signal
user_response = (
signal.user_facing_reason
if isinstance(signal, CopilotToolBlockerSignal)
@@ -2110,6 +2118,7 @@ async def _translate_to_agent_result(
ctx.last_workflow = last_workflow
ctx.last_workflow_yaml = last_workflow_yaml
ctx.last_test_ok = None
+ clear_terminal_evidence_on_workflow_edit(ctx)
# Inline REPLACE_WORKFLOW is untested by construction; emit a draft
# envelope without staging onto ctx so terminal auto-accept can't fire,
# and suppress the workflow payload so the canvas does not render it.
@@ -2714,6 +2723,9 @@ def _build_output_policy_blocked_result(
)
request_policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else None
add_saved_draft_copy = False
+ fallback_user_response: str | None = None
+ composed_from_recorded_evidence = False
+ evidence = terminal_evidence_from_ctx(ctx)
if (
request_policy is not None
and request_policy.clarification_question
@@ -2744,8 +2756,29 @@ def _build_output_policy_blocked_result(
"I could not safely return that chat reply, but the workflow draft is still saved. "
"Please review the draft or adjust the request and try again."
)
+ fallback_user_response = user_response
+ if terminal_evidence_has_recorded_state(evidence):
+ composed_response, tiers = compose_terminal_evidence_user_facing_reason(
+ "I could not safely return that chat reply.",
+ "Please review the recorded evidence or adjust the request and try again.",
+ evidence,
+ )
+ if any(tier != "draft" for tier in tiers):
+ user_response = composed_response
+ add_saved_draft_copy = "draft" in tiers
+ composed_from_recorded_evidence = True
else:
user_response = "I could not safely return that chat reply. Please adjust the request and try again."
+ fallback_user_response = user_response
+ if terminal_evidence_has_recorded_state(evidence):
+ composed_response, tiers = compose_terminal_evidence_user_facing_reason(
+ "I could not safely return that chat reply.",
+ "Please review the recorded evidence or adjust the request and try again.",
+ evidence,
+ )
+ if any(tier != "draft" for tier in tiers):
+ user_response = composed_response
+ composed_from_recorded_evidence = True
if preserved_workflow is not None and add_saved_draft_copy:
user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"
final_user_response, output_policy_outcome = apply_repeated_reply_guard(
@@ -2755,6 +2788,31 @@ def _build_output_policy_blocked_result(
reason_code="output_policy_block",
terminal_reason="output_policy_block",
)
+ if composed_from_recorded_evidence and fallback_user_response is not None:
+ composed_verdict = evaluate_output_policy(
+ request_policy=ctx.request_policy,
+ response_type="ASK_QUESTION",
+ user_response=final_user_response,
+ global_llm_context=None,
+ workflow_yaml=preserved_workflow_yaml or prior_workflow_yaml,
+ has_workflow_proposal=preserved_workflow is not None,
+ workflow_was_persisted=ctx.workflow_persisted,
+ workflow_attempted=bool(ctx.last_run_blocks_workflow_run_id),
+ unvalidated=ctx.last_test_ok is not True,
+ output_kind=verdict.output_kind,
+ )
+ if not composed_verdict.allowed:
+ LOG.warning(
+ "copilot output-policy recorded-evidence fallback failed output policy; using generic fallback",
+ output_policy_reasons=[code.value for code in composed_verdict.reason_codes],
+ )
+ final_user_response, output_policy_outcome = apply_repeated_reply_guard(
+ final_text=fallback_user_response,
+ attempted_kind=ResponseKind.CLARIFY,
+ blocked_signatures=ctx.blocked_reply_signatures,
+ reason_code="output_policy_block",
+ terminal_reason="output_policy_block",
+ )
return _make_agent_result(
ctx,
user_response=final_user_response,
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 2c3137183..13b4d6dd7 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -12,6 +12,7 @@ import structlog
from pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validator
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
BlockerKind = Literal[
"authority_denied",
@@ -159,14 +160,25 @@ def to_trace_data(signal: CopilotToolBlockerSignal) -> dict[str, Any]:
@dataclass(frozen=True)
-class LoopBlockerEvidence:
+class TerminalEvidence:
outcome_gate_reason: str | None = None
+ outcome_gate_workflow_run_id: str | None = None
+ recorded_run_outcome: RecordedRunOutcome | None = None
+ latest_workflow_run_id: str | None = None
anti_bot_blocked: bool = False
has_draft: bool = False
+@dataclass(frozen=True)
+class LoopBlockerEvidence(TerminalEvidence):
+ pass
+
+
class _LoopEvidenceCtx(Protocol):
last_outcome_gate_reason: str | None
+ last_outcome_gate_workflow_run_id: str | None
+ last_run_outcome: RecordedRunOutcome | None
+ last_run_blocks_workflow_run_id: str | None
last_test_anti_bot: str | None
staged_workflow: Any | None
staged_workflow_yaml: str | None
@@ -179,13 +191,61 @@ class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):
tool_blocker_signals: list[CopilotToolBlockerSignal]
+class _TerminalEvidenceResetCtx(Protocol):
+ last_run_blocks_workflow_run_id: str | None
+ last_successful_run_blocks_workflow_run_id: str | None
+ last_run_blocks_block_ids: list[str]
+ last_run_blocks_block_labels: list[str]
+ last_run_outcome: RecordedRunOutcome | None
+ last_run_outcome_block_labels: list[str]
+ last_outcome_gate_reason: str | None
+ last_outcome_gate_workflow_run_id: str | None
+ last_test_anti_bot: str | None
+ completion_verification_result: Any | None
+ outcome_verification_trace_snapshot: dict[str, Any]
+
+
+def terminal_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> TerminalEvidence:
+ recorded = getattr(ctx, "last_run_outcome", None)
+ return TerminalEvidence(
+ outcome_gate_reason=getattr(ctx, "last_outcome_gate_reason", None),
+ outcome_gate_workflow_run_id=getattr(ctx, "last_outcome_gate_workflow_run_id", None),
+ recorded_run_outcome=recorded if isinstance(recorded, RecordedRunOutcome) else None,
+ latest_workflow_run_id=getattr(ctx, "last_run_blocks_workflow_run_id", None),
+ anti_bot_blocked=bool(getattr(ctx, "last_test_anti_bot", None)),
+ has_draft=(
+ getattr(ctx, "staged_workflow", None) is not None
+ or getattr(ctx, "staged_workflow_yaml", None) is not None
+ or bool(getattr(ctx, "has_staged_proposal", False))
+ or getattr(ctx, "last_workflow", None) is not None
+ or getattr(ctx, "last_workflow_yaml", None) is not None
+ ),
+ )
+
+
+def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) -> None:
+ ctx.last_run_blocks_workflow_run_id = None
+ ctx.last_successful_run_blocks_workflow_run_id = None
+ ctx.last_run_blocks_block_ids = []
+ ctx.last_run_blocks_block_labels = []
+ ctx.last_run_outcome = None
+ ctx.last_run_outcome_block_labels = []
+ ctx.last_outcome_gate_reason = None
+ ctx.last_outcome_gate_workflow_run_id = None
+ ctx.last_test_anti_bot = None
+ ctx.completion_verification_result = None
+ ctx.outcome_verification_trace_snapshot = {}
+
+
def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence:
+ evidence = terminal_evidence_from_ctx(ctx)
return LoopBlockerEvidence(
- outcome_gate_reason=ctx.last_outcome_gate_reason,
- anti_bot_blocked=bool(ctx.last_test_anti_bot),
- has_draft=ctx.staged_workflow is not None
- or ctx.staged_workflow_yaml is not None
- or bool(ctx.has_staged_proposal),
+ outcome_gate_reason=evidence.outcome_gate_reason,
+ outcome_gate_workflow_run_id=evidence.outcome_gate_workflow_run_id,
+ recorded_run_outcome=evidence.recorded_run_outcome,
+ latest_workflow_run_id=evidence.latest_workflow_run_id,
+ anti_bot_blocked=evidence.anti_bot_blocked,
+ has_draft=evidence.has_draft,
)
@@ -317,6 +377,37 @@ _LOOP_VERDICT_RAW_ERROR_RE = re.compile(
)
+def _fresh_recorded_run_outcome(evidence: TerminalEvidence) -> RecordedRunOutcome | None:
+ recorded = evidence.recorded_run_outcome
+ if recorded is None or not recorded.workflow_run_id or not evidence.latest_workflow_run_id:
+ return None
+ if recorded.workflow_run_id != evidence.latest_workflow_run_id:
+ return None
+ return recorded
+
+
+def _recorded_run_outcome_tier(recorded: RecordedRunOutcome | None) -> str | None:
+ if recorded is None:
+ return None
+ if recorded.verdict == "demonstrated":
+ return "The latest run recorded a verified outcome."
+ if recorded.verdict == "not_demonstrated":
+ if recorded.reason_code == "blocker_reported":
+ return "The latest run recorded a blocker instead of a verified result."
+ if recorded.reason_code == "no_meaningful_output":
+ return "The latest run completed without meaningful output to return."
+ return "The latest run recorded workflow output, but it did not verify the requested outcome."
+ return "The latest run recorded workflow output, but it has not been verified end-to-end."
+
+
+def _fresh_outcome_gate_reason(evidence: TerminalEvidence) -> str | None:
+ if not evidence.latest_workflow_run_id or not evidence.outcome_gate_workflow_run_id:
+ return None
+ if evidence.latest_workflow_run_id != evidence.outcome_gate_workflow_run_id:
+ return None
+ return evidence.outcome_gate_reason
+
+
def _sanitize_loop_verdict_reason(reason: str | None) -> str | None:
if not reason:
return None
@@ -338,6 +429,51 @@ def _sanitize_loop_verdict_reason(reason: str | None) -> str | None:
return text
+def terminal_evidence_has_recorded_state(evidence: TerminalEvidence | None) -> bool:
+ if evidence is None:
+ return False
+ return bool(
+ _fresh_outcome_gate_reason(evidence)
+ or _fresh_recorded_run_outcome(evidence) is not None
+ or evidence.anti_bot_blocked
+ )
+
+
+def compose_terminal_evidence_user_facing_reason(
+ framing: str,
+ ask: str,
+ evidence: TerminalEvidence | None,
+ *,
+ blocked_tool: str | None = None,
+) -> tuple[str, tuple[str, ...]]:
+ template = f"{framing} {ask}"
+ draft_tier = ("draft",) if evidence is not None and evidence.has_draft else ()
+ if evidence is None:
+ return template, draft_tier
+
+ tier_candidates = (
+ ("recorded_run_outcome", _recorded_run_outcome_tier(_fresh_recorded_run_outcome(evidence))),
+ ("verdict", _sanitize_loop_verdict_reason(_fresh_outcome_gate_reason(evidence))),
+ ("anti_bot", _LOOP_ANTI_BOT_BLOCKER_COPY if evidence.anti_bot_blocked else None),
+ )
+ parts = [framing]
+ tiers: list[str] = []
+ for tier, text in tier_candidates:
+ if text is None:
+ continue
+ candidate = " ".join([*parts, text, ask])
+ try:
+ assert_clean_user_facing_text(candidate, blocked_tool=blocked_tool)
+ except ValueError:
+ continue
+ parts.append(text)
+ tiers.append(tier)
+ candidate = " ".join([*parts, ask])
+ if candidate == template:
+ return template, draft_tier
+ return candidate, (*tiers, *draft_tier)
+
+
def compose_loop_blocker_user_facing_reason(
internal_reason_code: str | None,
evidence: LoopBlockerEvidence | None,
@@ -350,30 +486,7 @@ def compose_loop_blocker_user_facing_reason(
if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":
return _LOOP_CREDENTIAL_TEMPLATE, draft_tier
framing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])
- template = f"{framing} {ask}"
- if evidence is None:
- return template, draft_tier
- verdict = _sanitize_loop_verdict_reason(evidence.outcome_gate_reason)
- blocker = _LOOP_ANTI_BOT_BLOCKER_COPY if evidence.anti_bot_blocked else None
- for include_verdict in (True, False):
- parts = [framing]
- tiers: list[str] = []
- if include_verdict and verdict is not None:
- parts.append(verdict)
- tiers.append("verdict")
- if blocker is not None:
- parts.append(blocker)
- tiers.append("anti_bot")
- parts.append(ask)
- candidate = " ".join(parts)
- if candidate == template:
- return template, draft_tier
- try:
- assert_clean_user_facing_text(candidate, blocked_tool=blocked_tool)
- except ValueError:
- continue
- return candidate, (*tiers, *draft_tier)
- return template, draft_tier
+ return compose_terminal_evidence_user_facing_reason(framing, ask, evidence, blocked_tool=blocked_tool)
def build_loop_blocker_signal(
@@ -416,14 +529,14 @@ def refresh_held_loop_blocker_evidence(ctx: _BlockerSignalCtx) -> None:
if not isinstance(held, CopilotToolBlockerSignal) or held.blocker_kind != "loop_detected":
return
evidence = loop_blocker_evidence_from_ctx(ctx)
- user_facing, tiers = compose_loop_blocker_user_facing_reason(
- held.internal_reason_code, evidence, blocked_tool=held.blocked_tool
- )
- if not tiers:
- return
- if user_facing == held.user_facing_reason and evidence.has_draft == held.preserves_workflow_draft:
- return
try:
+ user_facing, tiers = compose_loop_blocker_user_facing_reason(
+ held.internal_reason_code, evidence, blocked_tool=held.blocked_tool
+ )
+ if not tiers:
+ return
+ if user_facing == held.user_facing_reason and evidence.has_draft == held.preserves_workflow_draft:
+ return
refreshed = CopilotToolBlockerSignal(
blocker_kind=held.blocker_kind,
agent_steering_text=held.agent_steering_text,
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 7dfaccad0..4a8dc93ab 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -424,6 +424,7 @@ class CopilotContext(AgentContext):
# after a watchdog reconciliation read has cleared the retry guard.
last_run_blocks_workflow_run_id: str | None = None
last_successful_run_blocks_workflow_run_id: str | None = None
+ last_outcome_gate_workflow_run_id: str | None = None
# Consecutive test runs whose data-producing blocks completed with no
# meaningful output (missing, empty, or all-null fields). Resets when a
# run produces real data. Used to escalate when the agent is stuck
diff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.py
index 246a525c0..798197de3 100644
--- a/skyvern/forge/sdk/copilot/run_outcome.py
+++ b/skyvern/forge/sdk/copilot/run_outcome.py
@@ -18,6 +18,7 @@ class RecordedRunOutcome:
verdict: RecordedRunOutcomeVerdict
reason_code: RunOutcomeReasonCode | None = None
display_reason: str | None = None
+ workflow_run_id: str | None = None
def run_outcome_display_reason(text: str | None) -> str | None:
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 2bff031d6..3ae092218 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -184,6 +184,7 @@ class AgentContext:
# Latest evaluated outcome-gate verdict this turn. Deliberately not reset
# per-run: a later run that fails before verification keeps the verdict.
last_outcome_gate_reason: str | None = None
+ last_outcome_gate_workflow_run_id: str | None = None
last_failure_category_top: str | None = None
last_update_block_count: int | None = None
last_failed_workflow_yaml: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 9afc7145c..6f7b3ea60 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -6,6 +6,7 @@ import json
import time
from collections import defaultdict
from collections.abc import Mapping
+from dataclasses import replace
from datetime import datetime, timezone
from typing import Any, Literal
from urllib.parse import urlparse
@@ -1433,6 +1434,7 @@ def _record_run_blocks_result(
copilot_ctx.last_test_failure_reason = None
if completion_verification is not None and completion_verification.status == "evaluated":
copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_id
copilot_ctx.last_test_suspicious_success = False
copilot_ctx.last_run_outcome = None
copilot_ctx.last_run_outcome_block_labels = []
@@ -1618,6 +1620,8 @@ def _record_run_blocks_result(
def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:
+ if outcome.workflow_run_id is None:
+ outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))
copilot_ctx.last_run_outcome = outcome
copilot_ctx.last_run_outcome_block_labels = list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or [])
return outcome
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 5376ec6ef..e7e728135 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -14,6 +14,7 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationError
from skyvern.forge import app
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
+from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_edit
from skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_diagnostics
from skyvern.forge.sdk.copilot.code_block_synthesis import (
artifact_dependency_id,
@@ -1643,6 +1644,7 @@ def _record_workflow_update_result(
copilot_ctx.last_update_block_count = block_count
copilot_ctx.last_test_ok = None
copilot_ctx.last_test_failure_reason = None
+ clear_terminal_evidence_on_workflow_edit(copilot_ctx)
# A fresh workflow edit invalidates the prior test's failure state —
# otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise
# CopilotNonRetriableNavError with the old run's error, telling the user
diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.py
index 65219931f..75780d9f8 100644
--- a/tests/unit/test_copilot_agent_blocker_renderer.py
+++ b/tests/unit/test_copilot_agent_blocker_renderer.py
@@ -5,8 +5,12 @@ from typing import Any, get_args
import pytest
+from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.agent import (
_FALLBACK_BLOCKER_REPLY,
+ _RAW_SECRET_LEAK_REFUSAL,
+ _build_output_policy_blocked_result,
+ _build_turn_halt_exit_result,
_finalize_result_with_blocker_override,
_render_blocker_reply,
)
@@ -17,6 +21,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
RecoveryHint,
)
from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext
+from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind
# Source-of-truth deny list lives in blocker_signal.py. Re-importing here
# (instead of hand-copying) guarantees the test stays in sync if a new token
@@ -69,6 +77,38 @@ def _agent_result(user_response: str = "Agent prose reply with leaked TurnIntent
)
+def _blocked_result(
+ ctx: CopilotContext,
+ *reason_codes: OutputPolicyReason,
+ output_kind: CopilotOutputKind = CopilotOutputKind.REFUSAL,
+) -> AgentResult:
+ return _build_output_policy_blocked_result(
+ ctx,
+ OutputPolicyVerdict(
+ allowed=False,
+ output_kind=output_kind,
+ reason_codes=list(reason_codes),
+ ),
+ prior_global_llm_context=None,
+ prior_workflow_yaml=None,
+ )
+
+
+def _seed_terminal_evidence(ctx: CopilotContext, run_id: str = "wr_latest") -> None:
+ ctx.last_run_blocks_workflow_run_id = run_id
+ ctx.last_run_outcome = RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ display_reason="The requested record was not verified.",
+ workflow_run_id=run_id,
+ )
+ ctx.last_outcome_gate_reason = (
+ "Failed: The run completed but did not demonstrate the goal outcome(s): "
+ "the requested record was not verified. Add an end-state confirmation, then re-run."
+ )
+ ctx.last_outcome_gate_workflow_run_id = run_id
+
+
@pytest.mark.parametrize("recovery_hint", _ALL_RECOVERY_HINTS)
def test_render_picks_response_type_from_hint(recovery_hint: RecoveryHint) -> None:
signal = _signal(recovery_hint=recovery_hint)
@@ -178,13 +218,6 @@ def test_output_policy_blocked_result_zeroes_proposal_when_blocker_active() -> N
"""OutputPolicy hard-block must not surface a workflow proposal when a
blocker is set, even though the shim is intentionally skipped on that path.
"""
- from skyvern.forge.sdk.copilot.agent import _build_output_policy_blocked_result
- from skyvern.forge.sdk.copilot.output_policy import (
- CopilotOutputKind,
- OutputPolicyReason,
- OutputPolicyVerdict,
- )
-
ctx = _ctx()
# Mock a workflow on ctx to confirm the builder zeros it.
fake_workflow: Any = object()
@@ -192,22 +225,102 @@ def test_output_policy_blocked_result_zeroes_proposal_when_blocker_active() -> N
ctx.last_workflow_yaml = "title: X\nworkflow_definition:\n blocks: []\n"
ctx.blocker_signal = _signal(internal_reason_code="tool_error_pending_reconciliation_no_input")
- verdict = OutputPolicyVerdict(
- allowed=False,
- output_kind=CopilotOutputKind.REFUSAL,
- reason_codes=[OutputPolicyReason.RAW_SECRET_LEAK],
- )
- result = _build_output_policy_blocked_result(
- ctx,
- verdict,
- prior_global_llm_context=None,
- prior_workflow_yaml=None,
- )
+ result = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK)
assert result.updated_workflow is None
assert result.workflow_yaml is None
assert result.proposal_disposition == "no_proposal"
+def test_output_policy_generic_block_uses_recorded_terminal_evidence() -> None:
+ ctx = _ctx()
+ fake_workflow: Any = object()
+ ctx.last_workflow = fake_workflow
+ ctx.last_workflow_yaml = "title: Draft\n"
+ ctx.workflow_persisted = True
+ ctx.last_test_ok = True
+ _seed_terminal_evidence(ctx)
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+
+ result = _blocked_result(ctx, OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)
+
+ assert result.response_type == "ASK_QUESTION"
+ assert result.updated_workflow is fake_workflow
+ assert result.proposal_disposition == "review_tested"
+ assert "latest run recorded workflow output" in result.user_response
+ assert "did not demonstrate the goal outcome" in result.user_response
+ assert "verification challenge" in result.user_response
+ assert "workflow draft is still saved" in result.user_response
+ assert "wr_latest" not in result.user_response
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["terminalMessage"] == result.user_response
+ assert result.narrative_payload["responseType"] == "ASK_QUESTION"
+
+
+def test_output_policy_recorded_evidence_recheck_keeps_original_output_kind(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _ctx()
+ _seed_terminal_evidence(ctx)
+ seen_output_kinds: list[CopilotOutputKind] = []
+
+ def allow_policy(**kwargs: Any) -> OutputPolicyVerdict:
+ seen_output_kinds.append(kwargs["output_kind"])
+ return OutputPolicyVerdict(allowed=True, output_kind=kwargs["output_kind"], reason_codes=[])
+
+ monkeypatch.setattr(agent_module, "evaluate_output_policy", allow_policy)
+
+ result = _blocked_result(
+ ctx,
+ OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,
+ output_kind=CopilotOutputKind.REFUSAL,
+ )
+
+ assert result.response_type == "ASK_QUESTION"
+ assert seen_output_kinds == [CopilotOutputKind.REFUSAL]
+
+
+def test_output_policy_generic_block_requires_clean_terminal_evidence() -> None:
+ no_recorded = _ctx()
+ adversarial = _ctx()
+ adversarial.last_run_blocks_workflow_run_id = "wr_hidden"
+ adversarial.last_outcome_gate_reason = "update_and_run_blocks failed for wr_hidden; do not retry this step."
+ adversarial.last_outcome_gate_workflow_run_id = "wr_hidden"
+
+ for ctx in (no_recorded, adversarial):
+ result = _blocked_result(ctx, OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)
+ assert (
+ result.user_response
+ == "I could not safely return that chat reply. Please adjust the request and try again."
+ )
+ assert result.response_type == "ASK_QUESTION"
+ assert "update_and_run_blocks" not in result.user_response
+ assert "wr_hidden" not in result.user_response
+ assert "do not retry" not in result.user_response.lower()
+
+
+def test_output_policy_specific_branches_bypass_recorded_terminal_fallback() -> None:
+ ctx = _ctx()
+ _seed_terminal_evidence(ctx)
+ raw_secret = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK)
+ assert raw_secret.user_response == _RAW_SECRET_LEAK_REFUSAL
+ assert "latest run" not in raw_secret.user_response.lower()
+
+ clarification_ctx = _ctx()
+ clarification_ctx.request_policy = RequestPolicy(
+ user_response_policy="ask_clarification",
+ clarification_question="Please confirm which saved credential should be used.",
+ )
+ result = _blocked_result(
+ clarification_ctx,
+ OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,
+ output_kind=CopilotOutputKind.CLARIFICATION_REQUEST,
+ )
+
+ assert result.response_type == "ASK_QUESTION"
+ assert result.user_response == "Please confirm which saved credential should be used."
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["terminalMessage"] == result.user_response
+ assert result.narrative_payload["responseType"] == "ASK_QUESTION"
+
+
def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None:
"""Late-block-running blockers say 'wrap up with what you have' — they
surface the saved draft alongside the rendered chat reply."""
@@ -361,3 +474,19 @@ def test_output_policy_blocked_result_surfaces_workflow_when_no_blocker() -> Non
)
assert result.updated_workflow is fake_workflow
assert result.workflow_yaml == ctx.last_workflow_yaml
+
+
+def test_turn_halt_exit_keeps_halt_signal_when_context_signal_is_cleared() -> None:
+ ctx = _ctx()
+ signal = _signal(
+ kind="loop_detected",
+ user_facing="I'm stuck retrying the same step. Tell me what to change and I'll try a different approach.",
+ internal_reason_code="loop_detected_repeated_failed_step",
+ blocked_tool="run_blocks_and_collect_debug",
+ )
+ halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)
+ ctx.blocker_signal = None
+
+ result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ assert result.user_response == signal.user_facing_reason
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
index 188cb8a43..a45a2e21b 100644
--- a/tests/unit/test_copilot_blocker_signal.py
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -349,6 +349,8 @@ def test_stash_refreshes_held_loop_signal_with_evidence_recorded_after_the_stash
assert dict(loop_signal.extra) == {}
ctx.last_outcome_gate_reason = _LATE_RECORDED_REASON
+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"
+ ctx.last_run_blocks_workflow_run_id = "wr_latest"
ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
ctx.has_staged_proposal = True
diff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.py
index c46987a81..6bcf17a44 100644
--- a/tests/unit/test_copilot_frontier.py
+++ b/tests/unit/test_copilot_frontier.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.copilot.output_utils import (
sanitize_tool_result_for_llm,
summarize_tool_result,
)
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import (
_find_invalidated_labels,
_frontier_run_size_error,
@@ -1657,6 +1658,39 @@ def test_chokepoint_uses_passed_prior_when_last_workflow_absent() -> None:
assert "extract" in tools._unverified_current_workflow_labels(ctx)
+def test_workflow_update_clears_terminal_run_evidence() -> None:
+ new = _wf_def(
+ ("open", "goto_url", {"url": "https://example.com"}),
+ ("extract", "extraction", {"prompt": "extract CHANGED"}),
+ )
+ ctx = _make_ctx()
+ ctx.last_run_blocks_workflow_run_id = "wr_old"
+ ctx.last_successful_run_blocks_workflow_run_id = "wr_old"
+ ctx.last_run_blocks_block_ids = ["wrb_old"]
+ ctx.last_run_blocks_block_labels = ["extract"]
+ ctx.last_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_old")
+ ctx.last_run_outcome_block_labels = ["extract"]
+ ctx.last_outcome_gate_reason = "The prior run did not demonstrate the goal."
+ ctx.last_outcome_gate_workflow_run_id = "wr_old"
+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
+ ctx.completion_verification_result = object() # type: ignore[assignment]
+ ctx.outcome_verification_trace_snapshot = {"old": True}
+
+ _record_workflow_update_result(ctx, {"ok": True, "_workflow": _FakeWorkflow(new)}, None)
+
+ assert ctx.last_run_blocks_workflow_run_id is None
+ assert ctx.last_successful_run_blocks_workflow_run_id is None
+ assert ctx.last_run_blocks_block_ids == []
+ assert ctx.last_run_blocks_block_labels == []
+ assert ctx.last_run_outcome is None
+ assert ctx.last_run_outcome_block_labels == []
+ assert ctx.last_outcome_gate_reason is None
+ assert ctx.last_outcome_gate_workflow_run_id is None
+ assert ctx.last_test_anti_bot is None
+ assert ctx.completion_verification_result is None
+ assert ctx.outcome_verification_trace_snapshot == {}
+
+
def test_unavailable_prior_with_trust_fails_closed() -> None:
new = _wf_def(
("open", "goto_url", {"url": "https://example.com"}),
diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.py
index 40ce0a314..1808d83eb 100644
--- a/tests/unit/test_copilot_loop_detection_blocker.py
+++ b/tests/unit/test_copilot_loop_detection_blocker.py
@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer, _stash_and_emit_loop_blocker
from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_error
from skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_map
from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
@@ -161,6 +162,8 @@ _FULL_EVIDENCE_REASON = (
)
_FULL_EVIDENCE = LoopBlockerEvidence(
outcome_gate_reason=_FULL_EVIDENCE_REASON,
+ outcome_gate_workflow_run_id="wr_latest",
+ latest_workflow_run_id="wr_latest",
anti_bot_blocked=True,
has_draft=True,
)
@@ -286,6 +289,50 @@ def test_raw_runtime_error_reason_drops_verdict_tier(raw_error_reason: str) -> N
assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")
+ "recorded,latest_run_id,expected_tiers,expect_recorded_tier",
+ [
+ (
+ RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_new"),
+ "wr_new",
+ ["recorded_run_outcome", "draft"],
+ True,
+ ),
+ (None, None, ["draft"], False),
+ (
+ RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),
+ "wr_new",
+ ["draft"],
+ False,
+ ),
+ ],
+)
+def test_stale_outcome_gate_reason_only_keeps_matching_recorded_run(
+ recorded: RecordedRunOutcome | None,
+ latest_run_id: str | None,
+ expected_tiers: list[str],
+ expect_recorded_tier: bool,
+) -> None:
+ evidence = LoopBlockerEvidence(
+ outcome_gate_reason=(
+ "The run completed but did not demonstrate the goal outcome(s): stale criterion text from an older run."
+ ),
+ outcome_gate_workflow_run_id="wr_old",
+ recorded_run_outcome=recorded,
+ latest_workflow_run_id=latest_run_id,
+ has_draft=True,
+ )
+
+ signal = _build_loop_blocker_signal(
+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=evidence
+ )
+
+ assert "stale criterion text" not in signal.user_facing_reason
+ assert ("latest run recorded workflow output" in signal.user_facing_reason) is expect_recorded_tier
+ assert dict(signal.extra) == {"loop_evidence_tiers": expected_tiers}
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")
+
+
def test_evidence_verdict_sources_only_from_the_outcome_gate_field() -> None:
ctx = _ctx()
ctx.last_test_failure_reason = "Failed to execute code block. Reason: TimeoutError: Timeout 30000ms exceeded."
@@ -334,6 +381,8 @@ def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> Non
"The run completed but did not demonstrate the goal outcome(s): the requested record is checked "
"on a public registry site with a search form and expandable result rows."
)
+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"
+ ctx.last_run_blocks_workflow_run_id = "wr_latest"
ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
ctx.has_staged_proposal = True
return ctx
@@ -370,6 +419,8 @@ def test_tool_loop_error_entry_refreshes_stale_held_loop_signal() -> None:
ctx.last_outcome_gate_reason = (
"The run completed but did not demonstrate the goal outcome(s): the requested record is checked."
)
+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"
+ ctx.last_run_blocks_workflow_run_id = "wr_latest"
ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"
ctx.staged_workflow_yaml = "blocks: []"
@@ -425,7 +476,13 @@ def test_composed_loop_reply_passes_output_policy_allow_verdict() -> None:
signal = _build_loop_blocker_signal(
_BRANCH_MESSAGES["loop_detected_consecutive_same_tool"],
tool_name="evaluate",
- evidence=LoopBlockerEvidence(outcome_gate_reason=_FULL_EVIDENCE_REASON, anti_bot_blocked=True, has_draft=False),
+ evidence=LoopBlockerEvidence(
+ outcome_gate_reason=_FULL_EVIDENCE_REASON,
+ outcome_gate_workflow_run_id="wr_latest",
+ latest_workflow_run_id="wr_latest",
+ anti_bot_blocked=True,
+ has_draft=False,
+ ),
)
verdict = evaluate_output_policy(
request_policy=None,
diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.py
index 3dddbee85..a68724719 100644
--- a/tests/unit/test_copilot_run_outcome_event.py
+++ b/tests/unit/test_copilot_run_outcome_event.py
@@ -133,7 +133,10 @@ async def test_blocker_run_emits_hold_then_not_demonstrated() -> None:
assert final.display_reason is not None and "human verification challenge" in final.display_reason
assert ctx.last_test_suspicious_success is True
assert ctx.last_run_outcome == RecordedRunOutcome(
- verdict=final.verdict, reason_code=final.reason_code, display_reason=final.display_reason
+ verdict=final.verdict,
+ reason_code=final.reason_code,
+ display_reason=final.display_reason,
+ workflow_run_id="wr_test",
)
assert ctx.last_run_outcome_block_labels == final.block_labels
@@ -288,6 +291,20 @@ def test_failed_rerun_clears_prior_recorded_outcome() -> None:
assert ctx.last_run_outcome_block_labels == []
+def test_recorded_run_outcome_carries_producing_workflow_run_id() -> None:
+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])
+ outcome = _record_run_blocks_result(
+ ctx,
+ _run_result([_code_block("search_registry_person", {"records": []})]),
+ completion_verification=_evaluated(satisfied=False),
+ )
+
+ assert outcome is not None
+ assert outcome.workflow_run_id == "wr_test"
+ assert ctx.last_run_outcome is not None
+ assert ctx.last_run_outcome.workflow_run_id == "wr_test"
+
+
def test_both_consumers_route_through_single_producer() -> None:
source = inspect.getsource(copilot_tools)
assert source.count("await _verify_and_record_run_blocks_result(") == 2
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index 776e6e477..ed23f2aff 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -36,6 +36,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import (
)
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatRequest,
+ WorkflowCopilotChatSender,
WorkflowCopilotStreamErrorUpdate,
WorkflowCopilotStreamResponseUpdate,
)
@@ -913,13 +914,15 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_au
description="Original description",
workflow_definition=None,
)
+ terminal_message = "I could not safely return that chat reply."
agent_result = SimpleNamespace(
- user_response="I could not safely return that chat reply.",
+ user_response=terminal_message,
updated_workflow=None,
global_llm_context=None,
workflow_yaml=None,
workflow_was_persisted=False,
clear_proposed_workflow=False,
+ response_type="ASK_QUESTION",
unvalidated=False,
output_policy_diagnostics={
"final_output_policy_allowed": False,
@@ -949,6 +952,23 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_au
clear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]
assert not clear_calls, f"did not expect a clear call, got {update_calls!r}"
+ assistant_call = next(
+ call
+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list
+ if call.kwargs.get("sender") == WorkflowCopilotChatSender.AI
+ )
+ assert assistant_call.kwargs["content"] == terminal_message
+
+ response_frames = [
+ call.args[0]
+ for call in stream.send.await_args_list
+ if isinstance(call.args[0], WorkflowCopilotStreamResponseUpdate)
+ ]
+ assert len(response_frames) == 1
+ frame = response_frames[0]
+ assert frame.message == terminal_message
+ assert frame.response_type == "ASK_QUESTION"
+
@pytest.mark.asyncio
async def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest.MonkeyPatch) -> None: