Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10910 Preserve terminal evidence in Copilot hard blocks (#6579)
Andrew Neilson338c91b61183
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex c14631657..8e5fec877 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -36,7 +36,12 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,assert_clean_user_facing_text,+ clear_terminal_evidence_on_workflow_edit,+ compose_terminal_evidence_user_facing_reason,contains_internal_machinery_leak,+ refresh_held_loop_blocker_evidence,+ terminal_evidence_from_ctx,+ terminal_evidence_has_recorded_state,)from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_datafrom skyvern.forge.sdk.copilot.build_phase import initial_build_phase@@ -1241,6 +1246,9 @@ def _build_turn_halt_exit_result(halt: TurnHalt,) -> AgentResult:signal = halt.blocker_signal+ if isinstance(signal, CopilotToolBlockerSignal) and signal.blocker_kind == "loop_detected":+ refresh_held_loop_blocker_evidence(ctx)+ signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else signaluser_response = (signal.user_facing_reasonif isinstance(signal, CopilotToolBlockerSignal)@@ -2110,6 +2118,7 @@ async def _translate_to_agent_result(ctx.last_workflow = last_workflowctx.last_workflow_yaml = last_workflow_yamlctx.last_test_ok = None+ clear_terminal_evidence_on_workflow_edit(ctx)# Inline REPLACE_WORKFLOW is untested by construction; emit a draft# envelope without staging onto ctx so terminal auto-accept can't fire,# and suppress the workflow payload so the canvas does not render it.@@ -2714,6 +2723,9 @@ def _build_output_policy_blocked_result()request_policy = ctx.request_policy if isinstance(ctx.request_policy, RequestPolicy) else Noneadd_saved_draft_copy = False+ fallback_user_response: str | None = None+ composed_from_recorded_evidence = False+ evidence = terminal_evidence_from_ctx(ctx)if (request_policy is not Noneand request_policy.clarification_question@@ -2744,8 +2756,29 @@ def _build_output_policy_blocked_result("I could not safely return that chat reply, but the workflow draft is still saved. ""Please review the draft or adjust the request and try again.")+ fallback_user_response = user_response+ if terminal_evidence_has_recorded_state(evidence):+ composed_response, tiers = compose_terminal_evidence_user_facing_reason(+ "I could not safely return that chat reply.",+ "Please review the recorded evidence or adjust the request and try again.",+ evidence,+ )+ if any(tier != "draft" for tier in tiers):+ user_response = composed_response+ add_saved_draft_copy = "draft" in tiers+ composed_from_recorded_evidence = Trueelse:user_response = "I could not safely return that chat reply. Please adjust the request and try again."+ fallback_user_response = user_response+ if terminal_evidence_has_recorded_state(evidence):+ composed_response, tiers = compose_terminal_evidence_user_facing_reason(+ "I could not safely return that chat reply.",+ "Please review the recorded evidence or adjust the request and try again.",+ evidence,+ )+ if any(tier != "draft" for tier in tiers):+ user_response = composed_response+ composed_from_recorded_evidence = Trueif preserved_workflow is not None and add_saved_draft_copy:user_response = f"{user_response} {_SAVED_DRAFT_OUTPUT_POLICY_SUFFIX}"final_user_response, output_policy_outcome = apply_repeated_reply_guard(@@ -2755,6 +2788,31 @@ def _build_output_policy_blocked_result(reason_code="output_policy_block",terminal_reason="output_policy_block",)+ if composed_from_recorded_evidence and fallback_user_response is not None:+ composed_verdict = evaluate_output_policy(+ request_policy=ctx.request_policy,+ response_type="ASK_QUESTION",+ user_response=final_user_response,+ global_llm_context=None,+ workflow_yaml=preserved_workflow_yaml or prior_workflow_yaml,+ has_workflow_proposal=preserved_workflow is not None,+ workflow_was_persisted=ctx.workflow_persisted,+ workflow_attempted=bool(ctx.last_run_blocks_workflow_run_id),+ unvalidated=ctx.last_test_ok is not True,+ output_kind=verdict.output_kind,+ )+ if not composed_verdict.allowed:+ LOG.warning(+ "copilot output-policy recorded-evidence fallback failed output policy; using generic fallback",+ output_policy_reasons=[code.value for code in composed_verdict.reason_codes],+ )+ final_user_response, output_policy_outcome = apply_repeated_reply_guard(+ final_text=fallback_user_response,+ attempted_kind=ResponseKind.CLARIFY,+ blocked_signatures=ctx.blocked_reply_signatures,+ reason_code="output_policy_block",+ terminal_reason="output_policy_block",+ )return _make_agent_result(ctx,user_response=final_user_response,diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 2c3137183..13b4d6dd7 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -12,6 +12,7 @@ import structlogfrom pydantic import BaseModel, ConfigDict, Field, field_serializer, model_validatorfrom skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomeBlockerKind = Literal["authority_denied",@@ -159,14 +160,25 @@ def to_trace_data(signal: CopilotToolBlockerSignal) -> dict[str, Any]:@dataclass(frozen=True)-class LoopBlockerEvidence:+class TerminalEvidence:outcome_gate_reason: str | None = None+ outcome_gate_workflow_run_id: str | None = None+ recorded_run_outcome: RecordedRunOutcome | None = None+ latest_workflow_run_id: str | None = Noneanti_bot_blocked: bool = Falsehas_draft: bool = False+@dataclass(frozen=True)+class LoopBlockerEvidence(TerminalEvidence):+ pass++class _LoopEvidenceCtx(Protocol):last_outcome_gate_reason: str | None+ last_outcome_gate_workflow_run_id: str | None+ last_run_outcome: RecordedRunOutcome | None+ last_run_blocks_workflow_run_id: str | Nonelast_test_anti_bot: str | Nonestaged_workflow: Any | Nonestaged_workflow_yaml: str | None@@ -179,13 +191,61 @@ class _BlockerSignalCtx(_LoopEvidenceCtx, Protocol):tool_blocker_signals: list[CopilotToolBlockerSignal]+class _TerminalEvidenceResetCtx(Protocol):+ last_run_blocks_workflow_run_id: str | None+ last_successful_run_blocks_workflow_run_id: str | None+ last_run_blocks_block_ids: list[str]+ last_run_blocks_block_labels: list[str]+ last_run_outcome: RecordedRunOutcome | None+ last_run_outcome_block_labels: list[str]+ last_outcome_gate_reason: str | None+ last_outcome_gate_workflow_run_id: str | None+ last_test_anti_bot: str | None+ completion_verification_result: Any | None+ outcome_verification_trace_snapshot: dict[str, Any]+++def terminal_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> TerminalEvidence:+ recorded = getattr(ctx, "last_run_outcome", None)+ return TerminalEvidence(+ outcome_gate_reason=getattr(ctx, "last_outcome_gate_reason", None),+ outcome_gate_workflow_run_id=getattr(ctx, "last_outcome_gate_workflow_run_id", None),+ recorded_run_outcome=recorded if isinstance(recorded, RecordedRunOutcome) else None,+ latest_workflow_run_id=getattr(ctx, "last_run_blocks_workflow_run_id", None),+ anti_bot_blocked=bool(getattr(ctx, "last_test_anti_bot", None)),+ has_draft=(+ getattr(ctx, "staged_workflow", None) is not None+ or getattr(ctx, "staged_workflow_yaml", None) is not None+ or bool(getattr(ctx, "has_staged_proposal", False))+ or getattr(ctx, "last_workflow", None) is not None+ or getattr(ctx, "last_workflow_yaml", None) is not None+ ),+ )+++def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) -> None:+ ctx.last_run_blocks_workflow_run_id = None+ ctx.last_successful_run_blocks_workflow_run_id = None+ ctx.last_run_blocks_block_ids = []+ ctx.last_run_blocks_block_labels = []+ ctx.last_run_outcome = None+ ctx.last_run_outcome_block_labels = []+ ctx.last_outcome_gate_reason = None+ ctx.last_outcome_gate_workflow_run_id = None+ ctx.last_test_anti_bot = None+ ctx.completion_verification_result = None+ ctx.outcome_verification_trace_snapshot = {}++def loop_blocker_evidence_from_ctx(ctx: _LoopEvidenceCtx) -> LoopBlockerEvidence:+ evidence = terminal_evidence_from_ctx(ctx)return LoopBlockerEvidence(- outcome_gate_reason=ctx.last_outcome_gate_reason,- anti_bot_blocked=bool(ctx.last_test_anti_bot),- has_draft=ctx.staged_workflow is not None- or ctx.staged_workflow_yaml is not None- or bool(ctx.has_staged_proposal),+ outcome_gate_reason=evidence.outcome_gate_reason,+ outcome_gate_workflow_run_id=evidence.outcome_gate_workflow_run_id,+ recorded_run_outcome=evidence.recorded_run_outcome,+ latest_workflow_run_id=evidence.latest_workflow_run_id,+ anti_bot_blocked=evidence.anti_bot_blocked,+ has_draft=evidence.has_draft,)@@ -317,6 +377,37 @@ _LOOP_VERDICT_RAW_ERROR_RE = re.compile()+def _fresh_recorded_run_outcome(evidence: TerminalEvidence) -> RecordedRunOutcome | None:+ recorded = evidence.recorded_run_outcome+ if recorded is None or not recorded.workflow_run_id or not evidence.latest_workflow_run_id:+ return None+ if recorded.workflow_run_id != evidence.latest_workflow_run_id:+ return None+ return recorded+++def _recorded_run_outcome_tier(recorded: RecordedRunOutcome | None) -> str | None:+ if recorded is None:+ return None+ if recorded.verdict == "demonstrated":+ return "The latest run recorded a verified outcome."+ if recorded.verdict == "not_demonstrated":+ if recorded.reason_code == "blocker_reported":+ return "The latest run recorded a blocker instead of a verified result."+ if recorded.reason_code == "no_meaningful_output":+ return "The latest run completed without meaningful output to return."+ return "The latest run recorded workflow output, but it did not verify the requested outcome."+ return "The latest run recorded workflow output, but it has not been verified end-to-end."+++def _fresh_outcome_gate_reason(evidence: TerminalEvidence) -> str | None:+ if not evidence.latest_workflow_run_id or not evidence.outcome_gate_workflow_run_id:+ return None+ if evidence.latest_workflow_run_id != evidence.outcome_gate_workflow_run_id:+ return None+ return evidence.outcome_gate_reason++def _sanitize_loop_verdict_reason(reason: str | None) -> str | None:if not reason:return None@@ -338,6 +429,51 @@ def _sanitize_loop_verdict_reason(reason: str | None) -> str | None:return text+def terminal_evidence_has_recorded_state(evidence: TerminalEvidence | None) -> bool:+ if evidence is None:+ return False+ return bool(+ _fresh_outcome_gate_reason(evidence)+ or _fresh_recorded_run_outcome(evidence) is not None+ or evidence.anti_bot_blocked+ )+++def compose_terminal_evidence_user_facing_reason(+ framing: str,+ ask: str,+ evidence: TerminalEvidence | None,+ *,+ blocked_tool: str | None = None,+) -> tuple[str, tuple[str, ...]]:+ template = f"{framing} {ask}"+ draft_tier = ("draft",) if evidence is not None and evidence.has_draft else ()+ if evidence is None:+ return template, draft_tier++ tier_candidates = (+ ("recorded_run_outcome", _recorded_run_outcome_tier(_fresh_recorded_run_outcome(evidence))),+ ("verdict", _sanitize_loop_verdict_reason(_fresh_outcome_gate_reason(evidence))),+ ("anti_bot", _LOOP_ANTI_BOT_BLOCKER_COPY if evidence.anti_bot_blocked else None),+ )+ parts = [framing]+ tiers: list[str] = []+ for tier, text in tier_candidates:+ if text is None:+ continue+ candidate = " ".join([*parts, text, ask])+ try:+ assert_clean_user_facing_text(candidate, blocked_tool=blocked_tool)+ except ValueError:+ continue+ parts.append(text)+ tiers.append(tier)+ candidate = " ".join([*parts, ask])+ if candidate == template:+ return template, draft_tier+ return candidate, (*tiers, *draft_tier)++def compose_loop_blocker_user_facing_reason(internal_reason_code: str | None,evidence: LoopBlockerEvidence | None,@@ -350,30 +486,7 @@ def compose_loop_blocker_user_facing_reason(if internal_reason_code == "loop_detected_credential_or_parameter_misconfig":return _LOOP_CREDENTIAL_TEMPLATE, draft_tierframing, ask = _LOOP_BRANCH_COPY.get(internal_reason_code or "", _LOOP_BRANCH_COPY["loop_detected_generic"])- template = f"{framing} {ask}"- if evidence is None:- return template, draft_tier- verdict = _sanitize_loop_verdict_reason(evidence.outcome_gate_reason)- blocker = _LOOP_ANTI_BOT_BLOCKER_COPY if evidence.anti_bot_blocked else None- for include_verdict in (True, False):- parts = [framing]- tiers: list[str] = []- if include_verdict and verdict is not None:- parts.append(verdict)- tiers.append("verdict")- if blocker is not None:- parts.append(blocker)- tiers.append("anti_bot")- parts.append(ask)- candidate = " ".join(parts)- if candidate == template:- return template, draft_tier- try:- assert_clean_user_facing_text(candidate, blocked_tool=blocked_tool)- except ValueError:- continue- return candidate, (*tiers, *draft_tier)- return template, draft_tier+ return compose_terminal_evidence_user_facing_reason(framing, ask, evidence, blocked_tool=blocked_tool)def build_loop_blocker_signal(@@ -416,14 +529,14 @@ def refresh_held_loop_blocker_evidence(ctx: _BlockerSignalCtx) -> None:if not isinstance(held, CopilotToolBlockerSignal) or held.blocker_kind != "loop_detected":returnevidence = loop_blocker_evidence_from_ctx(ctx)- user_facing, tiers = compose_loop_blocker_user_facing_reason(- held.internal_reason_code, evidence, blocked_tool=held.blocked_tool- )- if not tiers:- return- if user_facing == held.user_facing_reason and evidence.has_draft == held.preserves_workflow_draft:- returntry:+ user_facing, tiers = compose_loop_blocker_user_facing_reason(+ held.internal_reason_code, evidence, blocked_tool=held.blocked_tool+ )+ if not tiers:+ return+ if user_facing == held.user_facing_reason and evidence.has_draft == held.preserves_workflow_draft:+ returnrefreshed = CopilotToolBlockerSignal(blocker_kind=held.blocker_kind,agent_steering_text=held.agent_steering_text,diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 7dfaccad0..4a8dc93ab 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -424,6 +424,7 @@ class CopilotContext(AgentContext):# after a watchdog reconciliation read has cleared the retry guard.last_run_blocks_workflow_run_id: str | None = Nonelast_successful_run_blocks_workflow_run_id: str | None = None+ last_outcome_gate_workflow_run_id: str | None = None# Consecutive test runs whose data-producing blocks completed with no# meaningful output (missing, empty, or all-null fields). Resets when a# run produces real data. Used to escalate when the agent is stuckdiff --git a/skyvern/forge/sdk/copilot/run_outcome.py b/skyvern/forge/sdk/copilot/run_outcome.pyindex 246a525c0..798197de3 100644--- a/skyvern/forge/sdk/copilot/run_outcome.py+++ b/skyvern/forge/sdk/copilot/run_outcome.py@@ -18,6 +18,7 @@ class RecordedRunOutcome:verdict: RecordedRunOutcomeVerdictreason_code: RunOutcomeReasonCode | None = Nonedisplay_reason: str | None = None+ workflow_run_id: str | None = Nonedef run_outcome_display_reason(text: str | None) -> str | None:diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 2bff031d6..3ae092218 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -184,6 +184,7 @@ class AgentContext:# Latest evaluated outcome-gate verdict this turn. Deliberately not reset# per-run: a later run that fails before verification keeps the verdict.last_outcome_gate_reason: str | None = None+ last_outcome_gate_workflow_run_id: str | None = Nonelast_failure_category_top: str | None = Nonelast_update_block_count: int | None = Nonelast_failed_workflow_yaml: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 9afc7145c..6f7b3ea60 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -6,6 +6,7 @@ import jsonimport timefrom collections import defaultdictfrom collections.abc import Mapping+from dataclasses import replacefrom datetime import datetime, timezonefrom typing import Any, Literalfrom urllib.parse import urlparse@@ -1433,6 +1434,7 @@ def _record_run_blocks_result(copilot_ctx.last_test_failure_reason = Noneif completion_verification is not None and completion_verification.status == "evaluated":copilot_ctx.last_outcome_gate_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)+ copilot_ctx.last_outcome_gate_workflow_run_id = copilot_ctx.last_run_blocks_workflow_run_idcopilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_run_outcome = Nonecopilot_ctx.last_run_outcome_block_labels = []@@ -1618,6 +1620,8 @@ def _record_run_blocks_result(def _stash_recorded_run_outcome(copilot_ctx: Any, outcome: RecordedRunOutcome) -> RecordedRunOutcome:+ if outcome.workflow_run_id is None:+ outcome = replace(outcome, workflow_run_id=getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))copilot_ctx.last_run_outcome = outcomecopilot_ctx.last_run_outcome_block_labels = list(getattr(copilot_ctx, "last_run_blocks_block_labels", []) or [])return outcomediff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 5376ec6ef..e7e728135 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -14,6 +14,7 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp+from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_editfrom skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_diagnosticsfrom skyvern.forge.sdk.copilot.code_block_synthesis import (artifact_dependency_id,@@ -1643,6 +1644,7 @@ def _record_workflow_update_result(copilot_ctx.last_update_block_count = block_countcopilot_ctx.last_test_ok = Nonecopilot_ctx.last_test_failure_reason = None+ clear_terminal_evidence_on_workflow_edit(copilot_ctx)# A fresh workflow edit invalidates the prior test's failure state —# otherwise an exhausted POST_UPDATE_NUDGE on the new draft would raise# CopilotNonRetriableNavError with the old run's error, telling the userdiff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.pyindex 65219931f..75780d9f8 100644--- a/tests/unit/test_copilot_agent_blocker_renderer.py+++ b/tests/unit/test_copilot_agent_blocker_renderer.py@@ -5,8 +5,12 @@ from typing import Any, get_argsimport pytest+from skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.agent import (_FALLBACK_BLOCKER_REPLY,+ _RAW_SECRET_LEAK_REFUSAL,+ _build_output_policy_blocked_result,+ _build_turn_halt_exit_result,_finalize_result_with_blocker_override,_render_blocker_reply,)@@ -17,6 +21,10 @@ from skyvern.forge.sdk.copilot.blocker_signal import (RecoveryHint,)from skyvern.forge.sdk.copilot.context import AgentResult, CopilotContext+from skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, OutputPolicyReason, OutputPolicyVerdict+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+from skyvern.forge.sdk.copilot.turn_halt import TurnHalt, TurnHaltKind# Source-of-truth deny list lives in blocker_signal.py. Re-importing here# (instead of hand-copying) guarantees the test stays in sync if a new token@@ -69,6 +77,38 @@ def _agent_result(user_response: str = "Agent prose reply with leaked TurnIntent)+def _blocked_result(+ ctx: CopilotContext,+ *reason_codes: OutputPolicyReason,+ output_kind: CopilotOutputKind = CopilotOutputKind.REFUSAL,+) -> AgentResult:+ return _build_output_policy_blocked_result(+ ctx,+ OutputPolicyVerdict(+ allowed=False,+ output_kind=output_kind,+ reason_codes=list(reason_codes),+ ),+ prior_global_llm_context=None,+ prior_workflow_yaml=None,+ )+++def _seed_terminal_evidence(ctx: CopilotContext, run_id: str = "wr_latest") -> None:+ ctx.last_run_blocks_workflow_run_id = run_id+ ctx.last_run_outcome = RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ display_reason="The requested record was not verified.",+ workflow_run_id=run_id,+ )+ ctx.last_outcome_gate_reason = (+ "Failed: The run completed but did not demonstrate the goal outcome(s): "+ "the requested record was not verified. Add an end-state confirmation, then re-run."+ )+ ctx.last_outcome_gate_workflow_run_id = run_id++@pytest.mark.parametrize("recovery_hint", _ALL_RECOVERY_HINTS)def test_render_picks_response_type_from_hint(recovery_hint: RecoveryHint) -> None:signal = _signal(recovery_hint=recovery_hint)@@ -178,13 +218,6 @@ def test_output_policy_blocked_result_zeroes_proposal_when_blocker_active() -> N"""OutputPolicy hard-block must not surface a workflow proposal when ablocker is set, even though the shim is intentionally skipped on that path."""- from skyvern.forge.sdk.copilot.agent import _build_output_policy_blocked_result- from skyvern.forge.sdk.copilot.output_policy import (- CopilotOutputKind,- OutputPolicyReason,- OutputPolicyVerdict,- )-ctx = _ctx()# Mock a workflow on ctx to confirm the builder zeros it.fake_workflow: Any = object()@@ -192,22 +225,102 @@ def test_output_policy_blocked_result_zeroes_proposal_when_blocker_active() -> Nctx.last_workflow_yaml = "title: X\nworkflow_definition:\n blocks: []\n"ctx.blocker_signal = _signal(internal_reason_code="tool_error_pending_reconciliation_no_input")- verdict = OutputPolicyVerdict(- allowed=False,- output_kind=CopilotOutputKind.REFUSAL,- reason_codes=[OutputPolicyReason.RAW_SECRET_LEAK],- )- result = _build_output_policy_blocked_result(- ctx,- verdict,- prior_global_llm_context=None,- prior_workflow_yaml=None,- )+ result = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK)assert result.updated_workflow is Noneassert result.workflow_yaml is Noneassert result.proposal_disposition == "no_proposal"+def test_output_policy_generic_block_uses_recorded_terminal_evidence() -> None:+ ctx = _ctx()+ fake_workflow: Any = object()+ ctx.last_workflow = fake_workflow+ ctx.last_workflow_yaml = "title: Draft\n"+ ctx.workflow_persisted = True+ ctx.last_test_ok = True+ _seed_terminal_evidence(ctx)+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"++ result = _blocked_result(ctx, OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)++ assert result.response_type == "ASK_QUESTION"+ assert result.updated_workflow is fake_workflow+ assert result.proposal_disposition == "review_tested"+ assert "latest run recorded workflow output" in result.user_response+ assert "did not demonstrate the goal outcome" in result.user_response+ assert "verification challenge" in result.user_response+ assert "workflow draft is still saved" in result.user_response+ assert "wr_latest" not in result.user_response+ assert result.narrative_payload is not None+ assert result.narrative_payload["terminalMessage"] == result.user_response+ assert result.narrative_payload["responseType"] == "ASK_QUESTION"+++def test_output_policy_recorded_evidence_recheck_keeps_original_output_kind(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = _ctx()+ _seed_terminal_evidence(ctx)+ seen_output_kinds: list[CopilotOutputKind] = []++ def allow_policy(**kwargs: Any) -> OutputPolicyVerdict:+ seen_output_kinds.append(kwargs["output_kind"])+ return OutputPolicyVerdict(allowed=True, output_kind=kwargs["output_kind"], reason_codes=[])++ monkeypatch.setattr(agent_module, "evaluate_output_policy", allow_policy)++ result = _blocked_result(+ ctx,+ OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK,+ output_kind=CopilotOutputKind.REFUSAL,+ )++ assert result.response_type == "ASK_QUESTION"+ assert seen_output_kinds == [CopilotOutputKind.REFUSAL]+++def test_output_policy_generic_block_requires_clean_terminal_evidence() -> None:+ no_recorded = _ctx()+ adversarial = _ctx()+ adversarial.last_run_blocks_workflow_run_id = "wr_hidden"+ adversarial.last_outcome_gate_reason = "update_and_run_blocks failed for wr_hidden; do not retry this step."+ adversarial.last_outcome_gate_workflow_run_id = "wr_hidden"++ for ctx in (no_recorded, adversarial):+ result = _blocked_result(ctx, OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK)+ assert (+ result.user_response+ == "I could not safely return that chat reply. Please adjust the request and try again."+ )+ assert result.response_type == "ASK_QUESTION"+ assert "update_and_run_blocks" not in result.user_response+ assert "wr_hidden" not in result.user_response+ assert "do not retry" not in result.user_response.lower()+++def test_output_policy_specific_branches_bypass_recorded_terminal_fallback() -> None:+ ctx = _ctx()+ _seed_terminal_evidence(ctx)+ raw_secret = _blocked_result(ctx, OutputPolicyReason.RAW_SECRET_LEAK)+ assert raw_secret.user_response == _RAW_SECRET_LEAK_REFUSAL+ assert "latest run" not in raw_secret.user_response.lower()++ clarification_ctx = _ctx()+ clarification_ctx.request_policy = RequestPolicy(+ user_response_policy="ask_clarification",+ clarification_question="Please confirm which saved credential should be used.",+ )+ result = _blocked_result(+ clarification_ctx,+ OutputPolicyReason.REQUEST_POLICY_CLARIFICATION_BYPASS,+ output_kind=CopilotOutputKind.CLARIFICATION_REQUEST,+ )++ assert result.response_type == "ASK_QUESTION"+ assert result.user_response == "Please confirm which saved credential should be used."+ assert result.narrative_payload is not None+ assert result.narrative_payload["terminalMessage"] == result.user_response+ assert result.narrative_payload["responseType"] == "ASK_QUESTION"++def test_shim_preserves_workflow_draft_when_signal_opts_in() -> None:"""Late-block-running blockers say 'wrap up with what you have' — theysurface the saved draft alongside the rendered chat reply."""@@ -361,3 +474,19 @@ def test_output_policy_blocked_result_surfaces_workflow_when_no_blocker() -> Non)assert result.updated_workflow is fake_workflowassert result.workflow_yaml == ctx.last_workflow_yaml+++def test_turn_halt_exit_keeps_halt_signal_when_context_signal_is_cleared() -> None:+ ctx = _ctx()+ signal = _signal(+ kind="loop_detected",+ user_facing="I'm stuck retrying the same step. Tell me what to change and I'll try a different approach.",+ internal_reason_code="loop_detected_repeated_failed_step",+ blocked_tool="run_blocks_and_collect_debug",+ )+ halt = TurnHalt(kind=TurnHaltKind.LOOP_DETECTED, blocker_signal=signal)+ ctx.blocker_signal = None++ result = _build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)++ assert result.user_response == signal.user_facing_reasondiff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.pyindex 188cb8a43..a45a2e21b 100644--- a/tests/unit/test_copilot_blocker_signal.py+++ b/tests/unit/test_copilot_blocker_signal.py@@ -349,6 +349,8 @@ def test_stash_refreshes_held_loop_signal_with_evidence_recorded_after_the_stashassert dict(loop_signal.extra) == {}ctx.last_outcome_gate_reason = _LATE_RECORDED_REASON+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"+ ctx.last_run_blocks_workflow_run_id = "wr_latest"ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"ctx.has_staged_proposal = Truediff --git a/tests/unit/test_copilot_frontier.py b/tests/unit/test_copilot_frontier.pyindex c46987a81..6bcf17a44 100644--- a/tests/unit/test_copilot_frontier.py+++ b/tests/unit/test_copilot_frontier.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.copilot.output_utils import (sanitize_tool_result_for_llm,summarize_tool_result,)+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import (_find_invalidated_labels,_frontier_run_size_error,@@ -1657,6 +1658,39 @@ def test_chokepoint_uses_passed_prior_when_last_workflow_absent() -> None:assert "extract" in tools._unverified_current_workflow_labels(ctx)+def test_workflow_update_clears_terminal_run_evidence() -> None:+ new = _wf_def(+ ("open", "goto_url", {"url": "https://example.com"}),+ ("extract", "extraction", {"prompt": "extract CHANGED"}),+ )+ ctx = _make_ctx()+ ctx.last_run_blocks_workflow_run_id = "wr_old"+ ctx.last_successful_run_blocks_workflow_run_id = "wr_old"+ ctx.last_run_blocks_block_ids = ["wrb_old"]+ ctx.last_run_blocks_block_labels = ["extract"]+ ctx.last_run_outcome = RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_old")+ ctx.last_run_outcome_block_labels = ["extract"]+ ctx.last_outcome_gate_reason = "The prior run did not demonstrate the goal."+ ctx.last_outcome_gate_workflow_run_id = "wr_old"+ ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"+ ctx.completion_verification_result = object() # type: ignore[assignment]+ ctx.outcome_verification_trace_snapshot = {"old": True}++ _record_workflow_update_result(ctx, {"ok": True, "_workflow": _FakeWorkflow(new)}, None)++ assert ctx.last_run_blocks_workflow_run_id is None+ assert ctx.last_successful_run_blocks_workflow_run_id is None+ assert ctx.last_run_blocks_block_ids == []+ assert ctx.last_run_blocks_block_labels == []+ assert ctx.last_run_outcome is None+ assert ctx.last_run_outcome_block_labels == []+ assert ctx.last_outcome_gate_reason is None+ assert ctx.last_outcome_gate_workflow_run_id is None+ assert ctx.last_test_anti_bot is None+ assert ctx.completion_verification_result is None+ assert ctx.outcome_verification_trace_snapshot == {}++def test_unavailable_prior_with_trust_fails_closed() -> None:new = _wf_def(("open", "goto_url", {"url": "https://example.com"}),diff --git a/tests/unit/test_copilot_loop_detection_blocker.py b/tests/unit/test_copilot_loop_detection_blocker.pyindex 40ce0a314..1808d83eb 100644--- a/tests/unit/test_copilot_loop_detection_blocker.py+++ b/tests/unit/test_copilot_loop_detection_blocker.py@@ -16,6 +16,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (from skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.mcp_adapter import SkyvernOverlayMCPServer, _stash_and_emit_loop_blockerfrom skyvern.forge.sdk.copilot.output_policy import CopilotOutputKind, evaluate_output_policy+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import _build_loop_blocker_signal, _tool_loop_errorfrom skyvern.forge.sdk.copilot.tools.mcp_hooks import get_skyvern_mcp_alias_mapfrom skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind@@ -161,6 +162,8 @@ _FULL_EVIDENCE_REASON = ()_FULL_EVIDENCE = LoopBlockerEvidence(outcome_gate_reason=_FULL_EVIDENCE_REASON,+ outcome_gate_workflow_run_id="wr_latest",+ latest_workflow_run_id="wr_latest",anti_bot_blocked=True,has_draft=True,)@@ -286,6 +289,50 @@ def test_raw_runtime_error_reason_drops_verdict_tier(raw_error_reason: str) -> Nassert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")+ "recorded,latest_run_id,expected_tiers,expect_recorded_tier",+ [+ (+ RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_new"),+ "wr_new",+ ["recorded_run_outcome", "draft"],+ True,+ ),+ (None, None, ["draft"], False),+ (+ RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),+ "wr_new",+ ["draft"],+ False,+ ),+ ],+)+def test_stale_outcome_gate_reason_only_keeps_matching_recorded_run(+ recorded: RecordedRunOutcome | None,+ latest_run_id: str | None,+ expected_tiers: list[str],+ expect_recorded_tier: bool,+) -> None:+ evidence = LoopBlockerEvidence(+ outcome_gate_reason=(+ "The run completed but did not demonstrate the goal outcome(s): stale criterion text from an older run."+ ),+ outcome_gate_workflow_run_id="wr_old",+ recorded_run_outcome=recorded,+ latest_workflow_run_id=latest_run_id,+ has_draft=True,+ )++ signal = _build_loop_blocker_signal(+ _BRANCH_MESSAGES["loop_detected_consecutive_same_tool"], tool_name="evaluate", evidence=evidence+ )++ assert "stale criterion text" not in signal.user_facing_reason+ assert ("latest run recorded workflow output" in signal.user_facing_reason) is expect_recorded_tier+ assert dict(signal.extra) == {"loop_evidence_tiers": expected_tiers}+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool="evaluate")++def test_evidence_verdict_sources_only_from_the_outcome_gate_field() -> None:ctx = _ctx()ctx.last_test_failure_reason = "Failed to execute code block. Reason: TimeoutError: Timeout 30000ms exceeded."@@ -334,6 +381,8 @@ def test_native_and_mcp_paths_carry_equivalent_evidence_bearing_signals() -> Non"The run completed but did not demonstrate the goal outcome(s): the requested record is checked ""on a public registry site with a search form and expandable result rows.")+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"+ ctx.last_run_blocks_workflow_run_id = "wr_latest"ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"ctx.has_staged_proposal = Truereturn ctx@@ -370,6 +419,8 @@ def test_tool_loop_error_entry_refreshes_stale_held_loop_signal() -> None:ctx.last_outcome_gate_reason = ("The run completed but did not demonstrate the goal outcome(s): the requested record is checked.")+ ctx.last_outcome_gate_workflow_run_id = "wr_latest"+ ctx.last_run_blocks_workflow_run_id = "wr_latest"ctx.last_test_anti_bot = "challenge-gated disabled submit/search control"ctx.staged_workflow_yaml = "blocks: []"@@ -425,7 +476,13 @@ def test_composed_loop_reply_passes_output_policy_allow_verdict() -> None:signal = _build_loop_blocker_signal(_BRANCH_MESSAGES["loop_detected_consecutive_same_tool"],tool_name="evaluate",- evidence=LoopBlockerEvidence(outcome_gate_reason=_FULL_EVIDENCE_REASON, anti_bot_blocked=True, has_draft=False),+ evidence=LoopBlockerEvidence(+ outcome_gate_reason=_FULL_EVIDENCE_REASON,+ outcome_gate_workflow_run_id="wr_latest",+ latest_workflow_run_id="wr_latest",+ anti_bot_blocked=True,+ has_draft=False,+ ),)verdict = evaluate_output_policy(request_policy=None,diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex 3dddbee85..a68724719 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -133,7 +133,10 @@ async def test_blocker_run_emits_hold_then_not_demonstrated() -> None:assert final.display_reason is not None and "human verification challenge" in final.display_reasonassert ctx.last_test_suspicious_success is Trueassert ctx.last_run_outcome == RecordedRunOutcome(- verdict=final.verdict, reason_code=final.reason_code, display_reason=final.display_reason+ verdict=final.verdict,+ reason_code=final.reason_code,+ display_reason=final.display_reason,+ workflow_run_id="wr_test",)assert ctx.last_run_outcome_block_labels == final.block_labels@@ -288,6 +291,20 @@ def test_failed_rerun_clears_prior_recorded_outcome() -> None:assert ctx.last_run_outcome_block_labels == []+def test_recorded_run_outcome_carries_producing_workflow_run_id() -> None:+ ctx = _ctx([_code_block("search_registry_person", {"records": []})])+ outcome = _record_run_blocks_result(+ ctx,+ _run_result([_code_block("search_registry_person", {"records": []})]),+ completion_verification=_evaluated(satisfied=False),+ )++ assert outcome is not None+ assert outcome.workflow_run_id == "wr_test"+ assert ctx.last_run_outcome is not None+ assert ctx.last_run_outcome.workflow_run_id == "wr_test"++def test_both_consumers_route_through_single_producer() -> None:source = inspect.getsource(copilot_tools)assert source.count("await _verify_and_record_run_blocks_result(") == 2diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.pyindex 776e6e477..ed23f2aff 100644--- a/tests/unit/test_workflow_copilot_route.py+++ b/tests/unit/test_workflow_copilot_route.py@@ -36,6 +36,7 @@ from skyvern.forge.sdk.routes.workflow_copilot import ()from skyvern.forge.sdk.schemas.workflow_copilot import (WorkflowCopilotChatRequest,+ WorkflowCopilotChatSender,WorkflowCopilotStreamErrorUpdate,WorkflowCopilotStreamResponseUpdate,)@@ -913,13 +914,15 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_audescription="Original description",workflow_definition=None,)+ terminal_message = "I could not safely return that chat reply."agent_result = SimpleNamespace(- user_response="I could not safely return that chat reply.",+ user_response=terminal_message,updated_workflow=None,global_llm_context=None,workflow_yaml=None,workflow_was_persisted=False,clear_proposed_workflow=False,+ response_type="ASK_QUESTION",unvalidated=False,output_policy_diagnostics={"final_output_policy_allowed": False,@@ -949,6 +952,23 @@ async def test_output_policy_block_preserves_unvalidated_prior_proposal_under_auclear_calls = [c for c in update_calls if c.kwargs.get("proposed_workflow") is None]assert not clear_calls, f"did not expect a clear call, got {update_calls!r}"+ assistant_call = next(+ call+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list+ if call.kwargs.get("sender") == WorkflowCopilotChatSender.AI+ )+ assert assistant_call.kwargs["content"] == terminal_message++ response_frames = [+ call.args[0]+ for call in stream.send.await_args_list+ if isinstance(call.args[0], WorkflowCopilotStreamResponseUpdate)+ ]+ assert len(response_frames) == 1+ frame = response_frames[0]+ assert frame.message == terminal_message+ assert frame.response_type == "ASK_QUESTION"+@pytest.mark.asyncioasync def test_unvalidated_timeout_wip_overrides_auto_accept(monkeypatch: pytest.MonkeyPatch) -> None: