Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11950 typed degraded completion terminal (#7214)
Andrew Neilson400fd43d7b0a
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 9ff53af97..d31449be6 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -153,6 +153,7 @@ from skyvern.forge.sdk.copilot.turn_halt import (_INVOLUNTARY_BLOCKER_REASON_CODES,CopilotTurnHalt,TurnHalt,+ TurnHaltKind,raise_if_turn_halt,turn_halt_to_trace_data,)@@ -1900,6 +1901,16 @@ def _build_turn_halt_exit_result(global_llm_context: str | None,halt: TurnHalt,) -> AgentResult:+ if halt.kind == TurnHaltKind.DELIVERED_UNVERIFIED:+ reply = _delivered_unverified_reply(ctx) or _BUILT_UNVERIFIED_COMPLETED_REPLY+ return _build_wip_exit_result(+ ctx,+ global_llm_context,+ default_reply=reply,+ unvalidated_reply=reply,+ tested_reply=reply,+ terminal_reason=f"turn_halt:{halt.kind.value}",+ )signal = halt.blocker_signalif isinstance(signal, CopilotToolBlockerSignal) and signal.blocker_kind == "loop_detected":refresh_held_loop_blocker_evidence(ctx)@@ -2031,6 +2042,33 @@ def _terminal_summary_scalar(value: Any) -> str | None:return cleaned+def _delivered_unverified_reply(ctx: CopilotContext) -> str | None:+ if getattr(ctx, "delivered_unverified_terminal", False) is not True:+ return None+ parts: list[str] = []+ observed_outputs = getattr(ctx, "delivered_unverified_observed_outputs", {})+ if not isinstance(observed_outputs, dict):+ observed_outputs = {}+ for key, value in observed_outputs.items():+ rendered = _terminal_summary_scalar(value)+ if rendered is None and isinstance(value, list | dict):+ try:+ rendered = redact_raw_secrets_for_prompt(json.dumps(value, sort_keys=True))+ except TypeError:+ rendered = None+ if isinstance(key, str) and key.strip() and rendered and not contains_internal_machinery_leak(rendered):+ parts.append(f"{key}: {rendered[:_VERIFIED_TERMINAL_VALUE_MAX_CHARS]}")+ if parts:+ return (+ "I built and ran the workflow. The latest run returned "+ f"{'; '.join(parts[:4])}. That value was not independently verified, so review the draft before using it."+ )+ return (+ "I built and ran the workflow, and the latest run returned the requested output. "+ "That output was not independently verified, so review the draft before using it."+ )++def _verified_output_value(ctx: CopilotContext, output_key: str | None) -> Any:if not output_key:return None@@ -2756,7 +2794,11 @@ def _build_wip_exit_result():full_test_ok = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is Trueunvalidated = not full_test_ok- if unvalidated and recorded_failure_reply:+ delivered_reply = _delivered_unverified_reply(ctx)+ if delivered_reply is not None:+ reply = delivered_reply+ unvalidated = True+ elif unvalidated and recorded_failure_reply:reply = recorded_failure_replyif halted_mid_progress:reply = _ensure_unvalidated_proposal_affordance(reply)diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 441d8b45b..d43bb8094 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -254,6 +254,9 @@ class _TerminalEvidenceResetCtx(Protocol):last_outcome_gate_reason: str | Nonelast_outcome_gate_workflow_run_id: str | Nonelast_test_anti_bot: str | None+ delivered_unverified_terminal: bool+ delivered_unverified_workflow_run_id: str | None+ delivered_unverified_observed_outputs: dict[str, Any]completion_verification_result: Any | Noneoutcome_verification_trace_snapshot: dict[str, Any]@@ -286,6 +289,9 @@ def clear_terminal_evidence_on_workflow_edit(ctx: _TerminalEvidenceResetCtx) ->ctx.last_outcome_gate_reason = Nonectx.last_outcome_gate_workflow_run_id = Nonectx.last_test_anti_bot = None+ ctx.delivered_unverified_terminal = False+ ctx.delivered_unverified_workflow_run_id = None+ ctx.delivered_unverified_observed_outputs = {}ctx.completion_verification_result = Nonectx.outcome_verification_trace_snapshot = {}diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 46b8e42ba..180e88adb 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -254,6 +254,11 @@ class CompletionVerificationResult:}+@dataclass(frozen=True)+class DeliveredUnverifiedTerminalState:+ observed_verdicts: tuple[CriterionVerdict, ...]++@dataclass(frozen=True)class RunEvidenceSnapshot:workflow_run_id: str | None = None@@ -330,6 +335,7 @@ _UNAVAILABLE = CompletionVerificationResult(status="unavailable")_MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"_INCOMPLETE_VALIDATION_CLASSIFICATION_CONTRACT = "incomplete typed classification contract"_MISSING_REGISTERED_DOWNLOAD_EVIDENCE = "run output did not include a non-empty registered browser download"+_DELIVERED_UNVERIFIED_OUTPUT_SOURCES = frozenset({"runtime_output", "registered_output_parameter"})def registered_download_completion_criterion() -> CompletionCriterion:@@ -2242,6 +2248,70 @@ def only_degraded_blocking(result: CompletionVerificationResult) -> bool:return blocking <= degraded+def _delivered_unverified_observed_verdict(verdict: CriterionVerdict) -> bool:+ if not verdict.evidence_ref or not verdict.output_path:+ return False+ normalized_path = verdict.output_path.removeprefix("output.").strip()+ if normalized_path.split(".")[-1] == "evidence_text":+ return False+ if verdict.evidence_source not in _DELIVERED_UNVERIFIED_OUTPUT_SOURCES:+ return False+ if verdict.self_emitted_judgment_not_independent:+ return False+ if verdict.requested_output_evidence_source == "independent_run_evidence":+ return False+ if verdict.grounding_mode in {"shape", "judgment_boolean"}:+ return False+ if verdict.satisfied:+ return (+ verdict.reason_code == "evidence_confirms"+ and verdict.grounding_mode == "exact_value"+ and verdict.has_exact_value+ )+ return _is_structural_requested_output_abstention(verdict) and not verdict.has_exact_value+++def degraded_contract_delivered_unverified_terminal_state(+ result: CompletionVerificationResult | None,+ *,+ run_ok: bool,+ workflow_run_id: str | None,+ latest_workflow_run_id: str | None,+ artifact_health_blocked: bool,+ terminal_blocked: bool,+) -> DeliveredUnverifiedTerminalState | None:+ if (+ result is None+ or result.status != "evaluated"+ or result.is_fully_satisfied()+ or not run_ok+ or not workflow_run_id+ or workflow_run_id != latest_workflow_run_id+ or artifact_health_blocked+ or terminal_blocked+ ):+ return None+ degraded = set(result.degraded_criterion_ids)+ if not degraded:+ return None+ verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}+ observed: list[CriterionVerdict] = []+ blocking: set[str] = set()+ for criterion_id in result.criterion_ids:+ verdict = verdict_by_id.get(criterion_id)+ if verdict is not None and _delivered_unverified_observed_verdict(verdict):+ observed.append(verdict)+ continue+ if verdict is not None and verdict.satisfied:+ continue+ if verdict is not None and result.is_structural_contingent_abstention(verdict):+ continue+ blocking.add(criterion_id)+ if not observed or not blocking or not blocking <= degraded:+ return None+ return DeliveredUnverifiedTerminalState(observed_verdicts=tuple(observed))++def carry_degraded_criterion_ids(result: CompletionVerificationResult,criteria: Iterable[CompletionCriterion],diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex ddf120e06..0c08d81f4 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -592,6 +592,9 @@ class CopilotContext(AgentContext):last_run_blocks_workflow_run_id: str | None = Nonelast_successful_run_blocks_workflow_run_id: str | None = Nonelast_outcome_gate_workflow_run_id: str | None = None+ delivered_unverified_terminal: bool = False+ delivered_unverified_workflow_run_id: str | None = None+ delivered_unverified_observed_outputs: dict[str, Any] = field(default_factory=dict)# Consecutive failed runs where navigation completed but the scraper# could not read the page (generic "failed to load the website" template).# Resets on any non-matching run outcome. Streak crosses workflow-shapediff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex f0fa05a7f..e42536077 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -56,6 +56,7 @@ class DiagnosisFailureType(StrEnum):ACTIVE_RUN_TERMINAL_EVIDENCE = "active_run_terminal_evidence"UNRECOVERABLE_TOOL_ERROR = "unrecoverable_tool_error"SCHEMA_INCOMPATIBILITY = "schema_incompatibility"+ DELIVERED_UNVERIFIED = "delivered_unverified"UNKNOWN = "unknown"@@ -177,11 +178,18 @@ def build_diagnosis_repair_contract(outcome_verified = outcome_fully_verified(ctx)completion_verification = getattr(ctx, "completion_verification_result", None)completion_verification_failed = _completion_verification_failed(completion_verification)+ delivered_unverified = bool(+ run_ok+ and getattr(ctx, "delivered_unverified_terminal", False) is True+ and workflow_run_id+ and workflow_run_id == getattr(ctx, "delivered_unverified_workflow_run_id", None)+ )failure_type = _failure_type(run_ok,suspicious,outcome_verified,completion_verification_failed,+ delivered_unverified,failed_blocks,categories,terminal_challenge_categories,@@ -247,6 +255,8 @@ def build_diagnosis_repair_contract("Stop re-authoring: the edited extraction schema declares fields that map to no output the ""workflow produces, so the mismatch is not repairable without user input.")+ elif failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED:+ decision_summary = "No repair selected; the latest run returned requested output but it was not verified."if (next_action == RepairNextAction.NO_CHANGEand user_goal_satisfied is True@@ -255,7 +265,9 @@ def build_diagnosis_repair_contract(completion_check = "Current run already satisfies the goal."else:completion_check = {- RepairNextAction.NO_CHANGE: "No repair selected; completion remains unverified.",+ RepairNextAction.NO_CHANGE: "No repair selected; completion is delivered but not independently verified."+ if failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED+ else "No repair selected; completion remains unverified.",RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",}.get(@@ -480,6 +492,7 @@ def _failure_type(suspicious: bool,outcome_verified: bool,completion_verification_failed: bool,+ delivered_unverified: bool,failed_blocks: list[str],categories: list[str],terminal_challenge_categories: list[str],@@ -530,7 +543,11 @@ def _failure_type(return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILUREif outcome_verified:return DiagnosisFailureType.NO_FAILURE- if failed_blocks or category_set & _REPAIRABLE_RUNTIME_CATEGORIES:+ if failed_blocks:+ return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ if delivered_unverified:+ return DiagnosisFailureType.DELIVERED_UNVERIFIED+ if category_set & _REPAIRABLE_RUNTIME_CATEGORIES:return DiagnosisFailureType.REPAIRABLE_BLOCK_FAILUREif completion_verification_failed:return DiagnosisFailureType.SUSPICIOUS_SUCCESS@@ -551,7 +568,7 @@ def _next_action(data: dict[str, Any],repair_context: CodeAuthoringRepairContext | None,) -> RepairNextAction:- if failure_type == DiagnosisFailureType.NO_FAILURE:+ if failure_type in {DiagnosisFailureType.NO_FAILURE, DiagnosisFailureType.DELIVERED_UNVERIFIED}:return RepairNextAction.NO_CHANGEif failure_type == DiagnosisFailureType.UNRECOVERABLE_TOOL_ERROR:return RepairNextAction.STOP@@ -645,6 +662,8 @@ def _verification_satisfaction() -> tuple[bool | None, bool | None]:if failure_type == DiagnosisFailureType.TERMINAL_CHALLENGE_BLOCKER:return False, False+ if failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED:+ return True, Falseif isinstance(data, dict) and data.get("active_run_terminal_evidence_detected") is True:trace = data.get("active_run_terminal_completion_verification")fully_satisfied = isinstance(trace, dict) and trace.get("fully_satisfied") is Truediff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 443ae6e00..5f37e460e 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -1520,10 +1520,17 @@ def _tool_visible_result_after_completion_verification(outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)if outcome_unverified_reason is None:return result+ data = result.get("data")+ run_id = data.get("workflow_run_id") if isinstance(data, dict) else None+ if (+ copilot_ctx.delivered_unverified_terminal+ and isinstance(run_id, str)+ and run_id == copilot_ctx.delivered_unverified_workflow_run_id+ ):+ return resultif not _outcome_failure_warrants_repair(copilot_ctx, completion_verification):return result- data = result.get("data")copied_data = dict(data) if isinstance(data, dict) else {}copied_data["failure_reason"] = outcome_unverified_reasoncopied_data["completion_verification"] = (diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 747f2d1be..ca2d9ffef 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -10,7 +10,7 @@ from collections import defaultdictfrom collections.abc import Mapping, Sequencefrom dataclasses import dataclass, replacefrom datetime import datetime, timezone-from typing import Any, Literal+from typing import Any, Literal, castfrom urllib.parse import urlparseimport structlog@@ -49,6 +49,8 @@ from skyvern.forge.sdk.copilot.completion_output_grounding import page_evidence_from skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,CriterionVerdict,+ DeliveredUnverifiedTerminalState,+ degraded_contract_delivered_unverified_terminal_state,only_structural_requested_output_abstentions,)from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema@@ -115,6 +117,7 @@ from skyvern.forge.sdk.copilot.runtime_authoring_repair import (from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.copilot.turn_halt import (+ stash_delivered_unverified_turn_halt,stash_repair_ceiling_turn_halt,stash_turn_halt_from_blocker_signal,)@@ -2480,6 +2483,43 @@ def _update_verification_evidence_from_run_result(copilot_ctx: AgentContext, resevidence.failure_reason = " ".join(failure_reason.split())[:240]+def _read_mapping_path(payload: dict[str, Any], path: str) -> Any:+ current: Any = payload+ for part in path.split("."):+ if not part or not isinstance(current, dict):+ return None+ current = current.get(part)+ return current+++def _delivered_unverified_single_block_outputs(+ result: dict[str, Any],+ terminal_state: DeliveredUnverifiedTerminalState,+) -> dict[str, Any]:+ data = result.get("data")+ blocks = data.get("blocks") if isinstance(data, dict) else None+ outputs: list[dict[str, Any]] = []+ if isinstance(blocks, list):+ for block in blocks:+ if isinstance(block, dict) and isinstance(block.get("extracted_data"), dict):+ outputs.append(cast(dict[str, Any], block["extracted_data"]))+ if len(outputs) != 1:+ return {}+ block_output = outputs[0]+ observed: dict[str, Any] = {}+ for verdict in terminal_state.observed_verdicts:+ output_path = verdict.output_path or ""+ normalized_path = output_path.removeprefix("output.").strip()+ if not normalized_path or normalized_path.split(".")[-1] == "evidence_text":+ continue+ value = _read_mapping_path(block_output, normalized_path)+ if value is None:+ value = _read_mapping_path(block_output, output_path)+ if value is not None:+ observed[normalized_path] = value+ return observed++def _record_run_blocks_result(copilot_ctx: Any, result: dict[str, Any], completion_verification: CompletionVerificationResult | None = None) -> RecordedRunOutcome | None:@@ -2513,6 +2553,9 @@ def _record_run_blocks_result(_emit_completion_verification_trace(copilot_ctx, completion_verification)copilot_ctx.last_run_blocks_workflow_run_id = run_id if isinstance(run_id, str) else Nonecopilot_ctx.last_successful_run_blocks_workflow_run_id = run_id if run_ok and isinstance(run_id, str) else None+ copilot_ctx.delivered_unverified_terminal = False+ copilot_ctx.delivered_unverified_workflow_run_id = None+ copilot_ctx.delivered_unverified_observed_outputs = {}# Watchdog cancels normally count as ok=False; only a coincident total# timeout softens to ``None`` to keep the unvalidated WIP rescue open.cancelled_by_watchdog = result.get(_INTERNAL_RUN_CANCELLED_BY_WATCHDOG_KEY) is True@@ -2693,6 +2736,37 @@ def _record_run_blocks_result(copilot_ctx.last_test_suspicious_success = Falsecopilot_ctx.last_test_failure_reason = Nonecopilot_ctx.suspicious_success_nudge_count = 0+ delivered_unverified = degraded_contract_delivered_unverified_terminal_state(+ completion_verification,+ run_ok=run_ok,+ workflow_run_id=run_id if isinstance(run_id, str) else None,+ latest_workflow_run_id=copilot_ctx.last_run_blocks_workflow_run_id,+ artifact_health_blocked=artifact_reason is not None,+ terminal_blocked=terminal_challenge is not None or bool(copilot_ctx.last_test_anti_bot),+ )+ if delivered_unverified is not None:+ observed_outputs = _delivered_unverified_single_block_outputs(result, delivered_unverified)+ if observed_outputs:+ workflow_run_id = run_id if isinstance(run_id, str) else None+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_failure_reason = None+ copilot_ctx.suspicious_success_nudge_count = 0+ copilot_ctx.last_full_workflow_test_ok = False+ copilot_ctx.delivered_unverified_terminal = True+ copilot_ctx.delivered_unverified_workflow_run_id = workflow_run_id+ copilot_ctx.delivered_unverified_observed_outputs = observed_outputs+ stash_delivered_unverified_turn_halt(copilot_ctx, workflow_run_id=workflow_run_id)+ update_repeated_failure_state(copilot_ctx, result)+ _update_verification_evidence_from_run_result(copilot_ctx, result)+ recorded_outcome = RecordedRunOutcome(+ verdict="not_evaluated",+ display_reason=run_outcome_display_reason(+ "The latest run returned the requested output, but it was not independently verified."+ ),+ workflow_run_id=workflow_run_id,+ )+ _record_adjudicated_build_test_outcome(copilot_ctx, result, completion_verification, recorded_outcome)+ return _stash_recorded_run_outcome(copilot_ctx, recorded_outcome)if empty_data_blocks and not completion_verification_evaluated:copilot_ctx.last_test_ok = Nonecopilot_ctx.last_test_suspicious_success = Truediff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex 7879f1561..aca046fd8 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -4,7 +4,7 @@ from __future__ import annotationsfrom dataclasses import dataclass, fieldfrom enum import StrEnum-from typing import Any+from typing import TYPE_CHECKING, Anyimport structlog@@ -27,6 +27,9 @@ from skyvern.forge.sdk.copilot.schema_incompatibility import SCHEMA_INCOMPATIBILLOG = structlog.get_logger()+if TYPE_CHECKING:+ from skyvern.forge.sdk.copilot.runtime import AgentContext+REPAIR_CEILING_REASON_CODE = "repair_ceiling_reached"ADVISORY_DISPATCH_STALLED_REASON_CODE = "advisory_dispatch_stalled"@@ -38,10 +41,12 @@ class TurnHaltKind(StrEnum):REPAIR_CEILING_REACHED = "repair_ceiling_reached"SCHEMA_INCOMPATIBILITY = "schema_incompatibility"OUTPUT_SOURCE_UNOBSERVABLE = "output_source_unobservable"+ DELIVERED_UNVERIFIED = "delivered_unverified"class TurnHaltVerdict(StrEnum):BLOCKED = "blocked"+ DELIVERED_UNVERIFIED = "delivered_unverified"_LOOP_TERMINAL_REASON_CODES = frozenset(@@ -280,6 +285,21 @@ def stash_repair_ceiling_turn_halt(return halt+def stash_delivered_unverified_turn_halt(ctx: AgentContext, *, workflow_run_id: str | None) -> TurnHalt | None:+ if isinstance(ctx.turn_halt, TurnHalt):+ return ctx.turn_halt+ run_refs = {"workflow_run_id": workflow_run_id} if workflow_run_id else {}+ halt = TurnHalt(+ kind=TurnHaltKind.DELIVERED_UNVERIFIED,+ verdict=TurnHaltVerdict.DELIVERED_UNVERIFIED,+ run_refs=run_refs,+ extra={"source": "run_execution"},+ )+ ctx.turn_halt = halt+ LOG.info("copilot turn halt stashed", **turn_halt_to_trace_data(halt))+ return halt++def raise_if_turn_halt(ctx: Any, *, verified: bool = False) -> None:"""Raise the stashed turn halt unless a verified outcome suppresses it.diff --git a/tests/unit/test_copilot_agent_blocker_renderer.py b/tests/unit/test_copilot_agent_blocker_renderer.pyindex 7ccb13a43..399ccfc98 100644--- a/tests/unit/test_copilot_agent_blocker_renderer.py+++ b/tests/unit/test_copilot_agent_blocker_renderer.py@@ -13,6 +13,7 @@ from skyvern.forge.sdk.copilot.agent import (_build_goal_satisfied_exit_result,_build_output_policy_blocked_result,_build_turn_halt_exit_result,+ _build_wip_exit_result,_finalize_result_with_blocker_override,_render_blocker_reply,_verified_workflow_success_reply,@@ -816,6 +817,30 @@ def test_unverified_blocker_still_renders_blocker_text() -> None:assert overridden.proposal_disposition != "review_tested"+def test_delivered_unverified_exit_renders_observed_output_as_unvalidated() -> None:+ ctx = _ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[SimpleNamespace()]))+ ctx.last_workflow_yaml = "workflow_definition:\n blocks: []\n"+ ctx.last_test_ok = True+ ctx.last_full_workflow_test_ok = False+ ctx.delivered_unverified_terminal = True+ ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}++ result = _build_wip_exit_result(+ ctx,+ None,+ default_reply="default",+ unvalidated_reply="unvalidated",+ tested_reply="tested",+ terminal_reason="turn_halt:delivered_unverified",+ )++ assert "document_name: Resale Demand Package" in result.user_response+ assert "not independently verified" in result.user_response+ assert result.proposal_disposition == "review_untested"+ assert result.workflow_yaml == ctx.last_workflow_yaml++def test_verified_outcome_does_not_suppress_voluntary_terminal_challenge() -> None:ctx = _ctx()_seed_verified_outcome(ctx)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex ec8f30693..4bff2c844 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -31,10 +31,12 @@ from skyvern.forge.sdk.copilot.completion_verification import (REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,CompletionVerificationResult,CriterionVerdict,+ DeliveredUnverifiedTerminalState,RunEvidenceSnapshot,_coerce_result,_structured_record_has_identifier,combine_verification_results,+ degraded_contract_delivered_unverified_terminal_state,evaluate_completion_criteria,grade_definition_criteria,grade_fallback_floor_reached_end_state_criteria,@@ -3130,6 +3132,122 @@ def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverifiedassert outcome.is_authoritative is False+def _degraded_delivered_result(*verdicts: CriterionVerdict) -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["__copilot_fallback_floor__run", *[verdict.criterion_id for verdict in verdicts]],+ verdicts=[+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run",+ state="unsatisfied",+ reason_code="no_evidence",+ ),+ *verdicts,+ ],+ degraded_criterion_ids=["__copilot_fallback_floor__run"],+ )+++def _observed_structural_abstention(+ criterion_id: str = "requested_output",+ *,+ evidence_source: str = "runtime_output",+ reason_code: str = "structurally_abstained",+ output_path: str = "output.document_name",+ grounding_mode: str | None = "missing",+) -> CriterionVerdict:+ return CriterionVerdict(+ criterion_id=criterion_id,+ state="unsatisfied",+ reason_code=reason_code,+ evidence_ref="block_outputs:extract.document_name",+ output_path=output_path,+ grounding_mode=grounding_mode,+ evidence_source=evidence_source,+ )+++def _delivered_terminal_state(+ result: CompletionVerificationResult,+ *,+ run_ok: bool = True,+ workflow_run_id: str | None = "wr_x",+ latest_workflow_run_id: str | None = "wr_x",+ artifact_health_blocked: bool = False,+ terminal_blocked: bool = False,+) -> DeliveredUnverifiedTerminalState | None:+ return degraded_contract_delivered_unverified_terminal_state(+ result,+ run_ok=run_ok,+ workflow_run_id=workflow_run_id,+ latest_workflow_run_id=latest_workflow_run_id,+ artifact_health_blocked=artifact_health_blocked,+ terminal_blocked=terminal_blocked,+ )+++def test_degraded_delivered_unverified_terminal_state_allows_observed_runtime_output() -> None:+ terminal_state = _delivered_terminal_state(_degraded_delivered_result(_observed_structural_abstention()))++ assert terminal_state is not None+ assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]+++ "kwargs",+ [+ {"run_ok": False},+ {"workflow_run_id": "wr_old"},+ {"latest_workflow_run_id": "wr_old"},+ {"artifact_health_blocked": True},+ {"terminal_blocked": True},+ ],+)+def test_degraded_delivered_unverified_terminal_state_rejects_run_and_blocker_exclusions(+ kwargs: dict[str, bool | str | None],+) -> None:+ assert _delivered_terminal_state(_degraded_delivered_result(_observed_structural_abstention()), **kwargs) is None+++ "verdict",+ [+ CriterionVerdict(+ criterion_id="requested_output",+ state="unsatisfied",+ reason_code="missing_exact_field",+ evidence_ref="block_outputs:extract.document_name",+ output_path="output.document_name",+ evidence_source="runtime_output",+ ),+ _observed_structural_abstention(output_path="output.evidence_text"),+ _observed_structural_abstention(grounding_mode="shape"),+ _observed_structural_abstention(evidence_source="independent_page_evidence"),+ ],+)+def test_degraded_delivered_unverified_terminal_state_rejects_non_value_output(verdict: CriterionVerdict) -> None:+ assert _delivered_terminal_state(_degraded_delivered_result(verdict)) is None+++def test_degraded_delivered_unverified_terminal_state_is_not_verified_success() -> None:+ ctx = _ctx_with_blocks("extraction")+ result = _clean_success_result()+ result["data"]["blocks"][0]["label"] = "extract"+ result["data"]["blocks"][0]["extracted_data"] = {"document_name": "Resale Demand Package"}+ verification = _degraded_delivered_result(_observed_structural_abstention())++ recorded = _record_run_blocks_result(ctx, result, completion_verification=verification)++ assert recorded is not None+ assert recorded.verdict == "not_evaluated"+ assert ctx.delivered_unverified_terminal is True+ assert ctx.delivered_unverified_observed_outputs == {"document_name": "Resale Demand Package"}+ assert ctx.turn_halt is not None+ assert ctx.turn_halt.kind.value == "delivered_unverified"+ assert ctx.last_full_workflow_test_ok is False+ assert ctx.last_test_suspicious_success is False++def test_record_run_blocks_verifies_structural_requested_output_with_run_corroborator() -> None:ctx = _ctx_with_blocks("extraction")verification = _mixed(diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 565adf9c9..241d6ebeb 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -1399,6 +1399,132 @@ def test_unverified_completion_evidence_does_not_suppress_suspicious_success(assert contract.verification_result.user_goal_satisfied is False+def test_degraded_delivered_unverified_completion_does_not_route_repair() -> None:+ ctx = _ctx()+ ctx.last_test_suspicious_success = True+ ctx.last_run_blocks_workflow_run_id = "wr_unverified"+ ctx.delivered_unverified_terminal = True+ ctx.delivered_unverified_workflow_run_id = "wr_unverified"+ ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["__copilot_fallback_floor__run", "requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run",+ state="unsatisfied",+ reason_code="no_evidence",+ ),+ CriterionVerdict(+ criterion_id="requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract.document_name",+ output_path="output.document_name",+ grounding_mode="missing",+ evidence_source="runtime_output",+ ),+ ],+ degraded_criterion_ids=["__copilot_fallback_floor__run"],+ )++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": True,+ "data": {+ "workflow_run_id": "wr_unverified",+ "overall_status": "completed",+ "frontier_start_label": "extract",+ "failure_categories": [{"category": "OUTCOME_UNVERIFIED"}],+ "blocks": [{"label": "extract", "block_type": "EXTRACTION", "status": "completed"}],+ },+ },+ ctx=ctx,+ workflow_updated=True,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.DELIVERED_UNVERIFIED+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE+ assert contract.verification_result.user_goal_satisfied is True+ assert contract.verification_result.completion_contract_satisfied is False+++def test_delivered_unverified_does_not_mask_failed_blocks() -> None:+ ctx = _ctx()+ ctx.delivered_unverified_terminal = True+ ctx.delivered_unverified_workflow_run_id = "wr_unverified"+ ctx.delivered_unverified_observed_outputs = {"document_name": "Resale Demand Package"}++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": True,+ "data": {+ "workflow_run_id": "wr_unverified",+ "overall_status": "completed",+ "blocks": [+ {+ "label": "extract",+ "block_type": "EXTRACTION",+ "status": "failed",+ }+ ],+ },+ },+ ctx=ctx,+ workflow_updated=True,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.REPAIRABLE_BLOCK_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR+++def test_degraded_path_without_terminal_state_still_routes_repair() -> None:+ ctx = _ctx()+ ctx.last_test_suspicious_success = True+ ctx.last_run_blocks_workflow_run_id = "wr_unverified"+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["__copilot_fallback_floor__run", "requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run",+ state="unsatisfied",+ reason_code="no_evidence",+ ),+ CriterionVerdict(+ criterion_id="requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:extract.document_name",+ output_path="output.document_name",+ grounding_mode="missing",+ evidence_source="runtime_output",+ ),+ ],+ degraded_criterion_ids=["__copilot_fallback_floor__run"],+ )++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result={+ "ok": True,+ "data": {+ "workflow_run_id": "wr_unverified",+ "overall_status": "completed",+ "frontier_start_label": "extract",+ "blocks": [{"label": "extract", "block_type": "EXTRACTION", "status": "completed"}],+ },+ },+ ctx=ctx,+ workflow_updated=True,+ )++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.SUSPICIOUS_SUCCESS+ assert contract.repair_decision.next_action == RepairNextAction.REPAIR++@pytest.mark.parametrize("completion_verification",[