Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11582: Stop repair loops on ungradeable requested-output checks (#6981)
Andrew Neilsoncae1885c3a8b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex 52cef77c1..913199e76 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -56,6 +56,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (build_turn_state,reconcile_completion_criteria,)+from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentionsfrom skyvern.forge.sdk.copilot.config import (SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD,BlockAuthoringPolicy,@@ -80,6 +81,7 @@ from skyvern.forge.sdk.copilot.context import ()from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failurefrom skyvern.forge.sdk.copilot.enforcement import (+ BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,artifact_health_blocked,outcome_fully_verified,synthesized_persistence_reopened_after_failed_run,@@ -1240,7 +1242,9 @@ def _completion_contract_not_violated(ctx: CopilotContext) -> bool:# Verification was required for this run but could not produce a verdict# (unavailable): do not surface the workflow as verified on run status alone.return False- return result.is_fully_satisfied()+ if result.is_fully_satisfied():+ return True+ return only_structural_requested_output_abstentions(result)def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:@@ -1336,6 +1340,7 @@ def _make_agent_result(def _should_apply_code_only_success_without_review(ctx: CopilotContext, disposition: object) -> bool:return (disposition == "auto_applicable"+ and verified_goal_claim_authorized(ctx)and ctx.block_authoring_policy == BlockAuthoringPolicy.CODE_ONLY_BROWSERand ctx.last_test_ok is Trueand ctx.last_full_workflow_test_ok is True@@ -1536,10 +1541,17 @@ def _build_exit_result()-async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:+async def _build_goal_satisfied_exit_result(+ ctx: CopilotContext,+ global_llm_context: str | None,+ *,+ terminal_reason: str = "verified_goal_satisfied",+ exit_site: str = "verified_goal_satisfied",+ flush_goal_satisfied: bool = True,+) -> AgentResult:# Bypass one extra LLM turn after a full workflow test already satisfies# the diagnosis contract.- if ctx.stream is not None:+ if flush_goal_satisfied and ctx.stream is not None:try:await flush_goal_satisfied_tool_result(ctx.stream, ctx)except Exception as flush_err:@@ -1564,7 +1576,7 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_contfinal_text=user_response,attempted_kind=derive_response_kind(ctx.turn_intent),blocked_signatures=ctx.blocked_reply_signatures,- terminal_reason="verified_goal_satisfied",+ terminal_reason=terminal_reason,turn_intent=ctx.turn_intent,tool_calls=[str(entry.get("tool") or entry.get("name") or "")@@ -1622,7 +1634,11 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_contworkflow_yaml=verified_yaml,workflow_was_persisted=ctx.workflow_persisted,total_tokens=ctx.total_tokens_used,- proposal_disposition="auto_applicable" if verified_workflow is not None else "no_proposal",+ proposal_disposition="auto_applicable"+ if verified_workflow is not None and verified_goal_claim_authorized(ctx)+ else "review_tested"+ if verified_workflow is not None+ else "no_proposal",turn_outcome=outcome,turn_id=ctx.turn_id,narrative_summary=ctx.narrative_summary,@@ -1633,7 +1649,17 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_contnarrative_summary=ctx.narrative_summary,),),- exit_site="verified_goal_satisfied",+ exit_site=exit_site,+ )+++async def _build_built_unverified_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:+ return await _build_goal_satisfied_exit_result(+ ctx,+ global_llm_context,+ terminal_reason=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,+ exit_site=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,+ flush_goal_satisfied=False,)@@ -2422,6 +2448,7 @@ def _build_wip_exit_result(verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)if outcome_fully_verified(ctx) and verified_workflow is not None:final_text, outcome = _guard(tested_reply)+ proposal_disposition = "auto_applicable" if verified_goal_claim_authorized(ctx) else "review_tested"return _finalize_result_with_blocker_override(ctx,_make_agent_result(@@ -2436,7 +2463,7 @@ def _build_wip_exit_result(staged_workflow=ctx.staged_workflow,canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,total_tokens=ctx.total_tokens_used,- proposal_disposition="auto_applicable",+ proposal_disposition=proposal_disposition,cancelled=cancelled,turn_outcome=outcome,turn_id=ctx.turn_id,@@ -2504,6 +2531,9 @@ def _build_wip_exit_result(else:reply = unvalidated_reply if unvalidated else tested_replyfinal_text, outcome = _guard(reply)+ proposal_disposition = "review_untested" if unvalidated else "review_tested"+ if not unvalidated and verified_goal_claim_authorized(ctx):+ proposal_disposition = "auto_applicable"return _finalize_result_with_blocker_override(ctx,_make_agent_result(@@ -2518,7 +2548,7 @@ def _build_wip_exit_result(staged_workflow=ctx.staged_workflow,canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,total_tokens=ctx.total_tokens_used,- proposal_disposition="review_untested" if unvalidated else "auto_applicable",+ proposal_disposition=proposal_disposition,cancelled=cancelled,turn_outcome=outcome,turn_id=ctx.turn_id,@@ -3943,6 +3973,7 @@ async def _run_copilot_turn_impl(from skyvern.cli.mcp_tools import mcp as skyvern_mcpfrom skyvern.forge.sdk.copilot.enforcement import (+ CopilotBuiltUnverified,CopilotGoalSatisfied,CopilotNonRetriableNavError,CopilotTotalTimeoutError,@@ -4180,6 +4211,13 @@ async def _run_copilot_turn_impl(workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,)return await _build_goal_satisfied_exit_result(ctx, global_llm_context)+ except CopilotBuiltUnverified:+ LOG.info(+ "Copilot run stopped after built-unverified repair-inert outcome",+ workflow_permanent_id=chat_request.workflow_permanent_id,+ workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,+ )+ return await _build_built_unverified_exit_result(ctx, global_llm_context)except CopilotTurnHalt as exc:LOG.info("Copilot run stopped after typed turn halt",diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 34b463103..a37b1e49f 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -329,6 +329,18 @@ def recorded_outcome_from_run_blocks_result("evidence_refs": "run output structure",},)+ if recorded_run_outcome.verdict == "not_evaluated":+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code=reason_code,+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "",+ key_provenance={"structural_failure_identity": "run outcome was not evaluated"},+ )structural_identity = verification_identityevidence_refs = output_refsif not structural_identity and not page_refs and not evidence_refs:diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex de5fb9e67..672b618d8 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -1915,6 +1915,20 @@ def _is_corroborated_structural_requested_output_abstention(verdict: CriterionVe)+def only_structural_requested_output_abstentions(result: CompletionVerificationResult) -> bool:+ if result.status != "evaluated" or not result.criterion_ids or result.is_fully_satisfied():+ return False+ verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}+ unmet: list[CriterionVerdict] = []+ for criterion_id in result.criterion_ids:+ verdict = verdict_by_id.get(criterion_id)+ if verdict is None:+ return False+ if not verdict.satisfied:+ unmet.append(verdict)+ return bool(unmet) and all(_is_structural_requested_output_abstention(verdict) for verdict in unmet)++def _is_contingent_abstention(verdict: CriterionVerdict,contingent_criterion_ids: Iterable[str],diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.pyindex 588f64164..3a89f9456 100644--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py@@ -8,6 +8,10 @@ from typing import TYPE_CHECKING, Anyfrom pydantic import BaseModel, ConfigDict, Field, ValidationError+from skyvern.forge.sdk.copilot.completion_verification import (+ CompletionVerificationResult,+ only_structural_requested_output_abstentions,+)from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.failure_tracking import (@@ -23,7 +27,6 @@ from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verifiedfrom skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_REif TYPE_CHECKING:- from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResultfrom skyvern.forge.sdk.copilot.context import CopilotContext_TEXT_MAX = 240@@ -243,16 +246,23 @@ def build_diagnosis_repair_contract("Stop re-authoring: the edited extraction schema declares fields that map to no output the ""workflow produces, so the mismatch is not repairable without user input.")- completion_check = {- RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",- RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",- RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",- }.get(- next_action,- f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"- if target_blocks- else "Run the repaired workflow path and confirm the requested goal is satisfied.",- )+ if (+ next_action == RepairNextAction.NO_CHANGE+ and user_goal_satisfied is True+ and completion_contract_satisfied is True+ ):+ completion_check = "Current run already satisfies the goal."+ else:+ completion_check = {+ RepairNextAction.NO_CHANGE: "No repair selected; completion remains unverified.",+ RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",+ RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",+ }.get(+ next_action,+ f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"+ if target_blocks+ else "Run the repaired workflow path and confirm the requested goal is satisfied.",+ )required_authority: list[str] = []if next_action == RepairNextAction.ASK:required_authority = ["may_answer_without_mutation"]@@ -651,7 +661,11 @@ def _verification_satisfaction(def _completion_verification_failed(completion_verification: CompletionVerificationResult | None) -> bool:- return completion_verification is not None and not completion_verification.is_fully_satisfied()+ if completion_verification is None or completion_verification.is_fully_satisfied():+ return False+ if completion_verification.status != "evaluated" or not completion_verification.criterion_ids:+ return True+ return not only_structural_requested_output_abstentions(completion_verification)def _missing_context(result: dict[str, Any], data: dict[str, Any], failure_type: DiagnosisFailureType) -> list[str]:diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 7dc3068c7..e351cdfd4 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (render_synthesized_offer_text,synthesize_code_block,)+from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentionsfrom skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.config import (DEFAULT_ENFORCEMENT_NUDGES,@@ -622,6 +623,13 @@ class CopilotGoalSatisfied(Exception):"""Raised when a tool proves the workflow already satisfies the turn."""+class CopilotBuiltUnverified(Exception):+ """Raised when clean tests should stop repair without claiming goal success."""+++BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON = "built_unverified_repair_inert"++def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:contract = ctx.latest_diagnosis_repair_contractif contract is None:@@ -635,10 +643,20 @@ def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:)+def _latest_diagnosis_contract_selects_no_repair(ctx: CopilotContext) -> bool:+ contract = ctx.latest_diagnosis_repair_contract+ return contract is not None and contract.repair_decision.next_action is RepairNextAction.NO_CHANGE++def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:return outcome_criteria_evaluated(ctx)+def _completion_verification_only_structural_abstentions(ctx: CopilotContext) -> bool:+ result = ctx.completion_verification_result+ return result is not None and only_structural_requested_output_abstentions(result)++def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:if outcome_fully_verified(ctx):return True@@ -657,6 +675,17 @@ def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:return not _verified_goal_likely_needs_more_work(ctx)+def built_unverified_repair_inert_context(ctx: CopilotContext) -> bool:+ return (+ ctx.last_test_ok is True+ and ctx.last_full_workflow_test_ok is True+ and _outcome_criteria_evaluated(ctx)+ and _latest_diagnosis_contract_selects_no_repair(ctx)+ and _completion_verification_only_structural_abstentions(ctx)+ and not _verified_goal_likely_needs_more_work(ctx)+ )++def verified_goal_claim_authorized(ctx: CopilotContext) -> bool:"""Whether the terminal may CLAIM a tested success. Turn completion keepsflowing through ``verified_goal_satisfied_context``; the claim tier additionally@@ -675,6 +704,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:"""return {"gate_satisfied": verified_goal_satisfied_context(ctx),+ "gate_built_unverified_repair_inert": built_unverified_repair_inert_context(ctx),"gate_claim_authorized": verified_goal_claim_authorized(ctx),"gate_last_test_ok": ctx.last_test_ok is True,"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,@@ -1384,8 +1414,10 @@ def _check_enforcement(raise_if_turn_halt(ctx, verified=verified)_raise_if_unrecoverable_contract_stop(ctx)- if verified:+ if verified_goal_satisfied_context(ctx):raise CopilotGoalSatisfied()+ if built_unverified_repair_inert_context(ctx):+ raise CopilotBuiltUnverified()if _needs_repair_ceiling_halt(ctx):contract = getattr(ctx, "latest_diagnosis_repair_contract", None)diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 20c18923c..28b1a307e 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (CriterionVerdict,RunEvidenceSnapshot,_contingent_metadata_for_criteria,+ _is_structural_requested_output_abstention,combine_verification_results,evaluate_completion_criteria,grade_definition_criteria,@@ -1242,7 +1243,10 @@ def _outcome_failure_warrants_repair(return True# Repair needs at least one affirmatively unsatisfied criterion; unknown alone# (absent judge signal, unmappable definition checks) never warrants repair.- if not any(verdict.state == "unsatisfied" for verdict in completion_verification.verdicts):+ if not any(+ verdict.state == "unsatisfied" and not _is_structural_requested_output_abstention(verdict)+ for verdict in completion_verification.verdicts+ ):return Falsereturn _current_workflow_has_evidence_block(copilot_ctx)diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 96d167f93..90c4322aa 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -41,6 +41,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (from skyvern.forge.sdk.copilot.completion_verification import (CompletionVerificationResult,CriterionVerdict,+ only_structural_requested_output_abstentions,)from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schemafrom skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy@@ -2475,6 +2476,7 @@ def _record_run_blocks_result(unverified = _unverified_current_workflow_labels(copilot_ctx)copilot_ctx.last_unverified_block_labels = unverifiedoutcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)+ outcome_failure_warrants_repair = _outcome_failure_warrants_repair(copilot_ctx, completion_verification)if outcome_unverified_reason is not None:# The workflow already has a confirmation block, yet the produced# evidence does not demonstrate the outcome (or contradicts it). Treat@@ -2484,7 +2486,7 @@ def _record_run_blocks_result(# so preserve streak state until produced evidence demonstrates the# outcome; terminal success stays withheld either way via the# verification result.- if _outcome_failure_warrants_repair(copilot_ctx, completion_verification):+ if outcome_failure_warrants_repair:copilot_ctx.last_test_suspicious_success = Truecopilot_ctx.last_test_failure_reason = outcome_unverified_reasonif isinstance(data, dict):@@ -2502,6 +2504,18 @@ def _record_run_blocks_result(copilot_ctx.last_good_workflow = copilot_ctx.last_workflowcopilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yamlcopilot_ctx.last_test_failure_reason = None+ elif (+ outcome_unverified_reason is not None+ and completion_verification is not None+ and only_structural_requested_output_abstentions(completion_verification)+ and not unverified+ ):+ copilot_ctx.last_full_workflow_test_ok = True+ copilot_ctx.last_unverified_block_labels = []+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml+ copilot_ctx.last_test_suspicious_success = False+ copilot_ctx.last_test_failure_reason = Noneelif outcome_unverified_reason is None and not unverified:copilot_ctx.last_full_workflow_test_ok = Truecopilot_ctx.last_unverified_block_labels = []@@ -2625,6 +2639,11 @@ def _adjudicated_run_outcome(return committedif completion_verification is not None and completion_verification.status == "evaluated":if not completion_verification.is_fully_satisfied():+ if only_structural_requested_output_abstentions(completion_verification):+ return RecordedRunOutcome(+ verdict="not_evaluated",+ display_reason=run_outcome_display_reason("Completion remains unverified."),+ )return RecordedRunOutcome(verdict="not_demonstrated",reason_code="outcome_not_demonstrated",diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 932ff66a9..f78c8c4d4 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.copilot import agent as agent_modulefrom skyvern.forge.sdk.copilot.agent import (_VERIFIED_WORKFLOW_SUCCESS_REPLY,+ _build_built_unverified_exit_result,_build_goal_satisfied_exit_result,_resolve_wrapped_exception_exit_result,)@@ -43,10 +44,15 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (+ BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,+ CopilotBuiltUnverified,CopilotGoalSatisfied,CopilotNonRetriableNavError,CopilotTotalTimeoutError,CopilotUnrecoverableToolError,+ _check_enforcement,+ built_unverified_repair_inert_context,+ verified_goal_claim_authorized,verified_goal_satisfied_context,)from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure@@ -125,6 +131,18 @@ def _verified_goal_contract(*, next_action: RepairNextAction = RepairNextAction.)+def _unverified_no_repair_contract() -> DiagnosisRepairContract:+ return DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),+ diagnosis_result=DiagnosisResult(),+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),+ verification_result=VerificationResult(+ user_goal_satisfied=False,+ completion_contract_satisfied=False,+ ),+ )++class TestFailedTestResponseNormalization:def test_rewrite_failed_test_response_avoids_success_language(self) -> None:from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response@@ -1137,6 +1155,30 @@ class TestVerifiedGoalSatisfiedStop:assert not verified_goal_satisfied_context(ctx)+ def test_check_enforcement_stops_after_structural_built_unverified_terminal(self) -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ )+ ],+ ),+ )++ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is True+ assert verified_goal_claim_authorized(ctx) is False+ with pytest.raises(CopilotBuiltUnverified):+ _check_enforcement(ctx)+def test_verified_goal_satisfied_context_rejects_undercovered_workflow(self) -> None:from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context@@ -1170,7 +1212,8 @@ class TestVerifiedGoalSatisfiedStop:assert result.updated_workflow is workflowassert result.workflow_yaml == "workflow_definition:\n blocks: []\n"- assert result.proposal_disposition == "auto_applicable"+ assert result.proposal_disposition == "review_tested"+ assert result.apply_without_review is False# No adjudicated outcome evidence: the turn ends but the claim renders# built-but-unverified instead of a tested-success claim.assert "not independently verified" in result.user_response.lower()@@ -1189,10 +1232,49 @@ class TestVerifiedGoalSatisfiedStop:verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],)verified = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)+ assert verified.proposal_disposition == "auto_applicable"assert "tested" in verified.user_response.lower()assert verified.narrative_payload is not Noneassert verified.narrative_payload["verifiedSuccess"] is True+ @pytest.mark.asyncio+ async def test_goal_satisfied_exit_result_renders_structural_abstention_as_built_unverified(self) -> None:+ workflow = object()+ ctx = _ctx(+ last_workflow=workflow,+ last_workflow_yaml="workflow_definition:\n blocks: []\n",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ )+ ],+ ),+ tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],+ )++ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is True+ assert verified_goal_claim_authorized(ctx) is False++ result = await _build_built_unverified_exit_result(ctx, global_llm_context=None)++ assert result.updated_workflow is workflow+ assert result.proposal_disposition == "review_tested"+ assert result.apply_without_review is False+ assert "not independently verified" in result.user_response.lower()+ assert result.turn_outcome is not None+ assert result.turn_outcome.terminal_reason == BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON+ assert result.narrative_payload is not None+ assert result.narrative_payload["verifiedSuccess"] is False+@pytest.mark.asyncioasync def test_goal_satisfied_exit_result_carries_outcome_adjudication(self) -> None:from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result@@ -1789,6 +1871,80 @@ class TestTranslateToAgentResultGating:assert agent_result.clear_proposed_workflow is Falseassert agent_result.proposal_disposition == "auto_applicable"+ def test_wip_exit_structural_abstention_stays_review_only(self) -> None:+ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy++ workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ last_workflow=workflow,+ last_workflow_yaml="title: Structural Draft",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ )+ ],+ ),+ )++ agent_result = agent_module._build_wip_exit_result(+ ctx,+ global_llm_context=None,+ default_reply="Timed out.",+ unvalidated_reply="Draft needs review.",+ tested_reply="Tested.",+ terminal_reason="max_turns",+ )++ assert verified_goal_claim_authorized(ctx) is False+ assert agent_result.updated_workflow is workflow+ assert agent_result.proposal_disposition == "review_tested"+ assert agent_result.apply_without_review is False+ assert agent_result.narrative_payload is not None+ assert agent_result.narrative_payload["verifiedSuccess"] is False++ def test_wip_exit_verified_claim_still_auto_applies_code_only(self) -> None:+ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy++ workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))+ ctx = _ctx(+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ last_workflow=workflow,+ last_workflow_yaml="title: Verified Draft",+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ has_staged_proposal=True,+ staged_workflow=workflow,+ latest_diagnosis_repair_contract=_verified_goal_contract(),+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ ),+ )++ agent_result = agent_module._build_wip_exit_result(+ ctx,+ global_llm_context=None,+ default_reply="Timed out.",+ unvalidated_reply="Draft needs review.",+ tested_reply="Tested.",+ terminal_reason="max_turns",+ )++ assert verified_goal_claim_authorized(ctx) is True+ assert agent_result.updated_workflow is workflow+ assert agent_result.proposal_disposition == "auto_applicable"+ assert agent_result.apply_without_review is True+def test_output_field_confirmation_question_is_blocked_when_contract_present(self) -> None:ctx = _ctx(request_policy=RequestPolicy(@@ -2497,6 +2653,7 @@ workflow_definition:assert agent_result.updated_workflow is wfassert agent_result.proposal_disposition == "review_untested"assert "could not verify the requested outcome" in agent_result.user_response.lower()+ assert "did not demonstrate" not in agent_result.user_response.lower()assert "output judge unavailable" in agent_result.user_response.lower()assert "untested" not in agent_result.user_response.lower()@@ -2535,6 +2692,11 @@ workflow_definition:last_update_block_count=3,has_staged_proposal=True,staged_workflow=wf,+ completion_verification_result=CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],+ ),)result = _fake_run_result({"type": "REPLY", "user_response": "All set."})agent_result = asyncio.run(diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex fff4c744a..a95a2c660 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -330,6 +330,45 @@ def test_outcome_not_demonstrated_keeps_authoritative_unsatisfied_criteria_identassert "address and statuses" not in payload_text+def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_structural",+ "blocks": [+ {+ "label": "publish_result",+ "status": "completed",+ "extracted_data": {"document_name": "Resale Demand Package"},+ }+ ],+ },+ }+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c0",+ state="unsatisfied",+ reason_code="structurally_abstained",+ output_path="output.document_name",+ )+ ],+ )++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_structural"),+ completion_verification=verification,+ )++ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "failed_run"+ assert outcome.is_authoritative is False++def test_authoring_repair_context_produces_structural_recorded_outcome() -> None:context = CodeAuthoringRepairContext(block_label="search_records",diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex 4d20d9718..988ab3a83 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -46,6 +46,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (VerificationResult,)from skyvern.forge.sdk.copilot.enforcement import (+ built_unverified_repair_inert_context,verified_goal_claim_authorized,verified_goal_satisfied_context,)@@ -1003,6 +1004,18 @@ def _legacy_verified_ctx() -> CopilotContext:)+def _no_repair_unverified_contract() -> DiagnosisRepairContract:+ return DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),+ diagnosis_result=DiagnosisResult(),+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),+ verification_result=VerificationResult(+ user_goal_satisfied=False,+ completion_contract_satisfied=False,+ ),+ )++def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:ctx = _legacy_verified_ctx()# Turn completion is unchanged: the legacy conjunction still satisfies the gate.@@ -1011,6 +1024,45 @@ def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:assert verified_goal_claim_authorized(ctx) is False+def test_structural_abstention_no_repair_terminalizes_without_authorizing_success_claim() -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "structurally_abstained")),+ )++ assert ctx.completion_verification_result is not None+ assert ctx.completion_verification_result.is_fully_satisfied() is False+ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is True+ assert verified_goal_claim_authorized(ctx) is False+++def test_structural_abstention_terminalization_rejects_real_unsatisfied_verdicts() -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "no_evidence")),+ )++ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is False+++def test_structural_abstention_terminalization_rejects_contradictions() -> None:+ ctx = _ctx(+ last_test_ok=True,+ last_full_workflow_test_ok=True,+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts")),+ )++ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is False++def test_claim_authorized_with_adjudicated_evidence() -> None:ctx = _legacy_verified_ctx()ctx.completion_verification_result = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 9281b2614..ebcc32dcf 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (_verification_satisfaction,)from skyvern.forge.sdk.copilot.enforcement import (+ built_unverified_repair_inert_context,outcome_fully_verified,verified_goal_satisfied_context,)@@ -2783,6 +2784,35 @@ def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_conassert verified_goal_satisfied_context(ctx) is True+def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverified() -> None:+ ctx = _ctx_with_blocks("extraction")+ verification = _mixed(+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="structurally_abstained")+ )++ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)++ assert recorded is not None+ assert recorded.verdict == "not_evaluated"+ assert recorded.reason_code is None+ assert ctx.last_test_suspicious_success is False+ assert ctx.last_test_failure_reason is None+ assert ctx.last_full_workflow_test_ok is True+ ctx.latest_diagnosis_repair_contract = DiagnosisRepairContract(+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),+ diagnosis_result=DiagnosisResult(),+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),+ verification_result=VerificationResult(user_goal_satisfied=False, completion_contract_satisfied=False),+ )+ assert verified_goal_satisfied_context(ctx) is False+ assert built_unverified_repair_inert_context(ctx) is True+ assert outcome_fully_verified(ctx) is False+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.is_authoritative is False++def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:ctx = _ctx_with_blocks("extraction")verification = _mixed(@@ -3398,12 +3428,32 @@ def test_active_terminal_watchdog_exit_cannot_promote_to_terminal_success() -> Ndef test_outcome_failure_warrants_repair() -> None:has_block = _ctx_with_blocks("extraction")nav_only = _ctx_with_blocks("goto_url", "navigation")+ structural_abstention = _mixed(+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:lookup.missing_value",+ )+ )+ mixed_abstention_and_failure = _mixed(+ CriterionVerdict(+ criterion_id="c_requested_output",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:lookup.missing_value",+ ),+ CriterionVerdict(criterion_id="c_real_failure", state="unsatisfied", reason_code="no_evidence"),+ )assert _outcome_failure_warrants_repair(nav_only, None) is False# Contradiction is a real failure regardless of which blocks exist.assert _outcome_failure_warrants_repair(nav_only, _contradicted("c0")) is True# Absence of evidence: failure only once a confirmation block exists.assert _outcome_failure_warrants_repair(has_block, _evaluated(("c0", False))) is Trueassert _outcome_failure_warrants_repair(nav_only, _evaluated(("c0", False))) is False+ assert structural_abstention.is_fully_satisfied() is False+ assert _outcome_failure_warrants_repair(has_block, structural_abstention) is False+ assert _outcome_failure_warrants_repair(has_block, mixed_abstention_and_failure) is True# --- Direction 2: recognition governed by evidence, not run status ---------------diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.pyindex 60dc3d9f7..409a513dc 100644--- a/tests/unit/test_copilot_diagnosis_repair_contract.py+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py@@ -67,6 +67,14 @@ def _contradictory_completion_verification() -> CompletionVerificationResult:)+def _structural_abstention_completion_verification() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0"],+ verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="structurally_abstained")],+ )++def _clean_completed_result() -> dict[str, object]:return {"ok": True,@@ -961,6 +969,27 @@ def test_clean_run_with_present_unsatisfied_completion_verification_fails_safe(assert contract.verification_result.remaining_blocker is not None+def test_clean_run_with_structural_abstention_completion_verification_does_not_repair() -> None:+ ctx = _ctx()+ ctx.completion_verification_result = _structural_abstention_completion_verification()++ contract = build_diagnosis_repair_contract(+ source_tool="update_and_run_blocks",+ result=_clean_completed_result(),+ ctx=ctx,+ workflow_updated=True,+ )+ ctx.latest_diagnosis_repair_contract = contract++ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE+ assert contract.repair_decision.next_action != RepairNextAction.REPAIR+ assert contract.repair_decision.completion_check == "No repair selected; completion remains unverified."+ assert contract.verification_result.user_goal_satisfied is False+ assert contract.verification_result.completion_contract_satisfied is False+ assert latest_diagnosis_contract_satisfies_goal(ctx) is False++def test_clean_run_with_satisfied_completion_verification_has_no_repair_or_blocker() -> None:ctx = _ctx()ctx.completion_verification_result = _satisfied_completion_verification()diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.pyindex a9aae5951..15505d001 100644--- a/tests/unit/test_copilot_response_coverage_gate.py+++ b/tests/unit/test_copilot_response_coverage_gate.py@@ -53,6 +53,7 @@ class _Ctx:self.composition_page_evidence = Noneself.last_update_block_count = Noneself.last_test_ok = None+ self.last_full_workflow_test_ok = Falseself.last_test_failure_reason = Noneself.last_test_suspicious_success = Falseself.last_test_anti_bot = Nonediff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 564bc7b33..eae9dafd7 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -112,7 +112,7 @@ class TestBuildTimeoutExitResult:assert result.updated_workflow is wfassert result.workflow_yaml == "version: '1.0'"- assert result.proposal_disposition == "auto_applicable"+ assert result.proposal_disposition == "review_tested"assert result.user_response == _TIMEOUT_REPLY_TESTEDdef test_missing_yaml_drops_untested_proposal(self) -> None:@@ -241,7 +241,7 @@ class TestBuildMaxTurnsExitResult:result = _build_max_turns_exit_result(ctx, global_llm_context=None)assert result.updated_workflow is wf- assert result.proposal_disposition == "auto_applicable"+ assert result.proposal_disposition == "review_tested"assert result.user_response == _MAX_TURNS_REPLY_TESTEDdef test_failed_test_drops_proposal(self) -> None:@@ -330,7 +330,7 @@ class TestBuildUnexpectedErrorExitResult:assert result.updated_workflow is wfassert result.workflow_yaml == "version: '1.0'"- assert result.proposal_disposition == "auto_applicable"+ assert result.proposal_disposition == "review_tested"assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTEDdef test_failed_test_drops_proposal(self) -> None:@@ -471,7 +471,7 @@ class TestBuildCancelExitResult:assert result.cancelled is Trueassert result.updated_workflow is wfassert result.workflow_yaml == "version: '1.0'"- assert result.proposal_disposition == "auto_applicable"+ assert result.proposal_disposition == "review_tested"assert result.user_response == _CANCEL_REPLY_TESTEDdef test_failed_test_drops_cancel_proposal(self) -> None: