Skip to content

Automate browser based workflows with AI

SKY-11582: Stop repair loops on ungradeable requested-output checks (#6981)

Andrew Neilsoncae1885c3a8b
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 52cef77c1..913199e76 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -56,6 +56,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
build_turn_state,
reconcile_completion_criteria,
)
+from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
from skyvern.forge.sdk.copilot.config import (
SYNTHESIZED_OFFER_REFRESH_STEP_THRESHOLD,
BlockAuthoringPolicy,
@@ -80,6 +81,7 @@ from skyvern.forge.sdk.copilot.context import (
)
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
+ BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,
artifact_health_blocked,
outcome_fully_verified,
synthesized_persistence_reopened_after_failed_run,
@@ -1240,7 +1242,9 @@ def _completion_contract_not_violated(ctx: CopilotContext) -> bool:
# Verification was required for this run but could not produce a verdict
# (unavailable): do not surface the workflow as verified on run status alone.
return False
- return result.is_fully_satisfied()
+ if result.is_fully_satisfied():
+ return True
+ return only_structural_requested_output_abstentions(result)
def _verified_workflow_or_none(ctx: CopilotContext) -> tuple[Any, str | None]:
@@ -1336,6 +1340,7 @@ def _make_agent_result(
def _should_apply_code_only_success_without_review(ctx: CopilotContext, disposition: object) -> bool:
return (
disposition == "auto_applicable"
+ and verified_goal_claim_authorized(ctx)
and ctx.block_authoring_policy == BlockAuthoringPolicy.CODE_ONLY_BROWSER
and ctx.last_test_ok is True
and ctx.last_full_workflow_test_ok is True
@@ -1536,10 +1541,17 @@ def _build_exit_result(
)
-async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
+async def _build_goal_satisfied_exit_result(
+ ctx: CopilotContext,
+ global_llm_context: str | None,
+ *,
+ terminal_reason: str = "verified_goal_satisfied",
+ exit_site: str = "verified_goal_satisfied",
+ flush_goal_satisfied: bool = True,
+) -> AgentResult:
# Bypass one extra LLM turn after a full workflow test already satisfies
# the diagnosis contract.
- if ctx.stream is not None:
+ if flush_goal_satisfied and ctx.stream is not None:
try:
await flush_goal_satisfied_tool_result(ctx.stream, ctx)
except Exception as flush_err:
@@ -1564,7 +1576,7 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_cont
final_text=user_response,
attempted_kind=derive_response_kind(ctx.turn_intent),
blocked_signatures=ctx.blocked_reply_signatures,
- terminal_reason="verified_goal_satisfied",
+ terminal_reason=terminal_reason,
turn_intent=ctx.turn_intent,
tool_calls=[
str(entry.get("tool") or entry.get("name") or "")
@@ -1622,7 +1634,11 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_cont
workflow_yaml=verified_yaml,
workflow_was_persisted=ctx.workflow_persisted,
total_tokens=ctx.total_tokens_used,
- proposal_disposition="auto_applicable" if verified_workflow is not None else "no_proposal",
+ proposal_disposition="auto_applicable"
+ if verified_workflow is not None and verified_goal_claim_authorized(ctx)
+ else "review_tested"
+ if verified_workflow is not None
+ else "no_proposal",
turn_outcome=outcome,
turn_id=ctx.turn_id,
narrative_summary=ctx.narrative_summary,
@@ -1633,7 +1649,17 @@ async def _build_goal_satisfied_exit_result(ctx: CopilotContext, global_llm_cont
narrative_summary=ctx.narrative_summary,
),
),
- exit_site="verified_goal_satisfied",
+ exit_site=exit_site,
+ )
+
+
+async def _build_built_unverified_exit_result(ctx: CopilotContext, global_llm_context: str | None) -> AgentResult:
+ return await _build_goal_satisfied_exit_result(
+ ctx,
+ global_llm_context,
+ terminal_reason=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,
+ exit_site=BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,
+ flush_goal_satisfied=False,
)
@@ -2422,6 +2448,7 @@ def _build_wip_exit_result(
verified_workflow, verified_yaml = _verified_workflow_or_none(ctx)
if outcome_fully_verified(ctx) and verified_workflow is not None:
final_text, outcome = _guard(tested_reply)
+ proposal_disposition = "auto_applicable" if verified_goal_claim_authorized(ctx) else "review_tested"
return _finalize_result_with_blocker_override(
ctx,
_make_agent_result(
@@ -2436,7 +2463,7 @@ def _build_wip_exit_result(
staged_workflow=ctx.staged_workflow,
canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,
total_tokens=ctx.total_tokens_used,
- proposal_disposition="auto_applicable",
+ proposal_disposition=proposal_disposition,
cancelled=cancelled,
turn_outcome=outcome,
turn_id=ctx.turn_id,
@@ -2504,6 +2531,9 @@ def _build_wip_exit_result(
else:
reply = unvalidated_reply if unvalidated else tested_reply
final_text, outcome = _guard(reply)
+ proposal_disposition = "review_untested" if unvalidated else "review_tested"
+ if not unvalidated and verified_goal_claim_authorized(ctx):
+ proposal_disposition = "auto_applicable"
return _finalize_result_with_blocker_override(
ctx,
_make_agent_result(
@@ -2518,7 +2548,7 @@ def _build_wip_exit_result(
staged_workflow=ctx.staged_workflow,
canonical_was_persisted_due_to_param_change=ctx.canonical_was_persisted_due_to_param_change,
total_tokens=ctx.total_tokens_used,
- proposal_disposition="review_untested" if unvalidated else "auto_applicable",
+ proposal_disposition=proposal_disposition,
cancelled=cancelled,
turn_outcome=outcome,
turn_id=ctx.turn_id,
@@ -3943,6 +3973,7 @@ async def _run_copilot_turn_impl(
from skyvern.cli.mcp_tools import mcp as skyvern_mcp
from skyvern.forge.sdk.copilot.enforcement import (
+ CopilotBuiltUnverified,
CopilotGoalSatisfied,
CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
@@ -4180,6 +4211,13 @@ async def _run_copilot_turn_impl(
workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,
)
return await _build_goal_satisfied_exit_result(ctx, global_llm_context)
+ except CopilotBuiltUnverified:
+ LOG.info(
+ "Copilot run stopped after built-unverified repair-inert outcome",
+ workflow_permanent_id=chat_request.workflow_permanent_id,
+ workflow_run_id=ctx.last_successful_run_blocks_workflow_run_id,
+ )
+ return await _build_built_unverified_exit_result(ctx, global_llm_context)
except CopilotTurnHalt as exc:
LOG.info(
"Copilot run stopped after typed turn halt",
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 34b463103..a37b1e49f 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -329,6 +329,18 @@ def recorded_outcome_from_run_blocks_result(
"evidence_refs": "run output structure",
},
)
+ if recorded_run_outcome.verdict == "not_evaluated":
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code=reason_code,
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "",
+ key_provenance={"structural_failure_identity": "run outcome was not evaluated"},
+ )
structural_identity = verification_identity
evidence_refs = output_refs
if not structural_identity and not page_refs and not evidence_refs:
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index de5fb9e67..672b618d8 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -1915,6 +1915,20 @@ def _is_corroborated_structural_requested_output_abstention(verdict: CriterionVe
)
+def only_structural_requested_output_abstentions(result: CompletionVerificationResult) -> bool:
+ if result.status != "evaluated" or not result.criterion_ids or result.is_fully_satisfied():
+ return False
+ verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
+ unmet: list[CriterionVerdict] = []
+ for criterion_id in result.criterion_ids:
+ verdict = verdict_by_id.get(criterion_id)
+ if verdict is None:
+ return False
+ if not verdict.satisfied:
+ unmet.append(verdict)
+ return bool(unmet) and all(_is_structural_requested_output_abstention(verdict) for verdict in unmet)
+
+
def _is_contingent_abstention(
verdict: CriterionVerdict,
contingent_criterion_ids: Iterable[str],
diff --git a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
index 588f64164..3a89f9456 100644
--- a/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
+++ b/skyvern/forge/sdk/copilot/diagnosis_repair_contract.py
@@ -8,6 +8,10 @@ from typing import TYPE_CHECKING, Any
from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ only_structural_requested_output_abstentions,
+)
from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.failure_tracking import (
@@ -23,7 +27,6 @@ from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.workflow_credential_utils import URL_CANDIDATE_RE
if TYPE_CHECKING:
- from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
from skyvern.forge.sdk.copilot.context import CopilotContext
_TEXT_MAX = 240
@@ -243,16 +246,23 @@ def build_diagnosis_repair_contract(
"Stop re-authoring: the edited extraction schema declares fields that map to no output the "
"workflow produces, so the mismatch is not repairable without user input."
)
- completion_check = {
- RepairNextAction.NO_CHANGE: "Current run already satisfies the goal.",
- RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",
- RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",
- }.get(
- next_action,
- f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"
- if target_blocks
- else "Run the repaired workflow path and confirm the requested goal is satisfied.",
- )
+ if (
+ next_action == RepairNextAction.NO_CHANGE
+ and user_goal_satisfied is True
+ and completion_contract_satisfied is True
+ ):
+ completion_check = "Current run already satisfies the goal."
+ else:
+ completion_check = {
+ RepairNextAction.NO_CHANGE: "No repair selected; completion remains unverified.",
+ RepairNextAction.ASK: "Resume diagnosis after the user supplies the missing context.",
+ RepairNextAction.STOP: "Do not rerun unchanged; user-visible blocker must be resolved first.",
+ }.get(
+ next_action,
+ f"Run repaired block labels and confirm success: {', '.join(target_blocks)}"
+ if target_blocks
+ else "Run the repaired workflow path and confirm the requested goal is satisfied.",
+ )
required_authority: list[str] = []
if next_action == RepairNextAction.ASK:
required_authority = ["may_answer_without_mutation"]
@@ -651,7 +661,11 @@ def _verification_satisfaction(
def _completion_verification_failed(completion_verification: CompletionVerificationResult | None) -> bool:
- return completion_verification is not None and not completion_verification.is_fully_satisfied()
+ if completion_verification is None or completion_verification.is_fully_satisfied():
+ return False
+ if completion_verification.status != "evaluated" or not completion_verification.criterion_ids:
+ return True
+ return not only_structural_requested_output_abstentions(completion_verification)
def _missing_context(result: dict[str, Any], data: dict[str, Any], failure_type: DiagnosisFailureType) -> list[str]:
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 7dc3068c7..e351cdfd4 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
render_synthesized_offer_text,
synthesize_code_block,
)
+from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
from skyvern.forge.sdk.copilot.config import (
DEFAULT_ENFORCEMENT_NUDGES,
@@ -622,6 +623,13 @@ class CopilotGoalSatisfied(Exception):
"""Raised when a tool proves the workflow already satisfies the turn."""
+class CopilotBuiltUnverified(Exception):
+ """Raised when clean tests should stop repair without claiming goal success."""
+
+
+BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON = "built_unverified_repair_inert"
+
+
def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:
contract = ctx.latest_diagnosis_repair_contract
if contract is None:
@@ -635,10 +643,20 @@ def latest_diagnosis_contract_satisfies_goal(ctx: CopilotContext) -> bool:
)
+def _latest_diagnosis_contract_selects_no_repair(ctx: CopilotContext) -> bool:
+ contract = ctx.latest_diagnosis_repair_contract
+ return contract is not None and contract.repair_decision.next_action is RepairNextAction.NO_CHANGE
+
+
def _outcome_criteria_evaluated(ctx: CopilotContext) -> bool:
return outcome_criteria_evaluated(ctx)
+def _completion_verification_only_structural_abstentions(ctx: CopilotContext) -> bool:
+ result = ctx.completion_verification_result
+ return result is not None and only_structural_requested_output_abstentions(result)
+
+
def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:
if outcome_fully_verified(ctx):
return True
@@ -657,6 +675,17 @@ def verified_goal_satisfied_context(ctx: CopilotContext) -> bool:
return not _verified_goal_likely_needs_more_work(ctx)
+def built_unverified_repair_inert_context(ctx: CopilotContext) -> bool:
+ return (
+ ctx.last_test_ok is True
+ and ctx.last_full_workflow_test_ok is True
+ and _outcome_criteria_evaluated(ctx)
+ and _latest_diagnosis_contract_selects_no_repair(ctx)
+ and _completion_verification_only_structural_abstentions(ctx)
+ and not _verified_goal_likely_needs_more_work(ctx)
+ )
+
+
def verified_goal_claim_authorized(ctx: CopilotContext) -> bool:
"""Whether the terminal may CLAIM a tested success. Turn completion keeps
flowing through ``verified_goal_satisfied_context``; the claim tier additionally
@@ -675,6 +704,7 @@ def gate_decision_trace_fields(ctx: CopilotContext) -> dict[str, bool]:
"""
return {
"gate_satisfied": verified_goal_satisfied_context(ctx),
+ "gate_built_unverified_repair_inert": built_unverified_repair_inert_context(ctx),
"gate_claim_authorized": verified_goal_claim_authorized(ctx),
"gate_last_test_ok": ctx.last_test_ok is True,
"gate_last_full_workflow_test_ok": ctx.last_full_workflow_test_ok is True,
@@ -1384,8 +1414,10 @@ def _check_enforcement(
raise_if_turn_halt(ctx, verified=verified)
_raise_if_unrecoverable_contract_stop(ctx)
- if verified:
+ if verified_goal_satisfied_context(ctx):
raise CopilotGoalSatisfied()
+ if built_unverified_repair_inert_context(ctx):
+ raise CopilotBuiltUnverified()
if _needs_repair_ceiling_halt(ctx):
contract = getattr(ctx, "latest_diagnosis_repair_contract", None)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 20c18923c..28b1a307e 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
CriterionVerdict,
RunEvidenceSnapshot,
_contingent_metadata_for_criteria,
+ _is_structural_requested_output_abstention,
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
@@ -1242,7 +1243,10 @@ def _outcome_failure_warrants_repair(
return True
# Repair needs at least one affirmatively unsatisfied criterion; unknown alone
# (absent judge signal, unmappable definition checks) never warrants repair.
- if not any(verdict.state == "unsatisfied" for verdict in completion_verification.verdicts):
+ if not any(
+ verdict.state == "unsatisfied" and not _is_structural_requested_output_abstention(verdict)
+ for verdict in completion_verification.verdicts
+ ):
return False
return _current_workflow_has_evidence_block(copilot_ctx)
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 96d167f93..90c4322aa 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -41,6 +41,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
CriterionVerdict,
+ only_structural_requested_output_abstentions,
)
from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
@@ -2475,6 +2476,7 @@ def _record_run_blocks_result(
unverified = _unverified_current_workflow_labels(copilot_ctx)
copilot_ctx.last_unverified_block_labels = unverified
outcome_unverified_reason = _outcome_unverified_reason(copilot_ctx, completion_verification)
+ outcome_failure_warrants_repair = _outcome_failure_warrants_repair(copilot_ctx, completion_verification)
if outcome_unverified_reason is not None:
# The workflow already has a confirmation block, yet the produced
# evidence does not demonstrate the outcome (or contradicts it). Treat
@@ -2484,7 +2486,7 @@ def _record_run_blocks_result(
# so preserve streak state until produced evidence demonstrates the
# outcome; terminal success stays withheld either way via the
# verification result.
- if _outcome_failure_warrants_repair(copilot_ctx, completion_verification):
+ if outcome_failure_warrants_repair:
copilot_ctx.last_test_suspicious_success = True
copilot_ctx.last_test_failure_reason = outcome_unverified_reason
if isinstance(data, dict):
@@ -2502,6 +2504,18 @@ def _record_run_blocks_result(
copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
copilot_ctx.last_test_failure_reason = None
+ elif (
+ outcome_unverified_reason is not None
+ and completion_verification is not None
+ and only_structural_requested_output_abstentions(completion_verification)
+ and not unverified
+ ):
+ copilot_ctx.last_full_workflow_test_ok = True
+ copilot_ctx.last_unverified_block_labels = []
+ copilot_ctx.last_good_workflow = copilot_ctx.last_workflow
+ copilot_ctx.last_good_workflow_yaml = copilot_ctx.last_workflow_yaml
+ copilot_ctx.last_test_suspicious_success = False
+ copilot_ctx.last_test_failure_reason = None
elif outcome_unverified_reason is None and not unverified:
copilot_ctx.last_full_workflow_test_ok = True
copilot_ctx.last_unverified_block_labels = []
@@ -2625,6 +2639,11 @@ def _adjudicated_run_outcome(
return committed
if completion_verification is not None and completion_verification.status == "evaluated":
if not completion_verification.is_fully_satisfied():
+ if only_structural_requested_output_abstentions(completion_verification):
+ return RecordedRunOutcome(
+ verdict="not_evaluated",
+ display_reason=run_outcome_display_reason("Completion remains unverified."),
+ )
return RecordedRunOutcome(
verdict="not_demonstrated",
reason_code="outcome_not_demonstrated",
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 932ff66a9..f78c8c4d4 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.agent import (
_VERIFIED_WORKFLOW_SUCCESS_REPLY,
+ _build_built_unverified_exit_result,
_build_goal_satisfied_exit_result,
_resolve_wrapped_exception_exit_result,
)
@@ -43,10 +44,15 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,
+ CopilotBuiltUnverified,
CopilotGoalSatisfied,
CopilotNonRetriableNavError,
CopilotTotalTimeoutError,
CopilotUnrecoverableToolError,
+ _check_enforcement,
+ built_unverified_repair_inert_context,
+ verified_goal_claim_authorized,
verified_goal_satisfied_context,
)
from skyvern.forge.sdk.copilot.recoverable_failure import build_recoverable_failure
@@ -125,6 +131,18 @@ def _verified_goal_contract(*, next_action: RepairNextAction = RepairNextAction.
)
+def _unverified_no_repair_contract() -> DiagnosisRepairContract:
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),
+ verification_result=VerificationResult(
+ user_goal_satisfied=False,
+ completion_contract_satisfied=False,
+ ),
+ )
+
+
class TestFailedTestResponseNormalization:
def test_rewrite_failed_test_response_avoids_success_language(self) -> None:
from skyvern.forge.sdk.copilot.agent import _rewrite_failed_test_response
@@ -1137,6 +1155,30 @@ class TestVerifiedGoalSatisfiedStop:
assert not verified_goal_satisfied_context(ctx)
+ def test_check_enforcement_stops_after_structural_built_unverified_terminal(self) -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ )
+ ],
+ ),
+ )
+
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is True
+ assert verified_goal_claim_authorized(ctx) is False
+ with pytest.raises(CopilotBuiltUnverified):
+ _check_enforcement(ctx)
+
def test_verified_goal_satisfied_context_rejects_undercovered_workflow(self) -> None:
from skyvern.forge.sdk.copilot.enforcement import verified_goal_satisfied_context
@@ -1170,7 +1212,8 @@ class TestVerifiedGoalSatisfiedStop:
assert result.updated_workflow is workflow
assert result.workflow_yaml == "workflow_definition:\n blocks: []\n"
- assert result.proposal_disposition == "auto_applicable"
+ assert result.proposal_disposition == "review_tested"
+ assert result.apply_without_review is False
# No adjudicated outcome evidence: the turn ends but the claim renders
# built-but-unverified instead of a tested-success claim.
assert "not independently verified" in result.user_response.lower()
@@ -1189,10 +1232,49 @@ class TestVerifiedGoalSatisfiedStop:
verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
)
verified = await _build_goal_satisfied_exit_result(ctx, global_llm_context=None)
+ assert verified.proposal_disposition == "auto_applicable"
assert "tested" in verified.user_response.lower()
assert verified.narrative_payload is not None
assert verified.narrative_payload["verifiedSuccess"] is True
+ @pytest.mark.asyncio
+ async def test_goal_satisfied_exit_result_renders_structural_abstention_as_built_unverified(self) -> None:
+ workflow = object()
+ ctx = _ctx(
+ last_workflow=workflow,
+ last_workflow_yaml="workflow_definition:\n blocks: []\n",
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ )
+ ],
+ ),
+ tool_activity=[{"tool": "update_and_run_blocks", "summary": "OK"}],
+ )
+
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is True
+ assert verified_goal_claim_authorized(ctx) is False
+
+ result = await _build_built_unverified_exit_result(ctx, global_llm_context=None)
+
+ assert result.updated_workflow is workflow
+ assert result.proposal_disposition == "review_tested"
+ assert result.apply_without_review is False
+ assert "not independently verified" in result.user_response.lower()
+ assert result.turn_outcome is not None
+ assert result.turn_outcome.terminal_reason == BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON
+ assert result.narrative_payload is not None
+ assert result.narrative_payload["verifiedSuccess"] is False
+
@pytest.mark.asyncio
async def test_goal_satisfied_exit_result_carries_outcome_adjudication(self) -> None:
from skyvern.forge.sdk.copilot.agent import _build_goal_satisfied_exit_result
@@ -1789,6 +1871,80 @@ class TestTranslateToAgentResultGating:
assert agent_result.clear_proposed_workflow is False
assert agent_result.proposal_disposition == "auto_applicable"
+ def test_wip_exit_structural_abstention_stays_review_only(self) -> None:
+ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+
+ workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_workflow=workflow,
+ last_workflow_yaml="title: Structural Draft",
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_unverified_no_repair_contract(),
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ )
+ ],
+ ),
+ )
+
+ agent_result = agent_module._build_wip_exit_result(
+ ctx,
+ global_llm_context=None,
+ default_reply="Timed out.",
+ unvalidated_reply="Draft needs review.",
+ tested_reply="Tested.",
+ terminal_reason="max_turns",
+ )
+
+ assert verified_goal_claim_authorized(ctx) is False
+ assert agent_result.updated_workflow is workflow
+ assert agent_result.proposal_disposition == "review_tested"
+ assert agent_result.apply_without_review is False
+ assert agent_result.narrative_payload is not None
+ assert agent_result.narrative_payload["verifiedSuccess"] is False
+
+ def test_wip_exit_verified_claim_still_auto_applies_code_only(self) -> None:
+ from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+
+ workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_workflow=workflow,
+ last_workflow_yaml="title: Verified Draft",
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ has_staged_proposal=True,
+ staged_workflow=workflow,
+ latest_diagnosis_repair_contract=_verified_goal_contract(),
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ ),
+ )
+
+ agent_result = agent_module._build_wip_exit_result(
+ ctx,
+ global_llm_context=None,
+ default_reply="Timed out.",
+ unvalidated_reply="Draft needs review.",
+ tested_reply="Tested.",
+ terminal_reason="max_turns",
+ )
+
+ assert verified_goal_claim_authorized(ctx) is True
+ assert agent_result.updated_workflow is workflow
+ assert agent_result.proposal_disposition == "auto_applicable"
+ assert agent_result.apply_without_review is True
+
def test_output_field_confirmation_question_is_blocked_when_contract_present(self) -> None:
ctx = _ctx(
request_policy=RequestPolicy(
@@ -2497,6 +2653,7 @@ workflow_definition:
assert agent_result.updated_workflow is wf
assert agent_result.proposal_disposition == "review_untested"
assert "could not verify the requested outcome" in agent_result.user_response.lower()
+ assert "did not demonstrate" not in agent_result.user_response.lower()
assert "output judge unavailable" in agent_result.user_response.lower()
assert "untested" not in agent_result.user_response.lower()
@@ -2535,6 +2692,11 @@ workflow_definition:
last_update_block_count=3,
has_staged_proposal=True,
staged_workflow=wf,
+ completion_verification_result=CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="satisfied", reason_code="evidence_confirms")],
+ ),
)
result = _fake_run_result({"type": "REPLY", "user_response": "All set."})
agent_result = asyncio.run(
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index fff4c744a..a95a2c660 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -330,6 +330,45 @@ def test_outcome_not_demonstrated_keeps_authoritative_unsatisfied_criteria_ident
assert "address and statuses" not in payload_text
+def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_structural",
+ "blocks": [
+ {
+ "label": "publish_result",
+ "status": "completed",
+ "extracted_data": {"document_name": "Resale Demand Package"},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.document_name",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_evaluated", workflow_run_id="wr_structural"),
+ completion_verification=verification,
+ )
+
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "failed_run"
+ assert outcome.is_authoritative is False
+
+
def test_authoring_repair_context_produces_structural_recorded_outcome() -> None:
context = CodeAuthoringRepairContext(
block_label="search_records",
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 4d20d9718..988ab3a83 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -46,6 +46,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
VerificationResult,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ built_unverified_repair_inert_context,
verified_goal_claim_authorized,
verified_goal_satisfied_context,
)
@@ -1003,6 +1004,18 @@ def _legacy_verified_ctx() -> CopilotContext:
)
+def _no_repair_unverified_contract() -> DiagnosisRepairContract:
+ return DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),
+ verification_result=VerificationResult(
+ user_goal_satisfied=False,
+ completion_contract_satisfied=False,
+ ),
+ )
+
+
def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:
ctx = _legacy_verified_ctx()
# Turn completion is unchanged: the legacy conjunction still satisfies the gate.
@@ -1011,6 +1024,45 @@ def test_claim_closure_turn_still_ends_but_claim_downgrades() -> None:
assert verified_goal_claim_authorized(ctx) is False
+def test_structural_abstention_no_repair_terminalizes_without_authorizing_success_claim() -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),
+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "structurally_abstained")),
+ )
+
+ assert ctx.completion_verification_result is not None
+ assert ctx.completion_verification_result.is_fully_satisfied() is False
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is True
+ assert verified_goal_claim_authorized(ctx) is False
+
+
+def test_structural_abstention_terminalization_rejects_real_unsatisfied_verdicts() -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),
+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "no_evidence")),
+ )
+
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is False
+
+
+def test_structural_abstention_terminalization_rejects_contradictions() -> None:
+ ctx = _ctx(
+ last_test_ok=True,
+ last_full_workflow_test_ok=True,
+ latest_diagnosis_repair_contract=_no_repair_unverified_contract(),
+ completion_verification_result=_evaluated(_verdict("c0", "unsatisfied", "evidence_contradicts")),
+ )
+
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is False
+
+
def test_claim_authorized_with_adjudicated_evidence() -> None:
ctx = _legacy_verified_ctx()
ctx.completion_verification_result = _evaluated(_verdict("c0", "satisfied", "evidence_confirms"))
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 9281b2614..ebcc32dcf 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
_verification_satisfaction,
)
from skyvern.forge.sdk.copilot.enforcement import (
+ built_unverified_repair_inert_context,
outcome_fully_verified,
verified_goal_satisfied_context,
)
@@ -2783,6 +2784,35 @@ def test_record_run_blocks_demonstrated_when_lone_definition_abstention_with_con
assert verified_goal_satisfied_context(ctx) is True
+def test_record_run_blocks_keeps_clean_structural_abstention_as_built_unverified() -> None:
+ ctx = _ctx_with_blocks("extraction")
+ verification = _mixed(
+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="structurally_abstained")
+ )
+
+ recorded = _record_run_blocks_result(ctx, _clean_success_result(), completion_verification=verification)
+
+ assert recorded is not None
+ assert recorded.verdict == "not_evaluated"
+ assert recorded.reason_code is None
+ assert ctx.last_test_suspicious_success is False
+ assert ctx.last_test_failure_reason is None
+ assert ctx.last_full_workflow_test_ok is True
+ ctx.latest_diagnosis_repair_contract = DiagnosisRepairContract(
+ diagnosis_input=DiagnosisInput(source_tool="update_and_run_blocks"),
+ diagnosis_result=DiagnosisResult(),
+ repair_decision=RepairDecision(next_action=RepairNextAction.NO_CHANGE),
+ verification_result=VerificationResult(user_goal_satisfied=False, completion_contract_satisfied=False),
+ )
+ assert verified_goal_satisfied_context(ctx) is False
+ assert built_unverified_repair_inert_context(ctx) is True
+ assert outcome_fully_verified(ctx) is False
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.is_authoritative is False
+
+
def test_committed_same_run_outcome_survives_later_contradictory_overwrite() -> None:
ctx = _ctx_with_blocks("extraction")
verification = _mixed(
@@ -3398,12 +3428,32 @@ def test_active_terminal_watchdog_exit_cannot_promote_to_terminal_success() -> N
def test_outcome_failure_warrants_repair() -> None:
has_block = _ctx_with_blocks("extraction")
nav_only = _ctx_with_blocks("goto_url", "navigation")
+ structural_abstention = _mixed(
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:lookup.missing_value",
+ )
+ )
+ mixed_abstention_and_failure = _mixed(
+ CriterionVerdict(
+ criterion_id="c_requested_output",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:lookup.missing_value",
+ ),
+ CriterionVerdict(criterion_id="c_real_failure", state="unsatisfied", reason_code="no_evidence"),
+ )
assert _outcome_failure_warrants_repair(nav_only, None) is False
# Contradiction is a real failure regardless of which blocks exist.
assert _outcome_failure_warrants_repair(nav_only, _contradicted("c0")) is True
# Absence of evidence: failure only once a confirmation block exists.
assert _outcome_failure_warrants_repair(has_block, _evaluated(("c0", False))) is True
assert _outcome_failure_warrants_repair(nav_only, _evaluated(("c0", False))) is False
+ assert structural_abstention.is_fully_satisfied() is False
+ assert _outcome_failure_warrants_repair(has_block, structural_abstention) is False
+ assert _outcome_failure_warrants_repair(has_block, mixed_abstention_and_failure) is True
# --- Direction 2: recognition governed by evidence, not run status ---------------
diff --git a/tests/unit/test_copilot_diagnosis_repair_contract.py b/tests/unit/test_copilot_diagnosis_repair_contract.py
index 60dc3d9f7..409a513dc 100644
--- a/tests/unit/test_copilot_diagnosis_repair_contract.py
+++ b/tests/unit/test_copilot_diagnosis_repair_contract.py
@@ -67,6 +67,14 @@ def _contradictory_completion_verification() -> CompletionVerificationResult:
)
+def _structural_abstention_completion_verification() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0"],
+ verdicts=[CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="structurally_abstained")],
+ )
+
+
def _clean_completed_result() -> dict[str, object]:
return {
"ok": True,
@@ -961,6 +969,27 @@ def test_clean_run_with_present_unsatisfied_completion_verification_fails_safe(
assert contract.verification_result.remaining_blocker is not None
+def test_clean_run_with_structural_abstention_completion_verification_does_not_repair() -> None:
+ ctx = _ctx()
+ ctx.completion_verification_result = _structural_abstention_completion_verification()
+
+ contract = build_diagnosis_repair_contract(
+ source_tool="update_and_run_blocks",
+ result=_clean_completed_result(),
+ ctx=ctx,
+ workflow_updated=True,
+ )
+ ctx.latest_diagnosis_repair_contract = contract
+
+ assert contract.diagnosis_result.suspected_failure_type == DiagnosisFailureType.NO_FAILURE
+ assert contract.repair_decision.next_action == RepairNextAction.NO_CHANGE
+ assert contract.repair_decision.next_action != RepairNextAction.REPAIR
+ assert contract.repair_decision.completion_check == "No repair selected; completion remains unverified."
+ assert contract.verification_result.user_goal_satisfied is False
+ assert contract.verification_result.completion_contract_satisfied is False
+ assert latest_diagnosis_contract_satisfies_goal(ctx) is False
+
+
def test_clean_run_with_satisfied_completion_verification_has_no_repair_or_blocker() -> None:
ctx = _ctx()
ctx.completion_verification_result = _satisfied_completion_verification()
diff --git a/tests/unit/test_copilot_response_coverage_gate.py b/tests/unit/test_copilot_response_coverage_gate.py
index a9aae5951..15505d001 100644
--- a/tests/unit/test_copilot_response_coverage_gate.py
+++ b/tests/unit/test_copilot_response_coverage_gate.py
@@ -53,6 +53,7 @@ class _Ctx:
self.composition_page_evidence = None
self.last_update_block_count = None
self.last_test_ok = None
+ self.last_full_workflow_test_ok = False
self.last_test_failure_reason = None
self.last_test_suspicious_success = False
self.last_test_anti_bot = None
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 564bc7b33..eae9dafd7 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -112,7 +112,7 @@ class TestBuildTimeoutExitResult:
assert result.updated_workflow is wf
assert result.workflow_yaml == "version: '1.0'"
- assert result.proposal_disposition == "auto_applicable"
+ assert result.proposal_disposition == "review_tested"
assert result.user_response == _TIMEOUT_REPLY_TESTED
def test_missing_yaml_drops_untested_proposal(self) -> None:
@@ -241,7 +241,7 @@ class TestBuildMaxTurnsExitResult:
result = _build_max_turns_exit_result(ctx, global_llm_context=None)
assert result.updated_workflow is wf
- assert result.proposal_disposition == "auto_applicable"
+ assert result.proposal_disposition == "review_tested"
assert result.user_response == _MAX_TURNS_REPLY_TESTED
def test_failed_test_drops_proposal(self) -> None:
@@ -330,7 +330,7 @@ class TestBuildUnexpectedErrorExitResult:
assert result.updated_workflow is wf
assert result.workflow_yaml == "version: '1.0'"
- assert result.proposal_disposition == "auto_applicable"
+ assert result.proposal_disposition == "review_tested"
assert result.user_response == _UNEXPECTED_ERROR_REPLY_TESTED
def test_failed_test_drops_proposal(self) -> None:
@@ -471,7 +471,7 @@ class TestBuildCancelExitResult:
assert result.cancelled is True
assert result.updated_workflow is wf
assert result.workflow_yaml == "version: '1.0'"
- assert result.proposal_disposition == "auto_applicable"
+ assert result.proposal_disposition == "review_tested"
assert result.user_response == _CANCEL_REPLY_TESTED
def test_failed_test_drops_cancel_proposal(self) -> None: