Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11865: mint-time satisfiability + typed fail-safe degrade on the classifier-fallback completion producer (#7135)
Andrew Neilsonfdfaf3bbf705
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 5248bd0bd..62b90f67f 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -13,12 +13,16 @@ import yamlfrom pydantic import BaseModel, ConfigDict, Fieldfrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict+from skyvern.forge.sdk.copilot.completion_verification import (+ CompletionVerificationResult,+ CriterionVerdict,+ only_degraded_blocking,+)from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schemafrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence-from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, RunOutcomeReasonCodeLOG = structlog.get_logger()@@ -51,6 +55,7 @@ BuildTestOutcomeReasonCode = Literal["output_policy_reject","scout_act_observe_hollow_after_interaction","required_input_unbound",+ "fallback_floor_turn_unsatisfiable",]_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"@@ -175,6 +180,19 @@ BindingFrontierFacet = Literal["selector_frontier",]_UNCROSSABLE_DIAGNOSTIC_REASONS = frozenset({"empty_page", "challenge_gated", "capture_degraded"})+_AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODES: frozenset[RunOutcomeReasonCode] = frozenset(+ {"outcome_not_demonstrated", "no_meaningful_output"}+)+++def _recorded_outcome_degrade_eligible(+ recorded_run_outcome: RecordedRunOutcome,+ failed_block: Mapping[str, object] | None,+) -> bool:+ if failed_block is not None:+ return False+ reason_code = recorded_run_outcome.reason_code+ return reason_code is None or reason_code in _AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODESclass RecordedOutcomeBindingConstraint(BaseModel):@@ -956,6 +974,22 @@ def recorded_outcome_from_run_blocks_result(observed_evidence_summary=recorded_run_outcome.display_reason or "",key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},)+ if (+ completion_verification is not None+ and only_degraded_blocking(completion_verification)+ and _recorded_outcome_degrade_eligible(recorded_run_outcome, failed_block)+ ):+ return RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="not_authoritative",+ reason_code="fallback_floor_turn_unsatisfiable",+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,+ block_labels=block_labels,+ authored_structure_signature=authored_structure_signature,+ observed_evidence_summary=recorded_run_outcome.display_reason or "",+ key_provenance={"structural_failure_identity": "turn-unsatisfiable fallback floor, no reachable route"},+ )return RecordedBuildTestOutcome(phase="persisted_block_run",attempted_tool="update_and_run_blocks",@@ -1008,12 +1042,22 @@ def recorded_outcome_from_run_blocks_result(verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"if not structural_identity and not page_refs and not output_refs:verdict = "not_authoritative"+ reason_code = "runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run"+ has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)+ if (+ verdict == "repairable_failure"+ and not has_runtime_failure_evidence+ and completion_verification is not None+ and only_degraded_blocking(completion_verification)+ ):+ verdict = "not_authoritative"+ reason_code = "fallback_floor_turn_unsatisfiable"return RecordedBuildTestOutcome(phase="persisted_block_run",attempted_tool="update_and_run_blocks",attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",verdict=verdict,- reason_code="runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run",+ reason_code=reason_code,workflow_run_id=workflow_run_id or None,block_labels=block_labels,structural_failure_identity=structural_identity,diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex 70a22fa04..230dd15b7 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -145,6 +145,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion}if criterion.requested_output_corroborator:item["requested_output_corroborator"] = True+ if criterion.mint_degrade is not None:+ item["mint_degrade"] = criterion.mint_degradeitems.append(item)return items@@ -217,6 +219,9 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:classification_output_key=classification_output_key,expected_classification=expected_classification,requested_output_corroborator=bool(item.get("requested_output_corroborator")),+ mint_degrade="turn_unsatisfiable_fallback"+ if item.get("mint_degrade") == "turn_unsatisfiable_fallback"+ else None,))return tuple(criteria)@@ -225,7 +230,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:contingent_key = criterion.contingent_on or ""contingent_path_key = criterion.contingent_antecedent_output_path or ""- deliverable_kind_key = criterion.deliverable_kind or ""+ deliverable_kind_key = f"{criterion.deliverable_kind or ''}\x1fmint_degrade:{criterion.mint_degrade or ''}"expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)expected_output_shape_key = criterion.expected_output_shape or ""requested_output_evidence_source_key = criterion.requested_output_evidence_source@@ -379,6 +384,7 @@ def apply_requested_output_producer_floor(if criterion.expected_output_value is Noneand criterion.expected_output_shape is Noneand criterion.deliverable_kind is None+ and criterion.mint_degrade is None}if not presence_only_ids:return criteria, ()diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 9b77339af..46b8e42ba 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,TerminalActionFamily,is_fallback_floor_base_criterion,+ is_turn_unsatisfiable_fallback_degraded,redact_raw_secrets_for_prompt,)from skyvern.utils.strings import escape_code_fences@@ -111,6 +112,7 @@ class CompletionVerificationResult:contingent_on_by_criterion_id: dict[str, str] = field(default_factory=dict)contingent_antecedent_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)structural_unfired_criterion_ids: list[str] = field(default_factory=list)+ degraded_criterion_ids: list[str] = field(default_factory=list)def is_fully_satisfied(self) -> bool:if self.status != "evaluated" or not self.criterion_ids:@@ -123,6 +125,7 @@ class CompletionVerificationResult:_is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)for verdict in verdict_by_id.values())+ degraded_id_set = set(self.degraded_criterion_ids)satisfied_run_plane_count = 0for criterion_id in self.criterion_ids:verdict = verdict_by_id.get(criterion_id)@@ -133,6 +136,10 @@ class CompletionVerificationResult:if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):satisfied_run_plane_count += 1continue+ # A turn-unsatisfiable fallback criterion has no reachable satisfaction route,+ # so an unsatisfied one can never be credited via corroboration or value-blind excusal.+ if criterion_id in degraded_id_set:+ return False# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,# so it must not veto a run whose observable outcome evidence is fully confirmed.if verdict is not None and _is_definition_plane_abstention(verdict):@@ -495,6 +502,8 @@ def structural_unfired_contingent_criterion_ids() -> list[str]:unfired_ids: list[str] = []for criterion in criteria:+ if is_turn_unsatisfiable_fallback_degraded(criterion):+ continuepath = criterion.contingent_antecedent_output_pathif not path:continue@@ -1266,7 +1275,10 @@ def grade_fallback_floor_reached_end_state_criteria(eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]if not eligible_criteria:return []- evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)+ floor_degraded = any(is_turn_unsatisfiable_fallback_degraded(criterion) for criterion in eligible_criteria)+ evidence_ref = None+ if not floor_degraded:+ evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)if evidence_ref is None:evidence_ref = _fallback_floor_carrier_evidence_ref(snapshot, carrier_verdicts)if evidence_ref is None:@@ -2153,6 +2165,7 @@ def combine_verification_results(contingent_on_by_id = dict(contingent_on_by_criterion_id or {})contingent_path_by_id = dict(contingent_antecedent_output_path_by_criterion_id or {})structural_unfired_ids = list(structural_unfired_criterion_ids)+ degraded_ids: list[str] = []if run_result is not None:contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))contingent_on_by_id.update(run_result.contingent_on_by_criterion_id)@@ -2160,6 +2173,7 @@ def combine_verification_results(structural_unfired_ids = list(dict.fromkeys([*structural_unfired_ids, *run_result.structural_unfired_criterion_ids]))+ degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))if run_result is not None and run_result.status != "evaluated":return CompletionVerificationResult(status=run_result.status,@@ -2169,6 +2183,7 @@ def combine_verification_results(contingent_on_by_criterion_id=contingent_on_by_id,contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,structural_unfired_criterion_ids=structural_unfired_ids,+ degraded_criterion_ids=degraded_ids,)if run_result is not None:run_result = replace(@@ -2197,9 +2212,47 @@ def combine_verification_results(contingent_on_by_criterion_id=contingent_on_by_id,contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,structural_unfired_criterion_ids=structural_unfired_ids,+ degraded_criterion_ids=degraded_ids,)+def only_degraded_blocking(result: CompletionVerificationResult) -> bool:+ """True only when every criterion that still blocks full satisfaction is a+ turn-unsatisfiable fallback criterion. A genuine unsatisfied/unknown non-degraded+ criterion, an empty blocking set, or no degraded ids all return False, so this+ never masks a real runtime failure or a mixed degraded+legitimate result."""+ if result.status != "evaluated":+ return False+ degraded = set(result.degraded_criterion_ids)+ if not degraded:+ return False+ verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}+ blocking: set[str] = set()+ for criterion_id in result.criterion_ids:+ verdict = verdict_by_id.get(criterion_id)+ if verdict is not None and verdict.satisfied:+ continue+ if verdict is not None and _is_definition_plane_abstention(verdict):+ continue+ if verdict is not None and result.is_structural_contingent_abstention(verdict):+ continue+ blocking.add(criterion_id)+ if not blocking:+ return False+ return blocking <= degraded+++def carry_degraded_criterion_ids(+ result: CompletionVerificationResult,+ criteria: Iterable[CompletionCriterion],+) -> CompletionVerificationResult:+ degraded = [criterion.id for criterion in criteria if is_turn_unsatisfiable_fallback_degraded(criterion)]+ if not degraded and not result.degraded_criterion_ids:+ return result+ merged = list(dict.fromkeys([*result.degraded_criterion_ids, *degraded]))+ return replace(result, degraded_criterion_ids=merged)++async def evaluate_completion_criteria(criteria: list[CompletionCriterion],snapshot: RunEvidenceSnapshot,diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex f22697499..38daae9d7 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -267,6 +267,7 @@ class CompletionCriterion:classification_output_key: str | None = Noneexpected_classification: ClassificationTarget | None = Nonerequested_output_corroborator: bool = False+ mint_degrade: Literal["turn_unsatisfiable_fallback"] | None = None@dataclass@@ -349,6 +350,8 @@ class RequestPolicy:data[f"{prefix}_output_path"] = criterion.output_pathdata[f"{prefix}_grounding_mode"] = _criterion_grounding_mode(criterion)data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None+ if criterion.mint_degrade is not None:+ data[f"{prefix}_mint_degrade"] = criterion.mint_degradedata[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_sourceif criterion.expected_output_shape:data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape@@ -1361,7 +1364,7 @@ def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicydef _apply_requested_output_completion_criteria(- policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None+ policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None, *, extract_literals: bool = False) -> None:schema_aliases = schema_output_path_aliases_from_criteria(policy.completion_criteria)config_aliases = _normalize_requested_output_aliases(aliases)@@ -1391,6 +1394,9 @@ def _apply_requested_output_completion_criteria(returnvalue_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)+ if extract_literals:+ for output_path, literal in _fallback_literal_expected_values(user_message, requested_specs).items():+ value_by_output_path.setdefault(output_path, literal)shape_by_output_path = _requested_output_shapes_from_criteria(policy.completion_criteria, requested_specs)source_by_output_path = _requested_output_evidence_sources_from_criteria(policy.completion_criteria, requested_specs@@ -1555,6 +1561,88 @@ def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:return criterion.id == _FALLBACK_FLOOR_BASE_ID+def is_turn_unsatisfiable_fallback_degraded(criterion: CompletionCriterion) -> bool:+ return criterion.mint_degrade == "turn_unsatisfiable_fallback"+++_FALLBACK_LITERAL_MIN_CHARS = 4+_FALLBACK_LITERAL_BINDER_RE = r"(?:equal to|equals|should be|must be|is expected to be|expected to be|will be|is|:|=)"+_FALLBACK_LITERAL_MAX_SCAN_CHARS = 4000+++def _fallback_literal_field_surface_forms(field_name: str, field_label: str) -> list[str]:+ forms: set[str] = set()+ for source in (field_name, field_label):+ collapsed = " ".join(source.split()).strip()+ if collapsed:+ forms.add(collapsed)+ forms.add(collapsed.replace(" ", "_"))+ return [form for form in sorted(forms, key=len, reverse=True) if form]+++def _fallback_literal_excluded_forms(field_name: str, field_label: str, output_path: str) -> set[str]:+ forms = {+ " ".join(_word_tokens(field_name)),+ " ".join(_word_tokens(field_label)),+ " ".join(_word_tokens(output_path)),+ }+ return {form for form in forms if form}+++def _fallback_literal_candidates_for_field(user_message: str, field_name: str, field_label: str) -> list[str]:+ bounded_message = user_message[:_FALLBACK_LITERAL_MAX_SCAN_CHARS]+ candidates: list[str] = []+ for surface in _fallback_literal_field_surface_forms(field_name, field_label):+ pattern = re.compile(+ r"\b"+ + re.escape(surface)+ + r"\b\s{0,8}+(?:"+ + _FALLBACK_LITERAL_BINDER_RE+ + r"\s{0,8}+)?(?P<quote>['\"`])(?P<quoted>[^'\"`]{1,80}+)(?P=quote)",+ re.IGNORECASE,+ )+ for match in pattern.finditer(bounded_message):+ quoted = match.group("quoted")+ if quoted is not None:+ candidates.append(quoted)+ return candidates+++def _fallback_literal_expected_values(+ user_message: str, requested_specs: list[tuple[str, str, str]]+) -> dict[str, ExpectedOutputValue]:+ if not user_message.strip():+ return {}+ values: dict[str, ExpectedOutputValue] = {}+ for field_name, output_path, field_label in requested_specs:+ excluded = _fallback_literal_excluded_forms(field_name, field_label, output_path)+ accepted: list[str] = []+ for raw in _fallback_literal_candidates_for_field(user_message, field_name, field_label):+ coerced = _coerce_expected_output_value(raw)+ if not isinstance(coerced, str) or len(coerced) < _FALLBACK_LITERAL_MIN_CHARS:+ continue+ if " ".join(_word_tokens(coerced)) in excluded:+ continue+ accepted.append(coerced)+ unique = list(dict.fromkeys(accepted))+ if len(unique) == 1:+ values[output_path] = unique[0]+ return values+++def _mark_turn_unsatisfiable_fallback_criteria(policy: RequestPolicy) -> None:+ marked: list[CompletionCriterion] = []+ for criterion in policy.completion_criteria:+ value_less = criterion.expected_output_value is None and criterion.expected_output_shape is None+ is_floor_base = is_fallback_floor_base_criterion(criterion)+ is_requested_output = criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)+ if value_less and (is_floor_base or is_requested_output):+ marked.append(replace(criterion, mint_degrade="turn_unsatisfiable_fallback"))+ else:+ marked.append(criterion)+ policy.completion_criteria = marked++def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:floor = [CompletionCriterion(@@ -1617,8 +1705,11 @@ def _classifier_fallback_policy(classifier_retry_count=retry_count,completion_contract_status="present" if fallback_criteria else "unknown",)- _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)+ _apply_requested_output_completion_criteria(+ policy, user_message, requested_output_path_aliases, extract_literals=True+ )_apply_classifier_typed_requested_output_corroborators(policy)+ _mark_turn_unsatisfiable_fallback_criteria(policy)if policy.graded_completion_criteria():policy.completion_contract_status = "present"return policydiff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 5296a3794..5a70c642e 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (RunEvidenceSnapshot,_contingent_metadata_for_criteria,_is_structural_requested_output_abstention,+ carry_degraded_criterion_ids,combine_verification_results,evaluate_completion_criteria,grade_definition_criteria,@@ -32,6 +33,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_validation_classification_criteria,is_fallback_floor_base_criterion,is_registered_download_completion_criterion,+ only_degraded_blocking,registered_download_completion_criterion,structural_unfired_contingent_criterion_ids,summarize_unsatisfied_outcomes,@@ -399,6 +401,7 @@ async def _maybe_run_completion_verification_from_page_observation()if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:verification = _no_gradeable_run_plane_result(criterion_ids)+ verification = carry_degraded_criterion_ids(verification, criteria)copilot_ctx.completion_verification_result = verificationrecord_completion_verification(copilot_ctx, verification)_record_adjudication_on_turn_state(copilot_ctx, verification)@@ -552,6 +555,7 @@ async def _maybe_run_completion_verification_from_page_observation(contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,)+ verification = carry_degraded_criterion_ids(verification, criteria)if (isinstance(existing, CompletionVerificationResult)and not verification.is_fully_satisfied()@@ -1156,6 +1160,13 @@ async def _maybe_run_completion_verification(return Nonecriteria = _completion_verification_criteria(copilot_ctx)criteria = _reconcile_download_completion_criterion(copilot_ctx, result, criteria)+ verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)+ return carry_degraded_criterion_ids(verification, criteria) if verification is not None else None+++async def _completion_verification_from_run_result(+ copilot_ctx: Any, result: dict[str, Any], handler_start: float, criteria: list[CompletionCriterion]+) -> CompletionVerificationResult | None:run_criteria, definition_criteria = _split_criteria_by_plane(criteria)criterion_ids = [criterion.id for criterion in criteria]contingent_ids, contingent_on_by_id, contingent_path_by_id = _contingent_metadata_for_criteria(criteria)@@ -1469,6 +1480,8 @@ def _outcome_failure_warrants_repair("""if completion_verification is None:return False+ if only_degraded_blocking(completion_verification):+ return Falseif any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):return True# Repair needs at least one affirmatively unsatisfied criterion; unknown alonediff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 9c48b68e7..280a3b3d2 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -2345,9 +2345,8 @@ def _terminal_challenge_completion_verification(criterion_ids = list(completion_verification.criterion_ids)if not criterion_ids:return completion_verification- return CompletionVerificationResult(- status="evaluated",- criterion_ids=criterion_ids,+ return replace(+ completion_verification,verdicts=[CriterionVerdict(criterion_id=criterion_id,diff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.pynew file mode 100644index 000000000..b0801597d--- /dev/null+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py@@ -0,0 +1,471 @@+from __future__ import annotations++import time+from dataclasses import replace++from skyvern.forge.sdk.copilot.build_test_outcome import recorded_outcome_from_run_blocks_result+from skyvern.forge.sdk.copilot.completion_criteria_store import (+ _criterion_reconcile_key,+ apply_requested_output_producer_floor,+ criteria_from_json,+ criteria_to_json,+)+from skyvern.forge.sdk.copilot.completion_verification import (+ CompletionVerificationResult,+ CriterionVerdict,+ RunEvidenceSnapshot,+ carry_degraded_criterion_ids,+ combine_verification_results,+ grade_fallback_floor_reached_end_state_criteria,+ only_degraded_blocking,+ structural_unfired_contingent_criterion_ids,+)+from skyvern.forge.sdk.copilot.request_policy import (+ CompletionCriterion,+ _classifier_fallback_policy,+ _fallback_literal_candidates_for_field,+ build_classifier_fallback_floor,+ is_turn_unsatisfiable_fallback_degraded,+)+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome+from skyvern.forge.sdk.copilot.tools.run_execution import _terminal_challenge_completion_verification++_DEGRADE = "turn_unsatisfiable_fallback"+++def _fallback_policy(user_message: str):+ return _classifier_fallback_policy(+ [], raw_secret_present=False, failure_kind="provider_error", user_message=user_message+ )+++def _requested_criterion(policy, suffix: str) -> CompletionCriterion | None:+ for criterion in policy.completion_criteria:+ if criterion.id.endswith(suffix):+ return criterion+ return None+++def _validation_review_payload() -> dict[str, object]:+ return {+ "all_checks_passed": True,+ "validation_only": True,+ "review_page_visible": True,+ "submit_or_finalize_clicked": False,+ "submitted_request": False,+ "confirmation_page_visible": False,+ "review_values": {+ "visible_reference": "Reference AB123456",+ "visible_start_date": "2026-06-22",+ "visible_account_holder": "EXAMPLE LABS INC",+ },+ "evidence_text": (+ "Visible Review page showed Reference AB123456, start date 2026-06-22, "+ "and account holder EXAMPLE LABS INC. No final control was clicked."+ ),+ }+++def test_quoted_literal_mints_typed_value_and_leaves_floor_degraded() -> None:+ policy = _fallback_policy('Go to the portal and return the order number "ABC12345" and the status')++ minted = _requested_criterion(policy, "output_order_number")+ assert minted is not None+ assert minted.expected_output_value == "ABC12345"+ assert minted.mint_degrade is None++ floor = _requested_criterion(policy, "__copilot_fallback_floor__run")+ assert floor is not None+ assert floor.mint_degrade == _DEGRADE+++def test_unquoted_value_noun_degrades_instead_of_minting() -> None:+ policy = _fallback_policy("read the record; the status should be active for each location")++ minted = _requested_criterion(policy, "output_status")+ assert minted is not None+ assert minted.expected_output_value is None+ assert minted.mint_degrade == _DEGRADE+++def test_extractor_mints_quoted_literal_after_binder() -> None:+ assert set(_fallback_literal_candidates_for_field('the status should be "active"', "status", "Status")) == {+ "active"+ }+++def test_extractor_rejects_unquoted_participle_predicate() -> None:+ assert _fallback_literal_candidates_for_field("the status should be updated", "status", "Status") == []+++def test_extractor_rejects_unquoted_verb_predicate() -> None:+ assert _fallback_literal_candidates_for_field("the status should be filled", "status", "Status") == []+++def test_extractor_rejects_unquoted_code_after_binder() -> None:+ assert _fallback_literal_candidates_for_field("order_id must be ABC12345", "order_id", "Order Id") == []+++def test_quoted_field_name_is_not_minted_as_value() -> None:+ policy = _fallback_policy('return the "status" field and the name for the record')++ minted = _requested_criterion(policy, "output_status")+ assert minted is None or minted.expected_output_value is None+++def test_multiple_literal_candidates_mint_nothing() -> None:+ policy = _fallback_policy("the status should be active but the status should be pending")++ minted = _requested_criterion(policy, "output_status")+ assert minted is None or minted.expected_output_value is None+++def test_short_literal_degrades_instead_of_minting() -> None:+ policy = _fallback_policy('return the status "ok" for the record')++ minted = _requested_criterion(policy, "output_status")+ assert minted is None or minted.expected_output_value is None+++def test_value_less_fallback_marks_floor_and_requested_outputs_degraded() -> None:+ policy = _fallback_policy("return the status for the record")++ degraded = [c for c in policy.completion_criteria if is_turn_unsatisfiable_fallback_degraded(c)]+ assert any(c.id == "__copilot_fallback_floor__run" for c in degraded)+ assert any(c.id.endswith("output_status") for c in degraded)+++def _degraded_floor() -> list[CompletionCriterion]:+ return [replace(criterion, mint_degrade=_DEGRADE) for criterion in build_classifier_fallback_floor([])]+++def _carrier_verdict() -> CriterionVerdict:+ return CriterionVerdict(+ criterion_id="__copilot_requested_output__output_reference",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="block_outputs:lookup",+ evidence_source="independent_page_evidence",+ )+++def test_degraded_floor_still_satisfies_via_carrier_arm() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={})+ verdicts = grade_fallback_floor_reached_end_state_criteria(+ _degraded_floor(), snapshot, carrier_verdicts=(_carrier_verdict(),)+ )+ assert [verdict.state for verdict in verdicts] == ["satisfied"]+++def test_degraded_floor_blocks_reached_end_state_arm() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})+ assert grade_fallback_floor_reached_end_state_criteria(_degraded_floor(), snapshot) == []+++def test_non_degraded_floor_accepts_reached_end_state_arm() -> None:+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})+ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)+ assert [verdict.state for verdict in verdicts] == ["satisfied"]+++def test_structural_unfired_skips_degraded_criterion() -> None:+ criterion = CompletionCriterion(+ id="c_degraded",+ outcome="captured blocker",+ level="run",+ contingent_antecedent_output_path="output.blocker",+ mint_degrade=_DEGRADE,+ )+ snapshot = RunEvidenceSnapshot(block_outputs={"blocker_output": None})+ assert structural_unfired_contingent_criterion_ids([criterion], snapshot) == []+++def _degraded_abstention_result() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["floor", "req"],+ verdicts=[+ CriterionVerdict(criterion_id="floor", state="satisfied", reason_code="evidence_confirms"),+ CriterionVerdict(+ criterion_id="req",+ state="unsatisfied",+ reason_code="structurally_abstained",+ evidence_ref="block_outputs:lookup",+ output_path="output.status",+ ),+ ],+ degraded_criterion_ids=["req"],+ )+++def test_degraded_abstention_is_not_credited_even_with_corroboration() -> None:+ assert _degraded_abstention_result().is_fully_satisfied() is False+++def test_non_degraded_abstention_credit_path_is_not_broken() -> None:+ result = replace(_degraded_abstention_result(), degraded_criterion_ids=[])+ # A satisfied observed-end-state corroborator credits the structural abstention when NOT degraded.+ result = replace(+ result,+ verdicts=[+ CriterionVerdict(+ criterion_id="floor",+ state="satisfied",+ reason_code="evidence_confirms",+ evidence_ref="observed_end_state_url",+ ),+ result.verdicts[1],+ ],+ )+ assert result.is_fully_satisfied() is True+++def test_sky11842_zero_evidence_non_fallback_still_fails_closed() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c1"],+ verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],+ )+ assert result.is_fully_satisfied() is False+ assert only_degraded_blocking(result) is False+++def test_degraded_criterion_ids_survive_combine_and_carry() -> None:+ run_result = carry_degraded_criterion_ids(+ CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["req"],+ verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],+ ),+ [CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)],+ )+ assert run_result is not None+ combined = combine_verification_results(["req"], run_result, [])+ carried = carry_degraded_criterion_ids(+ combined, [CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)]+ )+ assert carried is not None+ assert "req" in carried.degraded_criterion_ids+++def test_combine_alone_propagates_degraded_criterion_ids() -> None:+ run_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["req"],+ verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],+ degraded_criterion_ids=["req"],+ )+ combined = combine_verification_results(["req"], run_result, [])+ assert combined.degraded_criterion_ids == ["req"]+ assert only_degraded_blocking(combined) is True+++def test_fallback_literal_candidates_bounded_runtime_on_adversarial_message() -> None:+ adversarial = "status" + (" " * 20 + "=") * 4000+ start = time.perf_counter()+ _fallback_literal_candidates_for_field(adversarial, "status", "Status")+ assert time.perf_counter() - start < 2.0+++def test_degraded_criterion_ids_survive_terminal_challenge() -> None:+ criteria = [+ CompletionCriterion(id="__copilot_fallback_floor__run", outcome="x", level="run", mint_degrade=_DEGRADE)+ ]+ result = carry_degraded_criterion_ids(+ CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["__copilot_fallback_floor__run"],+ verdicts=[+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"+ )+ ],+ ),+ criteria,+ )+ assert result is not None+ assert result.degraded_criterion_ids == ["__copilot_fallback_floor__run"]++ challenged = _terminal_challenge_completion_verification(result, "run did not reach a verifiable terminal state")+ assert challenged is not None+ assert challenged.degraded_criterion_ids == ["__copilot_fallback_floor__run"]+ assert only_degraded_blocking(challenged) is True+++def test_only_degraded_blocking_true_for_degraded_only() -> None:+ assert only_degraded_blocking(_degraded_abstention_result()) is True+++def test_only_degraded_blocking_false_on_empty_blocking_set() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["req"],+ verdicts=[CriterionVerdict(criterion_id="req", state="satisfied", reason_code="evidence_confirms")],+ degraded_criterion_ids=["req"],+ )+ assert only_degraded_blocking(result) is False+++def test_only_degraded_blocking_false_when_mixed_with_legitimate_unsatisfied() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["req", "legit"],+ verdicts=[+ CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence"),+ CriterionVerdict(criterion_id="legit", state="unsatisfied", reason_code="evidence_contradicts"),+ ],+ degraded_criterion_ids=["req"],+ )+ assert only_degraded_blocking(result) is False+++def test_mint_degrade_survives_json_round_trip() -> None:+ criteria = (CompletionCriterion(id="req", outcome="return the status", level="run", mint_degrade=_DEGRADE),)+ restored = criteria_from_json(criteria_to_json(criteria))+ assert restored[0].mint_degrade == _DEGRADE+++def test_mint_degrade_is_identity_bearing_in_reconcile_key() -> None:+ base = CompletionCriterion(id="req", outcome="return the status", level="run", output_path="output.status")+ degraded = replace(base, mint_degrade=_DEGRADE)+ assert _criterion_reconcile_key(base) != _criterion_reconcile_key(degraded)+++def test_producer_floor_excludes_degraded_from_rekey() -> None:+ degraded = CompletionCriterion(+ id="__copilot_requested_output__output_status",+ outcome="return the status",+ level="run",+ output_path="output.status",+ mint_degrade=_DEGRADE,+ )+ floored, rekeyed = apply_requested_output_producer_floor([degraded])+ assert rekeyed == ()+ assert floored[0].output_path == "output.status"+++def _degraded_verification() -> CompletionVerificationResult:+ return CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["__copilot_fallback_floor__run"],+ verdicts=[+ CriterionVerdict(+ criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"+ )+ ],+ degraded_criterion_ids=["__copilot_fallback_floor__run"],+ )+++def test_recorded_branch_degraded_only_is_not_repairable() -> None:+ result = {+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"+++def test_raw_branch_degraded_only_is_not_repairable() -> None:+ result = {+ "ok": False,+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]},+ }+ outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())+ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"+++def test_raw_branch_runtime_failure_stays_repairable() -> None:+ result = {+ "ok": False,+ "data": {+ "workflow_run_id": "wr_x",+ "failure_type": "TERMINATED",+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],+ },+ }+ outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())+ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+++def test_recorded_branch_blocker_reported_stays_repairable() -> None:+ result = {+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="blocker_reported"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code == "blocker_reported"+++def test_recorded_branch_no_reason_code_with_failed_block_stays_repairable() -> None:+ result = {+ "data": {+ "workflow_run_id": "wr_x",+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],+ }+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+++def test_recorded_branch_no_reason_code_without_failed_block_degrades() -> None:+ result = {+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"+++def test_recorded_branch_no_meaningful_output_degrades() -> None:+ result = {+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="no_meaningful_output"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "not_authoritative"+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"+++def test_recorded_branch_ambiguous_code_with_failed_block_stays_repairable() -> None:+ result = {+ "data": {+ "workflow_run_id": "wr_x",+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],+ }+ }+ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),+ completion_verification=_degraded_verification(),+ )+ assert outcome is not None+ assert outcome.verdict == "repairable_failure"+ assert outcome.reason_code != "fallback_floor_turn_unsatisfiable"