Skip to content

Automate browser based workflows with AI

SKY-11865: mint-time satisfiability + typed fail-safe degrade on the classifier-fallback completion producer (#7135)

Andrew Neilsonfdfaf3bbf705
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 5248bd0bd..62b90f67f 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -13,12 +13,16 @@ import yaml
from pydantic import BaseModel, ConfigDict, Field
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ only_degraded_blocking,
+)
from skyvern.forge.sdk.copilot.composition_evidence import has_bounded_page_schema
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
-from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome, RunOutcomeReasonCode
LOG = structlog.get_logger()
@@ -51,6 +55,7 @@ BuildTestOutcomeReasonCode = Literal[
"output_policy_reject",
"scout_act_observe_hollow_after_interaction",
"required_input_unbound",
+ "fallback_floor_turn_unsatisfiable",
]
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
@@ -175,6 +180,19 @@ BindingFrontierFacet = Literal[
"selector_frontier",
]
_UNCROSSABLE_DIAGNOSTIC_REASONS = frozenset({"empty_page", "challenge_gated", "capture_degraded"})
+_AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODES: frozenset[RunOutcomeReasonCode] = frozenset(
+ {"outcome_not_demonstrated", "no_meaningful_output"}
+)
+
+
+def _recorded_outcome_degrade_eligible(
+ recorded_run_outcome: RecordedRunOutcome,
+ failed_block: Mapping[str, object] | None,
+) -> bool:
+ if failed_block is not None:
+ return False
+ reason_code = recorded_run_outcome.reason_code
+ return reason_code is None or reason_code in _AMBIGUOUS_NON_DEMONSTRATION_RUN_REASON_CODES
class RecordedOutcomeBindingConstraint(BaseModel):
@@ -956,6 +974,22 @@ def recorded_outcome_from_run_blocks_result(
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={"structural_failure_identity": "no typed verification/page/output identity available"},
)
+ if (
+ completion_verification is not None
+ and only_degraded_blocking(completion_verification)
+ and _recorded_outcome_degrade_eligible(recorded_run_outcome, failed_block)
+ ):
+ return RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="not_authoritative",
+ reason_code="fallback_floor_turn_unsatisfiable",
+ workflow_run_id=recorded_run_outcome.workflow_run_id or workflow_run_id or None,
+ block_labels=block_labels,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=recorded_run_outcome.display_reason or "",
+ key_provenance={"structural_failure_identity": "turn-unsatisfiable fallback floor, no reachable route"},
+ )
return RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
@@ -1008,12 +1042,22 @@ def recorded_outcome_from_run_blocks_result(
verdict: BuildTestOutcomeVerdict = "repairable_failure" if bool(result.get("ok")) is False else "progress_observed"
if not structural_identity and not page_refs and not output_refs:
verdict = "not_authoritative"
+ reason_code = "runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run"
+ has_runtime_failure_evidence = bool(failure_categories or failure_type or runtime_failure_identity or failed_block)
+ if (
+ verdict == "repairable_failure"
+ and not has_runtime_failure_evidence
+ and completion_verification is not None
+ and only_degraded_blocking(completion_verification)
+ ):
+ verdict = "not_authoritative"
+ reason_code = "fallback_floor_turn_unsatisfiable"
return RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
attempted_block_label=_safe_str(failed_block.get("label")) if failed_block is not None else "",
verdict=verdict,
- reason_code="runtime_block_failure" if failed_block is not None or not bool(result.get("ok")) else "failed_run",
+ reason_code=reason_code,
workflow_run_id=workflow_run_id or None,
block_labels=block_labels,
structural_failure_identity=structural_identity,
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 70a22fa04..230dd15b7 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -145,6 +145,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
}
if criterion.requested_output_corroborator:
item["requested_output_corroborator"] = True
+ if criterion.mint_degrade is not None:
+ item["mint_degrade"] = criterion.mint_degrade
items.append(item)
return items
@@ -217,6 +219,9 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
classification_output_key=classification_output_key,
expected_classification=expected_classification,
requested_output_corroborator=bool(item.get("requested_output_corroborator")),
+ mint_degrade="turn_unsatisfiable_fallback"
+ if item.get("mint_degrade") == "turn_unsatisfiable_fallback"
+ else None,
)
)
return tuple(criteria)
@@ -225,7 +230,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
- deliverable_kind_key = criterion.deliverable_kind or ""
+ deliverable_kind_key = f"{criterion.deliverable_kind or ''}\x1fmint_degrade:{criterion.mint_degrade or ''}"
expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)
expected_output_shape_key = criterion.expected_output_shape or ""
requested_output_evidence_source_key = criterion.requested_output_evidence_source
@@ -379,6 +384,7 @@ def apply_requested_output_producer_floor(
if criterion.expected_output_value is None
and criterion.expected_output_shape is None
and criterion.deliverable_kind is None
+ and criterion.mint_degrade is None
}
if not presence_only_ids:
return criteria, ()
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 9b77339af..46b8e42ba 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -27,6 +27,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
TerminalActionFamily,
is_fallback_floor_base_criterion,
+ is_turn_unsatisfiable_fallback_degraded,
redact_raw_secrets_for_prompt,
)
from skyvern.utils.strings import escape_code_fences
@@ -111,6 +112,7 @@ class CompletionVerificationResult:
contingent_on_by_criterion_id: dict[str, str] = field(default_factory=dict)
contingent_antecedent_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)
structural_unfired_criterion_ids: list[str] = field(default_factory=list)
+ degraded_criterion_ids: list[str] = field(default_factory=list)
def is_fully_satisfied(self) -> bool:
if self.status != "evaluated" or not self.criterion_ids:
@@ -123,6 +125,7 @@ class CompletionVerificationResult:
_is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)
for verdict in verdict_by_id.values()
)
+ degraded_id_set = set(self.degraded_criterion_ids)
satisfied_run_plane_count = 0
for criterion_id in self.criterion_ids:
verdict = verdict_by_id.get(criterion_id)
@@ -133,6 +136,10 @@ class CompletionVerificationResult:
if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):
satisfied_run_plane_count += 1
continue
+ # A turn-unsatisfiable fallback criterion has no reachable satisfaction route,
+ # so an unsatisfied one can never be credited via corroboration or value-blind excusal.
+ if criterion_id in degraded_id_set:
+ return False
# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,
# so it must not veto a run whose observable outcome evidence is fully confirmed.
if verdict is not None and _is_definition_plane_abstention(verdict):
@@ -495,6 +502,8 @@ def structural_unfired_contingent_criterion_ids(
) -> list[str]:
unfired_ids: list[str] = []
for criterion in criteria:
+ if is_turn_unsatisfiable_fallback_degraded(criterion):
+ continue
path = criterion.contingent_antecedent_output_path
if not path:
continue
@@ -1266,7 +1275,10 @@ def grade_fallback_floor_reached_end_state_criteria(
eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]
if not eligible_criteria:
return []
- evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)
+ floor_degraded = any(is_turn_unsatisfiable_fallback_degraded(criterion) for criterion in eligible_criteria)
+ evidence_ref = None
+ if not floor_degraded:
+ evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)
if evidence_ref is None:
evidence_ref = _fallback_floor_carrier_evidence_ref(snapshot, carrier_verdicts)
if evidence_ref is None:
@@ -2153,6 +2165,7 @@ def combine_verification_results(
contingent_on_by_id = dict(contingent_on_by_criterion_id or {})
contingent_path_by_id = dict(contingent_antecedent_output_path_by_criterion_id or {})
structural_unfired_ids = list(structural_unfired_criterion_ids)
+ degraded_ids: list[str] = []
if run_result is not None:
contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))
contingent_on_by_id.update(run_result.contingent_on_by_criterion_id)
@@ -2160,6 +2173,7 @@ def combine_verification_results(
structural_unfired_ids = list(
dict.fromkeys([*structural_unfired_ids, *run_result.structural_unfired_criterion_ids])
)
+ degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))
if run_result is not None and run_result.status != "evaluated":
return CompletionVerificationResult(
status=run_result.status,
@@ -2169,6 +2183,7 @@ def combine_verification_results(
contingent_on_by_criterion_id=contingent_on_by_id,
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
structural_unfired_criterion_ids=structural_unfired_ids,
+ degraded_criterion_ids=degraded_ids,
)
if run_result is not None:
run_result = replace(
@@ -2197,9 +2212,47 @@ def combine_verification_results(
contingent_on_by_criterion_id=contingent_on_by_id,
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
structural_unfired_criterion_ids=structural_unfired_ids,
+ degraded_criterion_ids=degraded_ids,
)
+def only_degraded_blocking(result: CompletionVerificationResult) -> bool:
+ """True only when every criterion that still blocks full satisfaction is a
+ turn-unsatisfiable fallback criterion. A genuine unsatisfied/unknown non-degraded
+ criterion, an empty blocking set, or no degraded ids all return False, so this
+ never masks a real runtime failure or a mixed degraded+legitimate result."""
+ if result.status != "evaluated":
+ return False
+ degraded = set(result.degraded_criterion_ids)
+ if not degraded:
+ return False
+ verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
+ blocking: set[str] = set()
+ for criterion_id in result.criterion_ids:
+ verdict = verdict_by_id.get(criterion_id)
+ if verdict is not None and verdict.satisfied:
+ continue
+ if verdict is not None and _is_definition_plane_abstention(verdict):
+ continue
+ if verdict is not None and result.is_structural_contingent_abstention(verdict):
+ continue
+ blocking.add(criterion_id)
+ if not blocking:
+ return False
+ return blocking <= degraded
+
+
+def carry_degraded_criterion_ids(
+ result: CompletionVerificationResult,
+ criteria: Iterable[CompletionCriterion],
+) -> CompletionVerificationResult:
+ degraded = [criterion.id for criterion in criteria if is_turn_unsatisfiable_fallback_degraded(criterion)]
+ if not degraded and not result.degraded_criterion_ids:
+ return result
+ merged = list(dict.fromkeys([*result.degraded_criterion_ids, *degraded]))
+ return replace(result, degraded_criterion_ids=merged)
+
+
async def evaluate_completion_criteria(
criteria: list[CompletionCriterion],
snapshot: RunEvidenceSnapshot,
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index f22697499..38daae9d7 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -267,6 +267,7 @@ class CompletionCriterion:
classification_output_key: str | None = None
expected_classification: ClassificationTarget | None = None
requested_output_corroborator: bool = False
+ mint_degrade: Literal["turn_unsatisfiable_fallback"] | None = None
@dataclass
@@ -349,6 +350,8 @@ class RequestPolicy:
data[f"{prefix}_output_path"] = criterion.output_path
data[f"{prefix}_grounding_mode"] = _criterion_grounding_mode(criterion)
data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None
+ if criterion.mint_degrade is not None:
+ data[f"{prefix}_mint_degrade"] = criterion.mint_degrade
data[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_source
if criterion.expected_output_shape:
data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape
@@ -1361,7 +1364,7 @@ def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicy
def _apply_requested_output_completion_criteria(
- policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
+ policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None, *, extract_literals: bool = False
) -> None:
schema_aliases = schema_output_path_aliases_from_criteria(policy.completion_criteria)
config_aliases = _normalize_requested_output_aliases(aliases)
@@ -1391,6 +1394,9 @@ def _apply_requested_output_completion_criteria(
return
value_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)
+ if extract_literals:
+ for output_path, literal in _fallback_literal_expected_values(user_message, requested_specs).items():
+ value_by_output_path.setdefault(output_path, literal)
shape_by_output_path = _requested_output_shapes_from_criteria(policy.completion_criteria, requested_specs)
source_by_output_path = _requested_output_evidence_sources_from_criteria(
policy.completion_criteria, requested_specs
@@ -1555,6 +1561,88 @@ def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:
return criterion.id == _FALLBACK_FLOOR_BASE_ID
+def is_turn_unsatisfiable_fallback_degraded(criterion: CompletionCriterion) -> bool:
+ return criterion.mint_degrade == "turn_unsatisfiable_fallback"
+
+
+_FALLBACK_LITERAL_MIN_CHARS = 4
+_FALLBACK_LITERAL_BINDER_RE = r"(?:equal to|equals|should be|must be|is expected to be|expected to be|will be|is|:|=)"
+_FALLBACK_LITERAL_MAX_SCAN_CHARS = 4000
+
+
+def _fallback_literal_field_surface_forms(field_name: str, field_label: str) -> list[str]:
+ forms: set[str] = set()
+ for source in (field_name, field_label):
+ collapsed = " ".join(source.split()).strip()
+ if collapsed:
+ forms.add(collapsed)
+ forms.add(collapsed.replace(" ", "_"))
+ return [form for form in sorted(forms, key=len, reverse=True) if form]
+
+
+def _fallback_literal_excluded_forms(field_name: str, field_label: str, output_path: str) -> set[str]:
+ forms = {
+ " ".join(_word_tokens(field_name)),
+ " ".join(_word_tokens(field_label)),
+ " ".join(_word_tokens(output_path)),
+ }
+ return {form for form in forms if form}
+
+
+def _fallback_literal_candidates_for_field(user_message: str, field_name: str, field_label: str) -> list[str]:
+ bounded_message = user_message[:_FALLBACK_LITERAL_MAX_SCAN_CHARS]
+ candidates: list[str] = []
+ for surface in _fallback_literal_field_surface_forms(field_name, field_label):
+ pattern = re.compile(
+ r"\b"
+ + re.escape(surface)
+ + r"\b\s{0,8}+(?:"
+ + _FALLBACK_LITERAL_BINDER_RE
+ + r"\s{0,8}+)?(?P<quote>['\"`])(?P<quoted>[^'\"`]{1,80}+)(?P=quote)",
+ re.IGNORECASE,
+ )
+ for match in pattern.finditer(bounded_message):
+ quoted = match.group("quoted")
+ if quoted is not None:
+ candidates.append(quoted)
+ return candidates
+
+
+def _fallback_literal_expected_values(
+ user_message: str, requested_specs: list[tuple[str, str, str]]
+) -> dict[str, ExpectedOutputValue]:
+ if not user_message.strip():
+ return {}
+ values: dict[str, ExpectedOutputValue] = {}
+ for field_name, output_path, field_label in requested_specs:
+ excluded = _fallback_literal_excluded_forms(field_name, field_label, output_path)
+ accepted: list[str] = []
+ for raw in _fallback_literal_candidates_for_field(user_message, field_name, field_label):
+ coerced = _coerce_expected_output_value(raw)
+ if not isinstance(coerced, str) or len(coerced) < _FALLBACK_LITERAL_MIN_CHARS:
+ continue
+ if " ".join(_word_tokens(coerced)) in excluded:
+ continue
+ accepted.append(coerced)
+ unique = list(dict.fromkeys(accepted))
+ if len(unique) == 1:
+ values[output_path] = unique[0]
+ return values
+
+
+def _mark_turn_unsatisfiable_fallback_criteria(policy: RequestPolicy) -> None:
+ marked: list[CompletionCriterion] = []
+ for criterion in policy.completion_criteria:
+ value_less = criterion.expected_output_value is None and criterion.expected_output_shape is None
+ is_floor_base = is_fallback_floor_base_criterion(criterion)
+ is_requested_output = criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)
+ if value_less and (is_floor_base or is_requested_output):
+ marked.append(replace(criterion, mint_degrade="turn_unsatisfiable_fallback"))
+ else:
+ marked.append(criterion)
+ policy.completion_criteria = marked
+
+
def build_classifier_fallback_floor(ids: list[str]) -> list[CompletionCriterion]:
floor = [
CompletionCriterion(
@@ -1617,8 +1705,11 @@ def _classifier_fallback_policy(
classifier_retry_count=retry_count,
completion_contract_status="present" if fallback_criteria else "unknown",
)
- _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
+ _apply_requested_output_completion_criteria(
+ policy, user_message, requested_output_path_aliases, extract_literals=True
+ )
_apply_classifier_typed_requested_output_corroborators(policy)
+ _mark_turn_unsatisfiable_fallback_criteria(policy)
if policy.graded_completion_criteria():
policy.completion_contract_status = "present"
return policy
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 5296a3794..5a70c642e 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -19,6 +19,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
RunEvidenceSnapshot,
_contingent_metadata_for_criteria,
_is_structural_requested_output_abstention,
+ carry_degraded_criterion_ids,
combine_verification_results,
evaluate_completion_criteria,
grade_definition_criteria,
@@ -32,6 +33,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_validation_classification_criteria,
is_fallback_floor_base_criterion,
is_registered_download_completion_criterion,
+ only_degraded_blocking,
registered_download_completion_criterion,
structural_unfired_contingent_criterion_ids,
summarize_unsatisfied_outcomes,
@@ -399,6 +401,7 @@ async def _maybe_run_completion_verification_from_page_observation(
)
if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:
verification = _no_gradeable_run_plane_result(criterion_ids)
+ verification = carry_degraded_criterion_ids(verification, criteria)
copilot_ctx.completion_verification_result = verification
record_completion_verification(copilot_ctx, verification)
_record_adjudication_on_turn_state(copilot_ctx, verification)
@@ -552,6 +555,7 @@ async def _maybe_run_completion_verification_from_page_observation(
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
)
+ verification = carry_degraded_criterion_ids(verification, criteria)
if (
isinstance(existing, CompletionVerificationResult)
and not verification.is_fully_satisfied()
@@ -1156,6 +1160,13 @@ async def _maybe_run_completion_verification(
return None
criteria = _completion_verification_criteria(copilot_ctx)
criteria = _reconcile_download_completion_criterion(copilot_ctx, result, criteria)
+ verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)
+ return carry_degraded_criterion_ids(verification, criteria) if verification is not None else None
+
+
+async def _completion_verification_from_run_result(
+ copilot_ctx: Any, result: dict[str, Any], handler_start: float, criteria: list[CompletionCriterion]
+) -> CompletionVerificationResult | None:
run_criteria, definition_criteria = _split_criteria_by_plane(criteria)
criterion_ids = [criterion.id for criterion in criteria]
contingent_ids, contingent_on_by_id, contingent_path_by_id = _contingent_metadata_for_criteria(criteria)
@@ -1469,6 +1480,8 @@ def _outcome_failure_warrants_repair(
"""
if completion_verification is None:
return False
+ if only_degraded_blocking(completion_verification):
+ return False
if any(verdict.reason_code == "evidence_contradicts" for verdict in completion_verification.verdicts):
return True
# Repair needs at least one affirmatively unsatisfied criterion; unknown alone
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 9c48b68e7..280a3b3d2 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -2345,9 +2345,8 @@ def _terminal_challenge_completion_verification(
criterion_ids = list(completion_verification.criterion_ids)
if not criterion_ids:
return completion_verification
- return CompletionVerificationResult(
- status="evaluated",
- criterion_ids=criterion_ids,
+ return replace(
+ completion_verification,
verdicts=[
CriterionVerdict(
criterion_id=criterion_id,
diff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
new file mode 100644
index 000000000..b0801597d
--- /dev/null
+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
@@ -0,0 +1,471 @@
+from __future__ import annotations
+
+import time
+from dataclasses import replace
+
+from skyvern.forge.sdk.copilot.build_test_outcome import recorded_outcome_from_run_blocks_result
+from skyvern.forge.sdk.copilot.completion_criteria_store import (
+ _criterion_reconcile_key,
+ apply_requested_output_producer_floor,
+ criteria_from_json,
+ criteria_to_json,
+)
+from skyvern.forge.sdk.copilot.completion_verification import (
+ CompletionVerificationResult,
+ CriterionVerdict,
+ RunEvidenceSnapshot,
+ carry_degraded_criterion_ids,
+ combine_verification_results,
+ grade_fallback_floor_reached_end_state_criteria,
+ only_degraded_blocking,
+ structural_unfired_contingent_criterion_ids,
+)
+from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ _classifier_fallback_policy,
+ _fallback_literal_candidates_for_field,
+ build_classifier_fallback_floor,
+ is_turn_unsatisfiable_fallback_degraded,
+)
+from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+from skyvern.forge.sdk.copilot.tools.run_execution import _terminal_challenge_completion_verification
+
+_DEGRADE = "turn_unsatisfiable_fallback"
+
+
+def _fallback_policy(user_message: str):
+ return _classifier_fallback_policy(
+ [], raw_secret_present=False, failure_kind="provider_error", user_message=user_message
+ )
+
+
+def _requested_criterion(policy, suffix: str) -> CompletionCriterion | None:
+ for criterion in policy.completion_criteria:
+ if criterion.id.endswith(suffix):
+ return criterion
+ return None
+
+
+def _validation_review_payload() -> dict[str, object]:
+ return {
+ "all_checks_passed": True,
+ "validation_only": True,
+ "review_page_visible": True,
+ "submit_or_finalize_clicked": False,
+ "submitted_request": False,
+ "confirmation_page_visible": False,
+ "review_values": {
+ "visible_reference": "Reference AB123456",
+ "visible_start_date": "2026-06-22",
+ "visible_account_holder": "EXAMPLE LABS INC",
+ },
+ "evidence_text": (
+ "Visible Review page showed Reference AB123456, start date 2026-06-22, "
+ "and account holder EXAMPLE LABS INC. No final control was clicked."
+ ),
+ }
+
+
+def test_quoted_literal_mints_typed_value_and_leaves_floor_degraded() -> None:
+ policy = _fallback_policy('Go to the portal and return the order number "ABC12345" and the status')
+
+ minted = _requested_criterion(policy, "output_order_number")
+ assert minted is not None
+ assert minted.expected_output_value == "ABC12345"
+ assert minted.mint_degrade is None
+
+ floor = _requested_criterion(policy, "__copilot_fallback_floor__run")
+ assert floor is not None
+ assert floor.mint_degrade == _DEGRADE
+
+
+def test_unquoted_value_noun_degrades_instead_of_minting() -> None:
+ policy = _fallback_policy("read the record; the status should be active for each location")
+
+ minted = _requested_criterion(policy, "output_status")
+ assert minted is not None
+ assert minted.expected_output_value is None
+ assert minted.mint_degrade == _DEGRADE
+
+
+def test_extractor_mints_quoted_literal_after_binder() -> None:
+ assert set(_fallback_literal_candidates_for_field('the status should be "active"', "status", "Status")) == {
+ "active"
+ }
+
+
+def test_extractor_rejects_unquoted_participle_predicate() -> None:
+ assert _fallback_literal_candidates_for_field("the status should be updated", "status", "Status") == []
+
+
+def test_extractor_rejects_unquoted_verb_predicate() -> None:
+ assert _fallback_literal_candidates_for_field("the status should be filled", "status", "Status") == []
+
+
+def test_extractor_rejects_unquoted_code_after_binder() -> None:
+ assert _fallback_literal_candidates_for_field("order_id must be ABC12345", "order_id", "Order Id") == []
+
+
+def test_quoted_field_name_is_not_minted_as_value() -> None:
+ policy = _fallback_policy('return the "status" field and the name for the record')
+
+ minted = _requested_criterion(policy, "output_status")
+ assert minted is None or minted.expected_output_value is None
+
+
+def test_multiple_literal_candidates_mint_nothing() -> None:
+ policy = _fallback_policy("the status should be active but the status should be pending")
+
+ minted = _requested_criterion(policy, "output_status")
+ assert minted is None or minted.expected_output_value is None
+
+
+def test_short_literal_degrades_instead_of_minting() -> None:
+ policy = _fallback_policy('return the status "ok" for the record')
+
+ minted = _requested_criterion(policy, "output_status")
+ assert minted is None or minted.expected_output_value is None
+
+
+def test_value_less_fallback_marks_floor_and_requested_outputs_degraded() -> None:
+ policy = _fallback_policy("return the status for the record")
+
+ degraded = [c for c in policy.completion_criteria if is_turn_unsatisfiable_fallback_degraded(c)]
+ assert any(c.id == "__copilot_fallback_floor__run" for c in degraded)
+ assert any(c.id.endswith("output_status") for c in degraded)
+
+
+def _degraded_floor() -> list[CompletionCriterion]:
+ return [replace(criterion, mint_degrade=_DEGRADE) for criterion in build_classifier_fallback_floor([])]
+
+
+def _carrier_verdict() -> CriterionVerdict:
+ return CriterionVerdict(
+ criterion_id="__copilot_requested_output__output_reference",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:lookup",
+ evidence_source="independent_page_evidence",
+ )
+
+
+def test_degraded_floor_still_satisfies_via_carrier_arm() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={})
+ verdicts = grade_fallback_floor_reached_end_state_criteria(
+ _degraded_floor(), snapshot, carrier_verdicts=(_carrier_verdict(),)
+ )
+ assert [verdict.state for verdict in verdicts] == ["satisfied"]
+
+
+def test_degraded_floor_blocks_reached_end_state_arm() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})
+ assert grade_fallback_floor_reached_end_state_criteria(_degraded_floor(), snapshot) == []
+
+
+def test_non_degraded_floor_accepts_reached_end_state_arm() -> None:
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})
+ verdicts = grade_fallback_floor_reached_end_state_criteria(build_classifier_fallback_floor([]), snapshot)
+ assert [verdict.state for verdict in verdicts] == ["satisfied"]
+
+
+def test_structural_unfired_skips_degraded_criterion() -> None:
+ criterion = CompletionCriterion(
+ id="c_degraded",
+ outcome="captured blocker",
+ level="run",
+ contingent_antecedent_output_path="output.blocker",
+ mint_degrade=_DEGRADE,
+ )
+ snapshot = RunEvidenceSnapshot(block_outputs={"blocker_output": None})
+ assert structural_unfired_contingent_criterion_ids([criterion], snapshot) == []
+
+
+def _degraded_abstention_result() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["floor", "req"],
+ verdicts=[
+ CriterionVerdict(criterion_id="floor", state="satisfied", reason_code="evidence_confirms"),
+ CriterionVerdict(
+ criterion_id="req",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:lookup",
+ output_path="output.status",
+ ),
+ ],
+ degraded_criterion_ids=["req"],
+ )
+
+
+def test_degraded_abstention_is_not_credited_even_with_corroboration() -> None:
+ assert _degraded_abstention_result().is_fully_satisfied() is False
+
+
+def test_non_degraded_abstention_credit_path_is_not_broken() -> None:
+ result = replace(_degraded_abstention_result(), degraded_criterion_ids=[])
+ # A satisfied observed-end-state corroborator credits the structural abstention when NOT degraded.
+ result = replace(
+ result,
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="floor",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ result.verdicts[1],
+ ],
+ )
+ assert result.is_fully_satisfied() is True
+
+
+def test_sky11842_zero_evidence_non_fallback_still_fails_closed() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c1"],
+ verdicts=[CriterionVerdict(criterion_id="c1", state="unsatisfied", reason_code="no_evidence")],
+ )
+ assert result.is_fully_satisfied() is False
+ assert only_degraded_blocking(result) is False
+
+
+def test_degraded_criterion_ids_survive_combine_and_carry() -> None:
+ run_result = carry_degraded_criterion_ids(
+ CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["req"],
+ verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],
+ ),
+ [CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)],
+ )
+ assert run_result is not None
+ combined = combine_verification_results(["req"], run_result, [])
+ carried = carry_degraded_criterion_ids(
+ combined, [CompletionCriterion(id="req", outcome="x", level="run", mint_degrade=_DEGRADE)]
+ )
+ assert carried is not None
+ assert "req" in carried.degraded_criterion_ids
+
+
+def test_combine_alone_propagates_degraded_criterion_ids() -> None:
+ run_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["req"],
+ verdicts=[CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence")],
+ degraded_criterion_ids=["req"],
+ )
+ combined = combine_verification_results(["req"], run_result, [])
+ assert combined.degraded_criterion_ids == ["req"]
+ assert only_degraded_blocking(combined) is True
+
+
+def test_fallback_literal_candidates_bounded_runtime_on_adversarial_message() -> None:
+ adversarial = "status" + (" " * 20 + "=") * 4000
+ start = time.perf_counter()
+ _fallback_literal_candidates_for_field(adversarial, "status", "Status")
+ assert time.perf_counter() - start < 2.0
+
+
+def test_degraded_criterion_ids_survive_terminal_challenge() -> None:
+ criteria = [
+ CompletionCriterion(id="__copilot_fallback_floor__run", outcome="x", level="run", mint_degrade=_DEGRADE)
+ ]
+ result = carry_degraded_criterion_ids(
+ CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_fallback_floor__run"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"
+ )
+ ],
+ ),
+ criteria,
+ )
+ assert result is not None
+ assert result.degraded_criterion_ids == ["__copilot_fallback_floor__run"]
+
+ challenged = _terminal_challenge_completion_verification(result, "run did not reach a verifiable terminal state")
+ assert challenged is not None
+ assert challenged.degraded_criterion_ids == ["__copilot_fallback_floor__run"]
+ assert only_degraded_blocking(challenged) is True
+
+
+def test_only_degraded_blocking_true_for_degraded_only() -> None:
+ assert only_degraded_blocking(_degraded_abstention_result()) is True
+
+
+def test_only_degraded_blocking_false_on_empty_blocking_set() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["req"],
+ verdicts=[CriterionVerdict(criterion_id="req", state="satisfied", reason_code="evidence_confirms")],
+ degraded_criterion_ids=["req"],
+ )
+ assert only_degraded_blocking(result) is False
+
+
+def test_only_degraded_blocking_false_when_mixed_with_legitimate_unsatisfied() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["req", "legit"],
+ verdicts=[
+ CriterionVerdict(criterion_id="req", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="legit", state="unsatisfied", reason_code="evidence_contradicts"),
+ ],
+ degraded_criterion_ids=["req"],
+ )
+ assert only_degraded_blocking(result) is False
+
+
+def test_mint_degrade_survives_json_round_trip() -> None:
+ criteria = (CompletionCriterion(id="req", outcome="return the status", level="run", mint_degrade=_DEGRADE),)
+ restored = criteria_from_json(criteria_to_json(criteria))
+ assert restored[0].mint_degrade == _DEGRADE
+
+
+def test_mint_degrade_is_identity_bearing_in_reconcile_key() -> None:
+ base = CompletionCriterion(id="req", outcome="return the status", level="run", output_path="output.status")
+ degraded = replace(base, mint_degrade=_DEGRADE)
+ assert _criterion_reconcile_key(base) != _criterion_reconcile_key(degraded)
+
+
+def test_producer_floor_excludes_degraded_from_rekey() -> None:
+ degraded = CompletionCriterion(
+ id="__copilot_requested_output__output_status",
+ outcome="return the status",
+ level="run",
+ output_path="output.status",
+ mint_degrade=_DEGRADE,
+ )
+ floored, rekeyed = apply_requested_output_producer_floor([degraded])
+ assert rekeyed == ()
+ assert floored[0].output_path == "output.status"
+
+
+def _degraded_verification() -> CompletionVerificationResult:
+ return CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["__copilot_fallback_floor__run"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="__copilot_fallback_floor__run", state="unsatisfied", reason_code="no_evidence"
+ )
+ ],
+ degraded_criterion_ids=["__copilot_fallback_floor__run"],
+ )
+
+
+def test_recorded_branch_degraded_only_is_not_repairable() -> None:
+ result = {
+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
+
+
+def test_raw_branch_degraded_only_is_not_repairable() -> None:
+ result = {
+ "ok": False,
+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]},
+ }
+ outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
+
+
+def test_raw_branch_runtime_failure_stays_repairable() -> None:
+ result = {
+ "ok": False,
+ "data": {
+ "workflow_run_id": "wr_x",
+ "failure_type": "TERMINATED",
+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
+ },
+ }
+ outcome = recorded_outcome_from_run_blocks_result(result, completion_verification=_degraded_verification())
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+
+
+def test_recorded_branch_blocker_reported_stays_repairable() -> None:
+ result = {
+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="blocker_reported"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code == "blocker_reported"
+
+
+def test_recorded_branch_no_reason_code_with_failed_block_stays_repairable() -> None:
+ result = {
+ "data": {
+ "workflow_run_id": "wr_x",
+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
+ }
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+
+
+def test_recorded_branch_no_reason_code_without_failed_block_degrades() -> None:
+ result = {
+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
+
+
+def test_recorded_branch_no_meaningful_output_degrades() -> None:
+ result = {
+ "data": {"workflow_run_id": "wr_x", "blocks": [{"label": "lookup", "extracted_data": {"status": "done"}}]}
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="no_meaningful_output"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "not_authoritative"
+ assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
+
+
+def test_recorded_branch_ambiguous_code_with_failed_block_stays_repairable() -> None:
+ result = {
+ "data": {
+ "workflow_run_id": "wr_x",
+ "blocks": [{"label": "lookup", "status": "failed", "extracted_data": {"status": "done"}}],
+ }
+ }
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(verdict="not_demonstrated", reason_code="outcome_not_demonstrated"),
+ completion_verification=_degraded_verification(),
+ )
+ assert outcome is not None
+ assert outcome.verdict == "repairable_failure"
+ assert outcome.reason_code != "fallback_floor_turn_unsatisfiable"