Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-12301): stop a never-fired conditional check from failing a completed workflow (#7313)
Andrew Neilson19a732d82669
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 4f3c6925f..2bddf38b6 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -26,6 +26,7 @@ from skyvern.forge.sdk.copilot.output_utils import parse_final_responsefrom skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,TerminalActionFamily,+ is_contingent_missing_antecedent_degraded,is_fallback_floor_base_criterion,is_turn_unsatisfiable_fallback_degraded,redact_raw_secrets_for_prompt,@@ -115,6 +116,7 @@ class CompletionVerificationResult:degraded_criterion_ids: list[str] = field(default_factory=list)floor_rekeyed_criterion_ids: list[str] = field(default_factory=list)floor_rekeyed_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)+ contingent_degraded_criterion_ids: list[str] = field(default_factory=list)requested_output_criteria_count: int = 0def is_fully_satisfied(self) -> bool:@@ -128,7 +130,10 @@ class CompletionVerificationResult:_is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)for verdict in verdict_by_id.values())- degraded_id_set = set(self.degraded_criterion_ids)+ abstained_id_set = self.abstained_criterion_ids()+ degraded_id_set = degraded_lane_criterion_ids(self) - (+ set(self.contingent_degraded_criterion_ids) & abstained_id_set+ )satisfied_run_plane_count = 0for criterion_id in self.criterion_ids:verdict = verdict_by_id.get(criterion_id)@@ -136,23 +141,23 @@ class CompletionVerificationResult:# A definition-plane satisfied verdict proves the workflow is configurable,# never that a run reached the outcome, so it cannot authorize verified success# on its own — only a satisfied run-plane verdict can.- if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):+ if criterion_id not in abstained_id_set and not verdict.reason_code.startswith(+ _DEFINITION_REASON_PREFIX+ ):satisfied_run_plane_count += 1continue- # A turn-unsatisfiable fallback criterion has no reachable satisfaction route,- # so an unsatisfied one can never be credited via corroboration or value-blind excusal.+ # A degraded criterion that did not structurally abstain has no reachable satisfaction+ # route, so an unsatisfied one is never creditable via corroboration or value-blind excusal.if criterion_id in degraded_id_set:return False+ # A structurally-unfired contingent criterion is N/A for this run: it neither vetoes nor+ # earns credit, so success must derive from the remaining genuinely-satisfied criteria.+ if criterion_id in abstained_id_set:+ continue# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,# so it must not veto a run whose observable outcome evidence is fully confirmed.if verdict is not None and _is_definition_plane_abstention(verdict):continue- if verdict is not None and _is_contingent_abstention(- verdict,- self.contingent_criterion_ids,- self.structural_unfired_criterion_ids,- ):- continueif verdict is not None and verdict.self_emitted_judgment_not_independent:if _has_independent_satisfied_requested_output_corroborator(verdict_by_id, criterion_id):satisfied_run_plane_count += 1@@ -173,12 +178,15 @@ class CompletionVerificationResult:return Falsereturn satisfied_run_plane_count > 0+ def abstained_criterion_ids(self) -> set[str]:+ """A criterion abstains when its antecedent is known not to have fired: the+ ``contingent_missing_antecedent`` degrade is that determination, so its lane counts alongside+ the contingent metadata even when the criterion carries no antecedent binding."""+ lane_ids = set(self.contingent_criterion_ids) | set(self.contingent_degraded_criterion_ids)+ return set(self.structural_unfired_criterion_ids) & lane_ids+def is_structural_contingent_abstention(self, verdict: CriterionVerdict) -> bool:- return _is_contingent_abstention(- verdict,- self.contingent_criterion_ids,- self.structural_unfired_criterion_ids,- )+ return _is_contingent_abstention(verdict, self.abstained_criterion_ids())def to_trace_data(self) -> dict[str, Any]:unmet = [@@ -208,6 +216,10 @@ class CompletionVerificationResult:data["contingent_criterion_ids"] = list(self.contingent_criterion_ids)if self.structural_unfired_criterion_ids:data["structural_unfired_criterion_ids"] = list(self.structural_unfired_criterion_ids)+ if self.contingent_degraded_criterion_ids:+ data["contingent_degraded_criterion_ids"] = list(self.contingent_degraded_criterion_ids)+ contingent_id_set = set(self.contingent_criterion_ids)+ structural_unfired_id_set = set(self.structural_unfired_criterion_ids)for index, verdict in enumerate(self.verdicts[:_MAX_TRACE_VERDICTS]):prefix = f"verdict_{index}"data[f"{prefix}_criterion_id"] = verdict.criterion_id@@ -239,9 +251,8 @@ class CompletionVerificationResult:data[f"{prefix}_contingent_on"] = contingent_onif contingent_path := self.contingent_antecedent_output_path_by_criterion_id.get(verdict.criterion_id):data[f"{prefix}_contingent_antecedent_output_path"] = contingent_path- data[f"{prefix}_structural_unfired"] = verdict.criterion_id in set(- self.structural_unfired_criterion_ids- )+ if verdict.criterion_id in contingent_id_set or verdict.criterion_id in structural_unfired_id_set:+ data[f"{prefix}_structural_unfired"] = verdict.criterion_id in structural_unfired_id_setevidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)if evidence_ref:data[f"{prefix}_evidence_ref"] = evidence_ref@@ -506,6 +517,10 @@ def _find_structured_field_values(value: Any, field_names: Collection[str]) -> Iyield from _find_structured_field_values(item, field_names)+_BLOCKER_PATH_FIELDS = frozenset({"blocker", "manual_service_blocker"})+_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})++def structural_unfired_contingent_criterion_ids(criteria: Iterable[CompletionCriterion],snapshot: RunEvidenceSnapshot,@@ -514,6 +529,12 @@ def structural_unfired_contingent_criterion_ids(for criterion in criteria:if is_turn_unsatisfiable_fallback_degraded(criterion):continue+ if is_contingent_missing_antecedent_degraded(criterion):+ # The degrade value is itself the source determination that the antecedent never fired,+ # so only affirmative evidence that a real blocker fired can keep the criterion in play.+ if not (_contingent_antecedent_blocker_shaped(criterion) and _has_real_blocker_evidence(snapshot)):+ unfired_ids.append(criterion.id)+ continuepath = criterion.contingent_antecedent_output_pathif not path:continue@@ -541,10 +562,16 @@ def _is_blocker_contingent_criterion(criterion: CompletionCriterion) -> bool:path = criterion.contingent_antecedent_output_pathif path is None:return False- return _output_path_field(path) in {"blocker", "manual_service_blocker"}+ return _output_path_field(path) in _BLOCKER_PATH_FIELDS-_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})+def _contingent_antecedent_blocker_shaped(criterion: CompletionCriterion) -> bool:+ """A pathless contingent criterion anchors to blocker evidence only when its own consequent+ writes the blocker field, which makes the run's blocker record evidence about its antecedent."""+ if criterion.contingent_antecedent_output_path is not None:+ return False+ path = criterion.output_path+ return path is not None and _output_path_field(path) in _BLOCKER_PATH_FIELDSdef _blocker_family_values(snapshot: RunEvidenceSnapshot) -> list[Any]:@@ -565,6 +592,8 @@ def _is_real_blocker_evidence(value: Any) -> bool:def _has_structural_no_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:+ """A registered blocker-family key is an explicit negative record whatever its value shape+ (None/""/false/"none"); only an absent key is an absent record, which ``_blocker_family_values`` guarantees."""for value in _blocker_family_values(snapshot):if _is_structural_no_blocker_marker(value):return True@@ -1285,7 +1314,7 @@ def grade_fallback_floor_reached_end_state_criteria(eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]if not eligible_criteria:return []- floor_degraded = any(is_turn_unsatisfiable_fallback_degraded(criterion) for criterion in eligible_criteria)+ floor_degraded = any(criterion.mint_degrade is not None for criterion in eligible_criteria)evidence_ref = Noneif not floor_degraded:evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)@@ -2148,14 +2177,9 @@ def only_structural_requested_output_abstentions(result: CompletionVerificationRreturn bool(unmet) and all(_is_structural_requested_output_abstention(verdict) for verdict in unmet)-def _is_contingent_abstention(- verdict: CriterionVerdict,- contingent_criterion_ids: Iterable[str],- structural_unfired_criterion_ids: Iterable[str],-) -> bool:+def _is_contingent_abstention(verdict: CriterionVerdict, abstained_criterion_ids: Collection[str]) -> bool:return (- verdict.criterion_id in set(contingent_criterion_ids)- and verdict.criterion_id in set(structural_unfired_criterion_ids)+ verdict.criterion_id in abstained_criterion_idsand not verdict.satisfiedand verdict.reason_code in _CONTINGENT_ABSTENTION_REASON_CODES)@@ -2194,6 +2218,7 @@ def combine_verification_results(degraded_ids: list[str] = []floor_rekeyed_ids: list[str] = []floor_rekeyed_path_by_id: dict[str, str] = {}+ contingent_degraded_ids: list[str] = []if run_result is not None:contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))contingent_on_by_id.update(run_result.contingent_on_by_criterion_id)@@ -2204,6 +2229,9 @@ def combine_verification_results(degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))floor_rekeyed_ids = list(dict.fromkeys([*floor_rekeyed_ids, *run_result.floor_rekeyed_criterion_ids]))floor_rekeyed_path_by_id.update(run_result.floor_rekeyed_output_path_by_criterion_id)+ contingent_degraded_ids = list(+ dict.fromkeys([*contingent_degraded_ids, *run_result.contingent_degraded_criterion_ids])+ )if run_result is not None and run_result.status != "evaluated":return CompletionVerificationResult(status=run_result.status,@@ -2216,6 +2244,7 @@ def combine_verification_results(degraded_criterion_ids=degraded_ids,floor_rekeyed_criterion_ids=floor_rekeyed_ids,floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,+ contingent_degraded_criterion_ids=contingent_degraded_ids,requested_output_criteria_count=requested_output_criteria_count,)if run_result is not None:@@ -2248,18 +2277,23 @@ def combine_verification_results(degraded_criterion_ids=degraded_ids,floor_rekeyed_criterion_ids=floor_rekeyed_ids,floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,+ contingent_degraded_criterion_ids=contingent_degraded_ids,requested_output_criteria_count=requested_output_criteria_count,)+def degraded_lane_criterion_ids(result: CompletionVerificationResult) -> set[str]:+ return {*result.degraded_criterion_ids, *result.contingent_degraded_criterion_ids}++def only_degraded_blocking(result: CompletionVerificationResult) -> bool:- """True only when every criterion that still blocks full satisfaction is a- turn-unsatisfiable fallback criterion. A genuine unsatisfied/unknown non-degraded- criterion, an empty blocking set, or no degraded ids all return False, so this- never masks a real runtime failure or a mixed degraded+legitimate result."""+ """True only when every criterion that still blocks full satisfaction is a degraded+ criterion. A genuine unsatisfied/unknown non-degraded criterion, an empty blocking set,+ or no degraded ids all return False, so this never masks a real runtime failure or a+ mixed degraded+legitimate result."""if result.status != "evaluated":return False- degraded = set(result.degraded_criterion_ids)+ degraded = degraded_lane_criterion_ids(result)if not degraded:return Falseverdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}@@ -2321,7 +2355,7 @@ def degraded_contract_delivered_unverified_terminal_state(or terminal_blocked):return None- degraded = set(result.degraded_criterion_ids)+ degraded = degraded_lane_criterion_ids(result)if not degraded:return Noneverdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}@@ -2406,11 +2440,25 @@ def carry_degraded_criterion_ids(result: CompletionVerificationResult,criteria: Iterable[CompletionCriterion],) -> CompletionVerificationResult:+ criteria = list(criteria)degraded = [criterion.id for criterion in criteria if is_turn_unsatisfiable_fallback_degraded(criterion)]- if not degraded and not result.degraded_criterion_ids:+ contingent_degraded = [+ criterion.id for criterion in criteria if is_contingent_missing_antecedent_degraded(criterion)+ ]+ if (+ not degraded+ and not contingent_degraded+ and not result.degraded_criterion_ids+ and not result.contingent_degraded_criterion_ids+ ):return result- merged = list(dict.fromkeys([*result.degraded_criterion_ids, *degraded]))- return replace(result, degraded_criterion_ids=merged)+ return replace(+ result,+ degraded_criterion_ids=list(dict.fromkeys([*result.degraded_criterion_ids, *degraded])),+ contingent_degraded_criterion_ids=list(+ dict.fromkeys([*result.contingent_degraded_criterion_ids, *contingent_degraded])+ ),+ )def carry_floor_rekeyed_criterion_ids(diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 7fd62293b..3f7b8acad 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -1723,7 +1723,11 @@ def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:def is_turn_unsatisfiable_fallback_degraded(criterion: CompletionCriterion) -> bool:- return criterion.mint_degrade is not None+ return criterion.mint_degrade == "turn_unsatisfiable_fallback"+++def is_contingent_missing_antecedent_degraded(criterion: CompletionCriterion) -> bool:+ return criterion.mint_degrade == "contingent_missing_antecedent"def resolve_mint_degrade(diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 08a550867..592a9029f 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -42,6 +42,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (RunEvidenceSnapshot,_coerce_result,_structured_record_has_identifier,+ carry_degraded_criterion_ids,carry_floor_rekeyed_criterion_ids,combine_verification_results,degraded_contract_delivered_unverified_terminal_state,@@ -55,6 +56,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_structured_record_criteria,grade_terminal_goal_record_criteria,grade_validation_classification_criteria,+ only_degraded_blocking,registered_download_completion_criterion,run_plane_all_no_evidence,structural_unfired_contingent_criterion_ids,@@ -91,6 +93,8 @@ from skyvern.forge.sdk.copilot.request_policy import (_apply_requested_output_completion_criteria,_parse_completion_criteria,build_classifier_fallback_floor,+ is_contingent_missing_antecedent_degraded,+ is_turn_unsatisfiable_fallback_degraded,)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools import (@@ -3221,6 +3225,28 @@ def test_degraded_delivered_unverified_terminal_state_allows_observed_runtime_ouassert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]+def test_degraded_delivered_unverified_terminal_state_spans_contingent_degraded_lane() -> None:+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c_contingent_degraded", "requested_output"],+ verdicts=[+ CriterionVerdict(+ criterion_id="c_contingent_degraded",+ state="unsatisfied",+ reason_code="no_evidence",+ ),+ _observed_structural_abstention(),+ ],+ contingent_degraded_criterion_ids=["c_contingent_degraded"],+ )++ terminal_state = _delivered_terminal_state(result)++ assert result.degraded_criterion_ids == []+ assert terminal_state is not None+ assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]++@pytest.mark.parametrize("kwargs",[@@ -11901,3 +11927,347 @@ def test_bound_post_run_page_evidence_drops_only_stamp_keys() -> None:}assert "workflow_run_id" not in boundassert "observed_after_workflow_run" not in bound+++def _blocker_contingent_criterion(cid: str = "c_contingent") -> CompletionCriterion:+ return _criterion(+ cid,+ "A blocker is reported to the user.",+ contingent_on="the site only exposes an email or manual submission path",+ output_path="output.blocker",+ mint_degrade="contingent_missing_antecedent",+ )+++def _registered_output_criteria() -> list[CompletionCriterion]:+ return [_criterion(f"c{index}", f"Requested output {index} is registered.") for index in range(1, 6)]+++def _graded_result(+ criteria: list[CompletionCriterion],+ snapshot: RunEvidenceSnapshot,+ unsatisfied_ids: set[str],+ *,+ unsatisfied_state: str = "unsatisfied",+ unsatisfied_reason_code: str = "evidence_contradicts",+ unsatisfied_evidence_ref: str | None = None,+ satisfied_evidence_ref: str | None = None,+) -> CompletionVerificationResult:+ contingent_ids = [criterion.id for criterion in criteria if criterion.contingent_on]+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[criterion.id for criterion in criteria],+ contingent_criterion_ids=contingent_ids,+ contingent_on_by_criterion_id={+ criterion.id: criterion.contingent_on for criterion in criteria if criterion.contingent_on+ },+ structural_unfired_criterion_ids=structural_unfired_contingent_criterion_ids(criteria, snapshot),+ verdicts=[+ CriterionVerdict(+ criterion_id=criterion.id,+ state=unsatisfied_state if criterion.id in unsatisfied_ids else "satisfied",+ reason_code=unsatisfied_reason_code if criterion.id in unsatisfied_ids else "evidence_confirms",+ evidence_ref=unsatisfied_evidence_ref if criterion.id in unsatisfied_ids else satisfied_evidence_ref,+ )+ for criterion in criteria+ ],+ )+ return carry_degraded_criterion_ids(result, criteria)+++_NON_ABSTAINING_CONTINGENT_CRITERIA: dict[str, tuple[CompletionCriterion, RunEvidenceSnapshot]] = {+ "real_blocker": (+ _blocker_contingent_criterion(),+ RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "Provider requires a phone call."}}),+ ),+ "real_blocker_alias_key": (+ _blocker_contingent_criterion(),+ RunEvidenceSnapshot(block_outputs={"blocker_output": "Provider requires a phone call."}),+ ),+}++[email protected]("arm", sorted(_NON_ABSTAINING_CONTINGENT_CRITERIA))+ "unsatisfied_state, unsatisfied_reason_code",+ [("unsatisfied", "structurally_abstained"), ("unknown", "definition_unknown")],+)+def test_non_abstaining_contingent_criterion_vetoes_via_excusable_verdict_shapes(+ arm: str, unsatisfied_state: str, unsatisfied_reason_code: str+) -> None:+ contingent, snapshot = _NON_ABSTAINING_CONTINGENT_CRITERIA[arm]+ criteria = [*_registered_output_criteria(), contingent]++ result = _graded_result(+ criteria,+ snapshot,+ {contingent.id},+ unsatisfied_state=unsatisfied_state,+ unsatisfied_reason_code=unsatisfied_reason_code,+ )++ assert result.structural_unfired_criterion_ids == []+ assert result.contingent_degraded_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is False++[email protected]("arm", sorted(_NON_ABSTAINING_CONTINGENT_CRITERIA))+def test_non_abstaining_contingent_criterion_vetoes_via_reperception_contradiction(arm: str) -> None:+ contingent, snapshot = _NON_ABSTAINING_CONTINGENT_CRITERIA[arm]+ criteria = [*_registered_output_criteria(), contingent]++ result = _graded_result(+ criteria,+ snapshot,+ {contingent.id},+ unsatisfied_evidence_ref="scout_synthesized_browser_steps_output",+ satisfied_evidence_ref="observed_end_state_url",+ )++ assert result.structural_unfired_criterion_ids == []+ assert result.is_fully_satisfied() is False+++def test_abstained_pathless_contingent_criterion_does_not_veto_via_abstention_reason_shapes() -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result(criteria, snapshot, {contingent.id}, unsatisfied_reason_code="no_evidence")++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is True+++def test_fallback_floor_base_criterion_degraded_as_contingent_stays_strict() -> None:+ floor = [+ replace(criterion, mint_degrade="contingent_missing_antecedent")+ for criterion in build_classifier_fallback_floor([])+ ]+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})++ verdicts = grade_fallback_floor_reached_end_state_criteria(floor, snapshot)++ assert [verdict.state for verdict in verdicts] != ["satisfied"]+++def test_pathless_blocker_contingent_criterion_abstains_and_does_not_veto() -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "", "submitted": True}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert is_contingent_missing_antecedent_degraded(contingent) is True+ assert is_turn_unsatisfiable_fallback_degraded(contingent) is False+ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.contingent_degraded_criterion_ids == [contingent.id]+ assert contingent.id not in result.degraded_criterion_ids+ assert result.is_fully_satisfied() is True+ assert contingent.id not in result.to_trace_data()["unmet_criterion_ids"]+++def test_abstained_pathless_contingent_criterion_earns_zero_credit() -> None:+ contingent = _blocker_contingent_criterion()+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result([contingent], snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is False+++def test_pathless_contingent_abstention_emits_structural_unfired_trace_record() -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ trace = _graded_result(criteria, snapshot, {contingent.id}).to_trace_data()++ assert trace["structural_unfired_criterion_ids"] == [contingent.id]+ assert trace["contingent_degraded_criterion_ids"] == [contingent.id]+ unfired_keys = {+ key: value+ for key, value in trace.items()+ if key.endswith("_structural_unfired") and not key.startswith("structural_unfired")+ }+ assert unfired_keys == {"verdict_5_structural_unfired": True}+++def test_real_blocker_evidence_keeps_pathless_contingent_criterion_blocking() -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "Provider requires a phone call."}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == []+ assert result.is_fully_satisfied() is False+++def test_missing_blocker_family_output_abstains_pathless_contingent_criterion() -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"submitted": True}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.contingent_degraded_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is True++[email protected]("registered_blocker", ["", None, False, "none", "null", "false"])+def test_registered_no_blocker_value_shapes_abstain_regardless_of_value(registered_blocker: str | bool | None) -> None:+ contingent = _blocker_contingent_criterion()+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": registered_blocker}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is True+++def test_non_blocker_shaped_pathless_contingent_criterion_abstains() -> None:+ contingent = _criterion(+ "c_contingent",+ "The out-of-stock notice is reported to the user.",+ contingent_on="the item is out of stock",+ output_path="output.stock_notice",+ mint_degrade="contingent_missing_antecedent",+ )+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.contingent_degraded_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is True+++def test_pathless_contingent_criterion_without_blocker_key_abstains_without_credit() -> None:+ contingent = _criterion(+ "c_contingent",+ "The earliest available start date is reported to the user.",+ contingent_on="the provider offers a scheduling window",+ output_path="output.desired_start_date",+ mint_degrade="contingent_missing_antecedent",+ )+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"submitted": True}})++ result = _graded_result(criteria, snapshot, {contingent.id})+ solo_result = _graded_result([contingent], snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.contingent_degraded_criterion_ids == [contingent.id]+ assert result.is_fully_satisfied() is True+ assert contingent.id not in result.to_trace_data()["unmet_criterion_ids"]+ assert solo_result.is_fully_satisfied() is False+++def test_pathed_blocker_contingent_criterion_still_abstains_on_empty_blocker() -> None:+ contingent = _criterion(+ "c_contingent",+ "A blocker is reported to the user.",+ contingent_on="the site blocks online submission",+ contingent_antecedent_output_path="output.blocker",+ )+ criteria = [*_registered_output_criteria(), contingent]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result(criteria, snapshot, {contingent.id})++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.contingent_degraded_criterion_ids == []+ assert result.is_fully_satisfied() is True+++def test_turn_unsatisfiable_fallback_still_vetoes_and_stays_in_blocking_lane() -> None:+ fallback = _criterion(+ "c_fallback",+ "The requested turn outcome is reached.",+ mint_degrade="turn_unsatisfiable_fallback",+ )+ criteria = [*_registered_output_criteria(), fallback]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result(criteria, snapshot, {fallback.id})++ assert is_turn_unsatisfiable_fallback_degraded(fallback) is True+ assert is_contingent_missing_antecedent_degraded(fallback) is False+ assert result.structural_unfired_criterion_ids == []+ assert result.degraded_criterion_ids == [fallback.id]+ assert result.contingent_degraded_criterion_ids == []+ assert result.is_fully_satisfied() is False+ assert only_degraded_blocking(result) is True+++def test_abstained_criterion_with_satisfied_verdict_is_not_a_credit_authorizer() -> None:+ contingent = _blocker_contingent_criterion()+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result([contingent], snapshot, set())++ assert result.structural_unfired_criterion_ids == [contingent.id]+ assert result.verdicts[0].satisfied is True+ assert result.is_fully_satisfied() is False+++def test_abstained_criterion_earns_no_credit_via_corroborated_requested_output_door() -> None:+ contingent = _blocker_contingent_criterion()+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result([contingent], snapshot, {contingent.id})+ result = replace(+ result,+ verdicts=[+ replace(+ result.verdicts[0],+ reason_code="structurally_abstained",+ evidence_ref="extract_result",+ output_path="output.blocker",+ ),+ CriterionVerdict(+ criterion_id="c_terminal_record",+ state="satisfied",+ reason_code="evidence_confirms",+ grounding_mode="terminal_record",+ ),+ ],+ )++ assert result.is_fully_satisfied() is False+++def test_abstained_criterion_earns_no_credit_via_self_emitted_corroborator_door() -> None:+ contingent = _blocker_contingent_criterion()+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result([contingent], snapshot, {contingent.id})+ result = replace(+ result,+ verdicts=[replace(result.verdicts[0], self_emitted_judgment_not_independent=True)],+ )++ assert result.is_fully_satisfied() is False+++def test_fallback_degraded_criterion_in_structural_unfired_set_still_vetoes() -> None:+ fallback = _criterion(+ "c_fallback",+ "The requested turn outcome is reached.",+ contingent_on="the site only exposes an email or manual submission path",+ mint_degrade="turn_unsatisfiable_fallback",+ )+ criteria = [*_registered_output_criteria(), fallback]+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})++ result = _graded_result(criteria, snapshot, {fallback.id})+ result = replace(result, structural_unfired_criterion_ids=[fallback.id])++ assert result.degraded_criterion_ids == [fallback.id]+ assert result.contingent_degraded_criterion_ids == []+ assert result.is_fully_satisfied() is Falsediff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.pyindex 7690fd520..52e5eef72 100644--- a/tests/unit/test_copilot_sky11865_mint_satisfiability.py+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py@@ -25,7 +25,9 @@ from skyvern.forge.sdk.copilot.request_policy import (_classifier_fallback_policy,_fallback_literal_candidates_for_field,build_classifier_fallback_floor,+ is_contingent_missing_antecedent_degraded,is_turn_unsatisfiable_fallback_degraded,+ resolve_mint_degrade,)from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcomefrom skyvern.forge.sdk.copilot.tools.run_execution import _terminal_challenge_completion_verification@@ -454,7 +456,7 @@ def test_recorded_branch_no_meaningful_output_degrades() -> None:assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"-def test_contingent_missing_antecedent_counts_as_degraded() -> None:+def test_contingent_missing_antecedent_routes_to_its_own_degrade_lane() -> None:criterion = CompletionCriterion(id="c1",outcome="A blocker is reported to the user.",@@ -462,8 +464,9 @@ def test_contingent_missing_antecedent_counts_as_degraded() -> None:contingent_on="the site blocks submission",mint_degrade="contingent_missing_antecedent",)- assert is_turn_unsatisfiable_fallback_degraded(criterion) is True- assert is_turn_unsatisfiable_fallback_degraded(replace(criterion, mint_degrade=None)) is False+ assert is_contingent_missing_antecedent_degraded(criterion) is True+ assert is_turn_unsatisfiable_fallback_degraded(criterion) is False+ assert is_contingent_missing_antecedent_degraded(replace(criterion, mint_degrade=None)) is Falsedef test_contingent_missing_antecedent_carries_and_blocks_credit_when_unsatisfied() -> None:@@ -486,8 +489,10 @@ def test_contingent_missing_antecedent_carries_and_blocks_credit_when_unsatisfie)],)- assert "c1" in result.degraded_criterion_ids+ assert "c1" in result.contingent_degraded_criterion_ids+ assert "c1" not in result.degraded_criterion_idsassert result.is_fully_satisfied() is False+ assert only_degraded_blocking(result) is Truedef test_satisfied_degraded_criterion_still_credits_on_merits() -> None:@@ -515,3 +520,12 @@ def test_recorded_branch_ambiguous_code_with_failed_block_stays_repairable() ->assert outcome is not Noneassert outcome.verdict == "repairable_failure"assert outcome.reason_code != "fallback_floor_turn_unsatisfiable"+++def test_turn_unsatisfiable_fallback_stamp_precedes_pathless_contingent_degrade() -> None:+ resolved = resolve_mint_degrade(+ "turn_unsatisfiable_fallback",+ "the site blocks submission",+ None,+ )+ assert resolved == "turn_unsatisfiable_fallback"