Skip to content

Automate browser based workflows with AI

fix(SKY-12301): stop a never-fired conditional check from failing a completed workflow (#7313)

Andrew Neilson19a732d82669
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 4f3c6925f..2bddf38b6 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.copilot.output_utils import parse_final_response
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
TerminalActionFamily,
+ is_contingent_missing_antecedent_degraded,
is_fallback_floor_base_criterion,
is_turn_unsatisfiable_fallback_degraded,
redact_raw_secrets_for_prompt,
@@ -115,6 +116,7 @@ class CompletionVerificationResult:
degraded_criterion_ids: list[str] = field(default_factory=list)
floor_rekeyed_criterion_ids: list[str] = field(default_factory=list)
floor_rekeyed_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)
+ contingent_degraded_criterion_ids: list[str] = field(default_factory=list)
requested_output_criteria_count: int = 0
def is_fully_satisfied(self) -> bool:
@@ -128,7 +130,10 @@ class CompletionVerificationResult:
_is_satisfied_observed_end_state_verdict(verdict) or _is_satisfied_terminal_record_verdict(verdict)
for verdict in verdict_by_id.values()
)
- degraded_id_set = set(self.degraded_criterion_ids)
+ abstained_id_set = self.abstained_criterion_ids()
+ degraded_id_set = degraded_lane_criterion_ids(self) - (
+ set(self.contingent_degraded_criterion_ids) & abstained_id_set
+ )
satisfied_run_plane_count = 0
for criterion_id in self.criterion_ids:
verdict = verdict_by_id.get(criterion_id)
@@ -136,23 +141,23 @@ class CompletionVerificationResult:
# A definition-plane satisfied verdict proves the workflow is configurable,
# never that a run reached the outcome, so it cannot authorize verified success
# on its own — only a satisfied run-plane verdict can.
- if not verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX):
+ if criterion_id not in abstained_id_set and not verdict.reason_code.startswith(
+ _DEFINITION_REASON_PREFIX
+ ):
satisfied_run_plane_count += 1
continue
- # A turn-unsatisfiable fallback criterion has no reachable satisfaction route,
- # so an unsatisfied one can never be credited via corroboration or value-blind excusal.
+ # A degraded criterion that did not structurally abstain has no reachable satisfaction
+ # route, so an unsatisfied one is never creditable via corroboration or value-blind excusal.
if criterion_id in degraded_id_set:
return False
+ # A structurally-unfired contingent criterion is N/A for this run: it neither vetoes nor
+ # earns credit, so success must derive from the remaining genuinely-satisfied criteria.
+ if criterion_id in abstained_id_set:
+ continue
# A definition-plane ``unknown`` is a YAML-grader abstention, not a refutation,
# so it must not veto a run whose observable outcome evidence is fully confirmed.
if verdict is not None and _is_definition_plane_abstention(verdict):
continue
- if verdict is not None and _is_contingent_abstention(
- verdict,
- self.contingent_criterion_ids,
- self.structural_unfired_criterion_ids,
- ):
- continue
if verdict is not None and verdict.self_emitted_judgment_not_independent:
if _has_independent_satisfied_requested_output_corroborator(verdict_by_id, criterion_id):
satisfied_run_plane_count += 1
@@ -173,12 +178,15 @@ class CompletionVerificationResult:
return False
return satisfied_run_plane_count > 0
+ def abstained_criterion_ids(self) -> set[str]:
+ """A criterion abstains when its antecedent is known not to have fired: the
+ ``contingent_missing_antecedent`` degrade is that determination, so its lane counts alongside
+ the contingent metadata even when the criterion carries no antecedent binding."""
+ lane_ids = set(self.contingent_criterion_ids) | set(self.contingent_degraded_criterion_ids)
+ return set(self.structural_unfired_criterion_ids) & lane_ids
+
def is_structural_contingent_abstention(self, verdict: CriterionVerdict) -> bool:
- return _is_contingent_abstention(
- verdict,
- self.contingent_criterion_ids,
- self.structural_unfired_criterion_ids,
- )
+ return _is_contingent_abstention(verdict, self.abstained_criterion_ids())
def to_trace_data(self) -> dict[str, Any]:
unmet = [
@@ -208,6 +216,10 @@ class CompletionVerificationResult:
data["contingent_criterion_ids"] = list(self.contingent_criterion_ids)
if self.structural_unfired_criterion_ids:
data["structural_unfired_criterion_ids"] = list(self.structural_unfired_criterion_ids)
+ if self.contingent_degraded_criterion_ids:
+ data["contingent_degraded_criterion_ids"] = list(self.contingent_degraded_criterion_ids)
+ contingent_id_set = set(self.contingent_criterion_ids)
+ structural_unfired_id_set = set(self.structural_unfired_criterion_ids)
for index, verdict in enumerate(self.verdicts[:_MAX_TRACE_VERDICTS]):
prefix = f"verdict_{index}"
data[f"{prefix}_criterion_id"] = verdict.criterion_id
@@ -239,9 +251,8 @@ class CompletionVerificationResult:
data[f"{prefix}_contingent_on"] = contingent_on
if contingent_path := self.contingent_antecedent_output_path_by_criterion_id.get(verdict.criterion_id):
data[f"{prefix}_contingent_antecedent_output_path"] = contingent_path
- data[f"{prefix}_structural_unfired"] = verdict.criterion_id in set(
- self.structural_unfired_criterion_ids
- )
+ if verdict.criterion_id in contingent_id_set or verdict.criterion_id in structural_unfired_id_set:
+ data[f"{prefix}_structural_unfired"] = verdict.criterion_id in structural_unfired_id_set
evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)
if evidence_ref:
data[f"{prefix}_evidence_ref"] = evidence_ref
@@ -506,6 +517,10 @@ def _find_structured_field_values(value: Any, field_names: Collection[str]) -> I
yield from _find_structured_field_values(item, field_names)
+_BLOCKER_PATH_FIELDS = frozenset({"blocker", "manual_service_blocker"})
+_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})
+
+
def structural_unfired_contingent_criterion_ids(
criteria: Iterable[CompletionCriterion],
snapshot: RunEvidenceSnapshot,
@@ -514,6 +529,12 @@ def structural_unfired_contingent_criterion_ids(
for criterion in criteria:
if is_turn_unsatisfiable_fallback_degraded(criterion):
continue
+ if is_contingent_missing_antecedent_degraded(criterion):
+ # The degrade value is itself the source determination that the antecedent never fired,
+ # so only affirmative evidence that a real blocker fired can keep the criterion in play.
+ if not (_contingent_antecedent_blocker_shaped(criterion) and _has_real_blocker_evidence(snapshot)):
+ unfired_ids.append(criterion.id)
+ continue
path = criterion.contingent_antecedent_output_path
if not path:
continue
@@ -541,10 +562,16 @@ def _is_blocker_contingent_criterion(criterion: CompletionCriterion) -> bool:
path = criterion.contingent_antecedent_output_path
if path is None:
return False
- return _output_path_field(path) in {"blocker", "manual_service_blocker"}
+ return _output_path_field(path) in _BLOCKER_PATH_FIELDS
-_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})
+def _contingent_antecedent_blocker_shaped(criterion: CompletionCriterion) -> bool:
+ """A pathless contingent criterion anchors to blocker evidence only when its own consequent
+ writes the blocker field, which makes the run's blocker record evidence about its antecedent."""
+ if criterion.contingent_antecedent_output_path is not None:
+ return False
+ path = criterion.output_path
+ return path is not None and _output_path_field(path) in _BLOCKER_PATH_FIELDS
def _blocker_family_values(snapshot: RunEvidenceSnapshot) -> list[Any]:
@@ -565,6 +592,8 @@ def _is_real_blocker_evidence(value: Any) -> bool:
def _has_structural_no_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:
+ """A registered blocker-family key is an explicit negative record whatever its value shape
+ (None/""/false/"none"); only an absent key is an absent record, which ``_blocker_family_values`` guarantees."""
for value in _blocker_family_values(snapshot):
if _is_structural_no_blocker_marker(value):
return True
@@ -1285,7 +1314,7 @@ def grade_fallback_floor_reached_end_state_criteria(
eligible_criteria = [criterion for criterion in criteria if is_fallback_floor_base_criterion(criterion)]
if not eligible_criteria:
return []
- floor_degraded = any(is_turn_unsatisfiable_fallback_degraded(criterion) for criterion in eligible_criteria)
+ floor_degraded = any(criterion.mint_degrade is not None for criterion in eligible_criteria)
evidence_ref = None
if not floor_degraded:
evidence_ref = _fallback_floor_reached_end_state_evidence_ref(snapshot, criteria)
@@ -2148,14 +2177,9 @@ def only_structural_requested_output_abstentions(result: CompletionVerificationR
return bool(unmet) and all(_is_structural_requested_output_abstention(verdict) for verdict in unmet)
-def _is_contingent_abstention(
- verdict: CriterionVerdict,
- contingent_criterion_ids: Iterable[str],
- structural_unfired_criterion_ids: Iterable[str],
-) -> bool:
+def _is_contingent_abstention(verdict: CriterionVerdict, abstained_criterion_ids: Collection[str]) -> bool:
return (
- verdict.criterion_id in set(contingent_criterion_ids)
- and verdict.criterion_id in set(structural_unfired_criterion_ids)
+ verdict.criterion_id in abstained_criterion_ids
and not verdict.satisfied
and verdict.reason_code in _CONTINGENT_ABSTENTION_REASON_CODES
)
@@ -2194,6 +2218,7 @@ def combine_verification_results(
degraded_ids: list[str] = []
floor_rekeyed_ids: list[str] = []
floor_rekeyed_path_by_id: dict[str, str] = {}
+ contingent_degraded_ids: list[str] = []
if run_result is not None:
contingent_ids = list(dict.fromkeys([*contingent_ids, *run_result.contingent_criterion_ids]))
contingent_on_by_id.update(run_result.contingent_on_by_criterion_id)
@@ -2204,6 +2229,9 @@ def combine_verification_results(
degraded_ids = list(dict.fromkeys([*degraded_ids, *run_result.degraded_criterion_ids]))
floor_rekeyed_ids = list(dict.fromkeys([*floor_rekeyed_ids, *run_result.floor_rekeyed_criterion_ids]))
floor_rekeyed_path_by_id.update(run_result.floor_rekeyed_output_path_by_criterion_id)
+ contingent_degraded_ids = list(
+ dict.fromkeys([*contingent_degraded_ids, *run_result.contingent_degraded_criterion_ids])
+ )
if run_result is not None and run_result.status != "evaluated":
return CompletionVerificationResult(
status=run_result.status,
@@ -2216,6 +2244,7 @@ def combine_verification_results(
degraded_criterion_ids=degraded_ids,
floor_rekeyed_criterion_ids=floor_rekeyed_ids,
floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,
+ contingent_degraded_criterion_ids=contingent_degraded_ids,
requested_output_criteria_count=requested_output_criteria_count,
)
if run_result is not None:
@@ -2248,18 +2277,23 @@ def combine_verification_results(
degraded_criterion_ids=degraded_ids,
floor_rekeyed_criterion_ids=floor_rekeyed_ids,
floor_rekeyed_output_path_by_criterion_id=floor_rekeyed_path_by_id,
+ contingent_degraded_criterion_ids=contingent_degraded_ids,
requested_output_criteria_count=requested_output_criteria_count,
)
+def degraded_lane_criterion_ids(result: CompletionVerificationResult) -> set[str]:
+ return {*result.degraded_criterion_ids, *result.contingent_degraded_criterion_ids}
+
+
def only_degraded_blocking(result: CompletionVerificationResult) -> bool:
- """True only when every criterion that still blocks full satisfaction is a
- turn-unsatisfiable fallback criterion. A genuine unsatisfied/unknown non-degraded
- criterion, an empty blocking set, or no degraded ids all return False, so this
- never masks a real runtime failure or a mixed degraded+legitimate result."""
+ """True only when every criterion that still blocks full satisfaction is a degraded
+ criterion. A genuine unsatisfied/unknown non-degraded criterion, an empty blocking set,
+ or no degraded ids all return False, so this never masks a real runtime failure or a
+ mixed degraded+legitimate result."""
if result.status != "evaluated":
return False
- degraded = set(result.degraded_criterion_ids)
+ degraded = degraded_lane_criterion_ids(result)
if not degraded:
return False
verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
@@ -2321,7 +2355,7 @@ def degraded_contract_delivered_unverified_terminal_state(
or terminal_blocked
):
return None
- degraded = set(result.degraded_criterion_ids)
+ degraded = degraded_lane_criterion_ids(result)
if not degraded:
return None
verdict_by_id = {verdict.criterion_id: verdict for verdict in result.verdicts}
@@ -2406,11 +2440,25 @@ def carry_degraded_criterion_ids(
result: CompletionVerificationResult,
criteria: Iterable[CompletionCriterion],
) -> CompletionVerificationResult:
+ criteria = list(criteria)
degraded = [criterion.id for criterion in criteria if is_turn_unsatisfiable_fallback_degraded(criterion)]
- if not degraded and not result.degraded_criterion_ids:
+ contingent_degraded = [
+ criterion.id for criterion in criteria if is_contingent_missing_antecedent_degraded(criterion)
+ ]
+ if (
+ not degraded
+ and not contingent_degraded
+ and not result.degraded_criterion_ids
+ and not result.contingent_degraded_criterion_ids
+ ):
return result
- merged = list(dict.fromkeys([*result.degraded_criterion_ids, *degraded]))
- return replace(result, degraded_criterion_ids=merged)
+ return replace(
+ result,
+ degraded_criterion_ids=list(dict.fromkeys([*result.degraded_criterion_ids, *degraded])),
+ contingent_degraded_criterion_ids=list(
+ dict.fromkeys([*result.contingent_degraded_criterion_ids, *contingent_degraded])
+ ),
+ )
def carry_floor_rekeyed_criterion_ids(
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 7fd62293b..3f7b8acad 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -1723,7 +1723,11 @@ def is_fallback_floor_base_criterion(criterion: CompletionCriterion) -> bool:
def is_turn_unsatisfiable_fallback_degraded(criterion: CompletionCriterion) -> bool:
- return criterion.mint_degrade is not None
+ return criterion.mint_degrade == "turn_unsatisfiable_fallback"
+
+
+def is_contingent_missing_antecedent_degraded(criterion: CompletionCriterion) -> bool:
+ return criterion.mint_degrade == "contingent_missing_antecedent"
def resolve_mint_degrade(
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 08a550867..592a9029f 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -42,6 +42,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
RunEvidenceSnapshot,
_coerce_result,
_structured_record_has_identifier,
+ carry_degraded_criterion_ids,
carry_floor_rekeyed_criterion_ids,
combine_verification_results,
degraded_contract_delivered_unverified_terminal_state,
@@ -55,6 +56,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_structured_record_criteria,
grade_terminal_goal_record_criteria,
grade_validation_classification_criteria,
+ only_degraded_blocking,
registered_download_completion_criterion,
run_plane_all_no_evidence,
structural_unfired_contingent_criterion_ids,
@@ -91,6 +93,8 @@ from skyvern.forge.sdk.copilot.request_policy import (
_apply_requested_output_completion_criteria,
_parse_completion_criteria,
build_classifier_fallback_floor,
+ is_contingent_missing_antecedent_degraded,
+ is_turn_unsatisfiable_fallback_degraded,
)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import (
@@ -3221,6 +3225,28 @@ def test_degraded_delivered_unverified_terminal_state_allows_observed_runtime_ou
assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]
+def test_degraded_delivered_unverified_terminal_state_spans_contingent_degraded_lane() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_contingent_degraded", "requested_output"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_contingent_degraded",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ ),
+ _observed_structural_abstention(),
+ ],
+ contingent_degraded_criterion_ids=["c_contingent_degraded"],
+ )
+
+ terminal_state = _delivered_terminal_state(result)
+
+ assert result.degraded_criterion_ids == []
+ assert terminal_state is not None
+ assert [verdict.criterion_id for verdict in terminal_state.observed_verdicts] == ["requested_output"]
+
+
@pytest.mark.parametrize(
"kwargs",
[
@@ -11901,3 +11927,347 @@ def test_bound_post_run_page_evidence_drops_only_stamp_keys() -> None:
}
assert "workflow_run_id" not in bound
assert "observed_after_workflow_run" not in bound
+
+
+def _blocker_contingent_criterion(cid: str = "c_contingent") -> CompletionCriterion:
+ return _criterion(
+ cid,
+ "A blocker is reported to the user.",
+ contingent_on="the site only exposes an email or manual submission path",
+ output_path="output.blocker",
+ mint_degrade="contingent_missing_antecedent",
+ )
+
+
+def _registered_output_criteria() -> list[CompletionCriterion]:
+ return [_criterion(f"c{index}", f"Requested output {index} is registered.") for index in range(1, 6)]
+
+
+def _graded_result(
+ criteria: list[CompletionCriterion],
+ snapshot: RunEvidenceSnapshot,
+ unsatisfied_ids: set[str],
+ *,
+ unsatisfied_state: str = "unsatisfied",
+ unsatisfied_reason_code: str = "evidence_contradicts",
+ unsatisfied_evidence_ref: str | None = None,
+ satisfied_evidence_ref: str | None = None,
+) -> CompletionVerificationResult:
+ contingent_ids = [criterion.id for criterion in criteria if criterion.contingent_on]
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ contingent_criterion_ids=contingent_ids,
+ contingent_on_by_criterion_id={
+ criterion.id: criterion.contingent_on for criterion in criteria if criterion.contingent_on
+ },
+ structural_unfired_criterion_ids=structural_unfired_contingent_criterion_ids(criteria, snapshot),
+ verdicts=[
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state=unsatisfied_state if criterion.id in unsatisfied_ids else "satisfied",
+ reason_code=unsatisfied_reason_code if criterion.id in unsatisfied_ids else "evidence_confirms",
+ evidence_ref=unsatisfied_evidence_ref if criterion.id in unsatisfied_ids else satisfied_evidence_ref,
+ )
+ for criterion in criteria
+ ],
+ )
+ return carry_degraded_criterion_ids(result, criteria)
+
+
+_NON_ABSTAINING_CONTINGENT_CRITERIA: dict[str, tuple[CompletionCriterion, RunEvidenceSnapshot]] = {
+ "real_blocker": (
+ _blocker_contingent_criterion(),
+ RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "Provider requires a phone call."}}),
+ ),
+ "real_blocker_alias_key": (
+ _blocker_contingent_criterion(),
+ RunEvidenceSnapshot(block_outputs={"blocker_output": "Provider requires a phone call."}),
+ ),
+}
+
+
[email protected]("arm", sorted(_NON_ABSTAINING_CONTINGENT_CRITERIA))
+ "unsatisfied_state, unsatisfied_reason_code",
+ [("unsatisfied", "structurally_abstained"), ("unknown", "definition_unknown")],
+)
+def test_non_abstaining_contingent_criterion_vetoes_via_excusable_verdict_shapes(
+ arm: str, unsatisfied_state: str, unsatisfied_reason_code: str
+) -> None:
+ contingent, snapshot = _NON_ABSTAINING_CONTINGENT_CRITERIA[arm]
+ criteria = [*_registered_output_criteria(), contingent]
+
+ result = _graded_result(
+ criteria,
+ snapshot,
+ {contingent.id},
+ unsatisfied_state=unsatisfied_state,
+ unsatisfied_reason_code=unsatisfied_reason_code,
+ )
+
+ assert result.structural_unfired_criterion_ids == []
+ assert result.contingent_degraded_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is False
+
+
[email protected]("arm", sorted(_NON_ABSTAINING_CONTINGENT_CRITERIA))
+def test_non_abstaining_contingent_criterion_vetoes_via_reperception_contradiction(arm: str) -> None:
+ contingent, snapshot = _NON_ABSTAINING_CONTINGENT_CRITERIA[arm]
+ criteria = [*_registered_output_criteria(), contingent]
+
+ result = _graded_result(
+ criteria,
+ snapshot,
+ {contingent.id},
+ unsatisfied_evidence_ref="scout_synthesized_browser_steps_output",
+ satisfied_evidence_ref="observed_end_state_url",
+ )
+
+ assert result.structural_unfired_criterion_ids == []
+ assert result.is_fully_satisfied() is False
+
+
+def test_abstained_pathless_contingent_criterion_does_not_veto_via_abstention_reason_shapes() -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id}, unsatisfied_reason_code="no_evidence")
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is True
+
+
+def test_fallback_floor_base_criterion_degraded_as_contingent_stays_strict() -> None:
+ floor = [
+ replace(criterion, mint_degrade="contingent_missing_antecedent")
+ for criterion in build_classifier_fallback_floor([])
+ ]
+ snapshot = RunEvidenceSnapshot(block_outputs={"submit_request": _validation_review_payload()})
+
+ verdicts = grade_fallback_floor_reached_end_state_criteria(floor, snapshot)
+
+ assert [verdict.state for verdict in verdicts] != ["satisfied"]
+
+
+def test_pathless_blocker_contingent_criterion_abstains_and_does_not_veto() -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "", "submitted": True}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert is_contingent_missing_antecedent_degraded(contingent) is True
+ assert is_turn_unsatisfiable_fallback_degraded(contingent) is False
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.contingent_degraded_criterion_ids == [contingent.id]
+ assert contingent.id not in result.degraded_criterion_ids
+ assert result.is_fully_satisfied() is True
+ assert contingent.id not in result.to_trace_data()["unmet_criterion_ids"]
+
+
+def test_abstained_pathless_contingent_criterion_earns_zero_credit() -> None:
+ contingent = _blocker_contingent_criterion()
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result([contingent], snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is False
+
+
+def test_pathless_contingent_abstention_emits_structural_unfired_trace_record() -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ trace = _graded_result(criteria, snapshot, {contingent.id}).to_trace_data()
+
+ assert trace["structural_unfired_criterion_ids"] == [contingent.id]
+ assert trace["contingent_degraded_criterion_ids"] == [contingent.id]
+ unfired_keys = {
+ key: value
+ for key, value in trace.items()
+ if key.endswith("_structural_unfired") and not key.startswith("structural_unfired")
+ }
+ assert unfired_keys == {"verdict_5_structural_unfired": True}
+
+
+def test_real_blocker_evidence_keeps_pathless_contingent_criterion_blocking() -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": "Provider requires a phone call."}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == []
+ assert result.is_fully_satisfied() is False
+
+
+def test_missing_blocker_family_output_abstains_pathless_contingent_criterion() -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"submitted": True}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.contingent_degraded_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is True
+
+
[email protected]("registered_blocker", ["", None, False, "none", "null", "false"])
+def test_registered_no_blocker_value_shapes_abstain_regardless_of_value(registered_blocker: str | bool | None) -> None:
+ contingent = _blocker_contingent_criterion()
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": registered_blocker}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is True
+
+
+def test_non_blocker_shaped_pathless_contingent_criterion_abstains() -> None:
+ contingent = _criterion(
+ "c_contingent",
+ "The out-of-stock notice is reported to the user.",
+ contingent_on="the item is out of stock",
+ output_path="output.stock_notice",
+ mint_degrade="contingent_missing_antecedent",
+ )
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.contingent_degraded_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is True
+
+
+def test_pathless_contingent_criterion_without_blocker_key_abstains_without_credit() -> None:
+ contingent = _criterion(
+ "c_contingent",
+ "The earliest available start date is reported to the user.",
+ contingent_on="the provider offers a scheduling window",
+ output_path="output.desired_start_date",
+ mint_degrade="contingent_missing_antecedent",
+ )
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"submitted": True}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+ solo_result = _graded_result([contingent], snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.contingent_degraded_criterion_ids == [contingent.id]
+ assert result.is_fully_satisfied() is True
+ assert contingent.id not in result.to_trace_data()["unmet_criterion_ids"]
+ assert solo_result.is_fully_satisfied() is False
+
+
+def test_pathed_blocker_contingent_criterion_still_abstains_on_empty_blocker() -> None:
+ contingent = _criterion(
+ "c_contingent",
+ "A blocker is reported to the user.",
+ contingent_on="the site blocks online submission",
+ contingent_antecedent_output_path="output.blocker",
+ )
+ criteria = [*_registered_output_criteria(), contingent]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result(criteria, snapshot, {contingent.id})
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.contingent_degraded_criterion_ids == []
+ assert result.is_fully_satisfied() is True
+
+
+def test_turn_unsatisfiable_fallback_still_vetoes_and_stays_in_blocking_lane() -> None:
+ fallback = _criterion(
+ "c_fallback",
+ "The requested turn outcome is reached.",
+ mint_degrade="turn_unsatisfiable_fallback",
+ )
+ criteria = [*_registered_output_criteria(), fallback]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result(criteria, snapshot, {fallback.id})
+
+ assert is_turn_unsatisfiable_fallback_degraded(fallback) is True
+ assert is_contingent_missing_antecedent_degraded(fallback) is False
+ assert result.structural_unfired_criterion_ids == []
+ assert result.degraded_criterion_ids == [fallback.id]
+ assert result.contingent_degraded_criterion_ids == []
+ assert result.is_fully_satisfied() is False
+ assert only_degraded_blocking(result) is True
+
+
+def test_abstained_criterion_with_satisfied_verdict_is_not_a_credit_authorizer() -> None:
+ contingent = _blocker_contingent_criterion()
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result([contingent], snapshot, set())
+
+ assert result.structural_unfired_criterion_ids == [contingent.id]
+ assert result.verdicts[0].satisfied is True
+ assert result.is_fully_satisfied() is False
+
+
+def test_abstained_criterion_earns_no_credit_via_corroborated_requested_output_door() -> None:
+ contingent = _blocker_contingent_criterion()
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result([contingent], snapshot, {contingent.id})
+ result = replace(
+ result,
+ verdicts=[
+ replace(
+ result.verdicts[0],
+ reason_code="structurally_abstained",
+ evidence_ref="extract_result",
+ output_path="output.blocker",
+ ),
+ CriterionVerdict(
+ criterion_id="c_terminal_record",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ grounding_mode="terminal_record",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_abstained_criterion_earns_no_credit_via_self_emitted_corroborator_door() -> None:
+ contingent = _blocker_contingent_criterion()
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result([contingent], snapshot, {contingent.id})
+ result = replace(
+ result,
+ verdicts=[replace(result.verdicts[0], self_emitted_judgment_not_independent=True)],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_fallback_degraded_criterion_in_structural_unfired_set_still_vetoes() -> None:
+ fallback = _criterion(
+ "c_fallback",
+ "The requested turn outcome is reached.",
+ contingent_on="the site only exposes an email or manual submission path",
+ mint_degrade="turn_unsatisfiable_fallback",
+ )
+ criteria = [*_registered_output_criteria(), fallback]
+ snapshot = RunEvidenceSnapshot(block_outputs={"extract_result": {"blocker": ""}})
+
+ result = _graded_result(criteria, snapshot, {fallback.id})
+ result = replace(result, structural_unfired_criterion_ids=[fallback.id])
+
+ assert result.degraded_criterion_ids == [fallback.id]
+ assert result.contingent_degraded_criterion_ids == []
+ assert result.is_fully_satisfied() is False
diff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
index 7690fd520..52e5eef72 100644
--- a/tests/unit/test_copilot_sky11865_mint_satisfiability.py
+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
@@ -25,7 +25,9 @@ from skyvern.forge.sdk.copilot.request_policy import (
_classifier_fallback_policy,
_fallback_literal_candidates_for_field,
build_classifier_fallback_floor,
+ is_contingent_missing_antecedent_degraded,
is_turn_unsatisfiable_fallback_degraded,
+ resolve_mint_degrade,
)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools.run_execution import _terminal_challenge_completion_verification
@@ -454,7 +456,7 @@ def test_recorded_branch_no_meaningful_output_degrades() -> None:
assert outcome.reason_code == "fallback_floor_turn_unsatisfiable"
-def test_contingent_missing_antecedent_counts_as_degraded() -> None:
+def test_contingent_missing_antecedent_routes_to_its_own_degrade_lane() -> None:
criterion = CompletionCriterion(
id="c1",
outcome="A blocker is reported to the user.",
@@ -462,8 +464,9 @@ def test_contingent_missing_antecedent_counts_as_degraded() -> None:
contingent_on="the site blocks submission",
mint_degrade="contingent_missing_antecedent",
)
- assert is_turn_unsatisfiable_fallback_degraded(criterion) is True
- assert is_turn_unsatisfiable_fallback_degraded(replace(criterion, mint_degrade=None)) is False
+ assert is_contingent_missing_antecedent_degraded(criterion) is True
+ assert is_turn_unsatisfiable_fallback_degraded(criterion) is False
+ assert is_contingent_missing_antecedent_degraded(replace(criterion, mint_degrade=None)) is False
def test_contingent_missing_antecedent_carries_and_blocks_credit_when_unsatisfied() -> None:
@@ -486,8 +489,10 @@ def test_contingent_missing_antecedent_carries_and_blocks_credit_when_unsatisfie
)
],
)
- assert "c1" in result.degraded_criterion_ids
+ assert "c1" in result.contingent_degraded_criterion_ids
+ assert "c1" not in result.degraded_criterion_ids
assert result.is_fully_satisfied() is False
+ assert only_degraded_blocking(result) is True
def test_satisfied_degraded_criterion_still_credits_on_merits() -> None:
@@ -515,3 +520,12 @@ def test_recorded_branch_ambiguous_code_with_failed_block_stays_repairable() ->
assert outcome is not None
assert outcome.verdict == "repairable_failure"
assert outcome.reason_code != "fallback_floor_turn_unsatisfiable"
+
+
+def test_turn_unsatisfiable_fallback_stamp_precedes_pathless_contingent_degrade() -> None:
+ resolved = resolve_mint_degrade(
+ "turn_unsatisfiable_fallback",
+ "the site blocks submission",
+ None,
+ )
+ assert resolved == "turn_unsatisfiable_fallback"