Skip to content

Automate browser based workflows with AI

SKY-12309: Avoid failing successful Copilot runs when an optional blocker is absent (#7549)

Andrew Neilsoned492e4f5649
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
index ae8d47f72..b63c779cd 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
@@ -1,12 +1,14 @@
Classify the request-owned output slots for the latest Workflow Copilot turn. Return only JSON.
Schema-valid examples:
-{"version":"1","slots":[{"source_id":"u0","source_quote":"required_columns","plane":"definition","pinability":"shapeless_valid"}]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"processed record count","plane":"run","pinability":"shapeless_valid"}]}
-{"version":"1","slots":[{"source_id":"u0","source_quote":"status must equal complete","plane":"run","pinability":"pinned"}]}
+{"version":"1","slots":[]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"required_columns","plane":"definition","pinability":"shapeless_valid","antecedent_family":"unconditional"}]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"processed record count","plane":"run","pinability":"shapeless_valid","antecedent_family":"unconditional"}]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"status must equal complete","plane":"run","pinability":"pinned","antecedent_family":"unconditional"}]}
+{"version":"1","slots":[{"source_id":"u0","source_quote":"report the blocker if online submission is unavailable","plane":"run","pinability":"shapeless_valid","antecedent_family":"blocker"}]}
Rules:
-- Return version exactly "1" and at least one slot.
+- Return version exactly "1". Return an empty slots list when the user requests no output member; empty membership is an explicit contract, not an error.
- For every request-owned output member, copy the smallest unique exact quote that identifies it from one supplied user source. Return that source_id and quote exactly; the server owns paths, IDs, order, and count.
- An output member is a value the user asks the workflow to expose in its returned or defined output contract. A task, action, navigation goal, or evidence-gathering instruction is not a separate output member merely because the workflow performs or classifies it. Include its result only when the user separately asks to expose that value and it is not already represented by a more explicit requested output member.
- Context outside the supplied user sources may inform plane and pinability, but it is never an anchor source.
@@ -16,7 +18,11 @@ Rules:
- pinned: the request itself fixes an exact expected value or exact supported expected shape.
- shapeless_valid: the output is valid without fixing an exact value or supported shape. Evidence-decided discovery questions, including whether questions, use this value and never a guessed polarity. A boolean or classification answer space alone never supplies a request-fixed polarity and is not pinned.
- unpinnable: the requested output cannot be safely bound to a satisfiable output criterion.
-- Include every request-owned output member exactly once. Do not emit paths, path segments, IDs, aliases, labels, counts, expected values, explanations, or unknown fields.
+- antecedent_family is required on every slot:
+ - unconditional: this output is required regardless of any runtime condition.
+ - blocker: this output is a consequent requested only if a provider/site blocker, manual-only path, or equivalent inability to complete online occurs.
+ - undecidable: the source does not establish whether the output is unconditional or blocker-contingent. Never guess unconditional.
+- Include every request-owned output member exactly once. A conditional fallback is still an output member even when another classifier might omit its condition. Do not emit paths, path segments, IDs, aliases, labels, counts, expected values, explanations, or unknown fields.
- Do not infer slots from sampled page shapes or non-user context. Do not normalize, rename, or paraphrase source quotes.
Ordered, deduplicated user request sources:
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index dff3f788e..50aa2f01a 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -12,13 +12,18 @@ from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from typing import Any, Literal, cast, get_args
-from skyvern.forge.sdk.copilot.completion_output_grounding import split_requested_output_criteria
+from skyvern.forge.sdk.copilot.completion_output_grounding import (
+ _normalize_output_path,
+ split_requested_output_criteria,
+)
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
run_plane_all_no_evidence,
)
from skyvern.forge.sdk.copilot.request_policy import (
+ ANTECEDENT_FAMILY_VALUES,
REQUESTED_OUTPUT_PATH_MINT_SOURCES,
+ AntecedentFamily,
CompletionCriterion,
CriterionKind,
ExpectedOutputShape,
@@ -63,6 +68,15 @@ _MINT_DISPOSITIONS = frozenset(get_args(MintDisposition))
_PINABILITIES = frozenset(get_args(Pinability))
+def _normalize_floor_rekeyed_association(marker: object, path: object) -> tuple[bool, str | None]:
+ if marker is not True or not isinstance(path, str) or not path.strip().startswith("output."):
+ return False, None
+ normalized_path = _normalize_output_path(path)
+ if normalized_path:
+ return True, f"output.{normalized_path}"
+ return False, None
+
+
@dataclass(frozen=True)
class StoredCriteriaSet:
set_id: str
@@ -164,8 +178,17 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
item["terminal_action_verification_mode"] = criterion.terminal_action_verification_mode
if criterion.requested_output_corroborator:
item["requested_output_corroborator"] = True
+ if criterion.antecedent_family is not None:
+ item["antecedent_family"] = criterion.antecedent_family
if criterion.requested_output_path_mint_source is not None:
item["requested_output_path_mint_source"] = criterion.requested_output_path_mint_source
+ floor_rekeyed, floor_rekeyed_from_path = _normalize_floor_rekeyed_association(
+ criterion.requested_output_floor_rekeyed,
+ criterion.floor_rekeyed_from_path,
+ )
+ if floor_rekeyed:
+ item["requested_output_floor_rekeyed"] = True
+ item["floor_rekeyed_from_path"] = floor_rekeyed_from_path
if criterion.mint_degrade is not None:
item["mint_degrade"] = criterion.mint_degrade
if criterion.judgment_truth_condition is not None:
@@ -181,6 +204,41 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
return items
+def criterion_authority_projection(criterion: CompletionCriterion) -> dict[str, Any]:
+ """Canonical values for persisted fields that can change grading authority.
+
+ Repository admission compares only fields present in a stored row, preserving
+ omitted legacy defaults while rejecting values the tolerant decoder normalized.
+ """
+ item = criteria_to_json([criterion])[0]
+ floor_rekeyed, floor_rekeyed_from_path = _normalize_floor_rekeyed_association(
+ criterion.requested_output_floor_rekeyed,
+ criterion.floor_rekeyed_from_path,
+ )
+ item.update(
+ {
+ "antecedent_family": criterion.antecedent_family,
+ "deliverable_confirmation_criterion_id": criterion.deliverable_confirmation_criterion_id,
+ "terminal_action_verification_mode": criterion.terminal_action_verification_mode,
+ "requested_output_corroborator": criterion.requested_output_corroborator,
+ "requested_output_path_mint_source": criterion.requested_output_path_mint_source,
+ "requested_output_floor_rekeyed": floor_rekeyed,
+ "floor_rekeyed_from_path": floor_rekeyed_from_path,
+ "mint_degrade": criterion.mint_degrade,
+ "judgment_predicate": (
+ criterion.judgment_truth_condition.predicate if criterion.judgment_truth_condition else None
+ ),
+ "judgment_polarity_when_holds": (
+ criterion.judgment_truth_condition.polarity_when_holds if criterion.judgment_truth_condition else None
+ ),
+ "request_slot_id": criterion.request_slot_id,
+ "pinability": criterion.pinability,
+ "mint_disposition": criterion.mint_disposition,
+ }
+ )
+ return item
+
+
def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
if not isinstance(raw, list):
return ()
@@ -208,6 +266,13 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(
item.get("contingent_antecedent_output_path")
)
+ antecedent_family_raw = item.get("antecedent_family")
+ antecedent_family = cast(
+ AntecedentFamily | None,
+ antecedent_family_raw
+ if isinstance(antecedent_family_raw, str) and antecedent_family_raw in ANTECEDENT_FAMILY_VALUES
+ else None,
+ )
kind_raw = item.get("kind")
kind = kind_raw if isinstance(kind_raw, str) and kind_raw in _CRITERION_KINDS else "outcome"
family_raw = item.get("terminal_action_family")
@@ -237,6 +302,10 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
pinability = pinability_raw if pinability_raw in _PINABILITIES else None
mint_disposition_raw = item.get("mint_disposition")
mint_disposition = mint_disposition_raw if mint_disposition_raw in _MINT_DISPOSITIONS else "decidable"
+ requested_output_floor_rekeyed, floor_rekeyed_from_path = _normalize_floor_rekeyed_association(
+ item.get("requested_output_floor_rekeyed"),
+ item.get("floor_rekeyed_from_path"),
+ )
stored_output_path = output_path.strip() if isinstance(output_path, str) and output_path.strip() else None
stored_expected_output_value = _coerce_expected_output_value(expected_output_value)
stored_expected_output_shape = cast(ExpectedOutputShape | None, expected_output_shape)
@@ -284,6 +353,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
outcome=outcome,
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ antecedent_family=antecedent_family,
deliverable_kind=stored_deliverable_kind,
deliverable_confirmation_criterion_id=deliverable_confirmation_criterion_id,
declared_deliverable_kind=_normalize_deliverable_kind(item.get("declared_deliverable_kind")),
@@ -303,6 +373,8 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
classification_output_key=classification_output_key,
expected_classification=expected_classification,
requested_output_corroborator=bool(item.get("requested_output_corroborator")),
+ requested_output_floor_rekeyed=requested_output_floor_rekeyed,
+ floor_rekeyed_from_path=floor_rekeyed_from_path,
mint_degrade=resolve_mint_degrade(
item.get("mint_degrade"), contingent_on, contingent_antecedent_output_path
),
@@ -320,6 +392,7 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
+ antecedent_family_key = criterion.antecedent_family or ""
deliverable_kind_key = (
f"{criterion.deliverable_kind or ''}\x1fdeclared:{criterion.declared_deliverable_kind or ''}"
f"\x1fconfirmation:{criterion.deliverable_confirmation_criterion_id or ''}"
@@ -335,9 +408,17 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
expected_classification_key = (
str(criterion.expected_classification) if criterion.expected_classification is not None else ""
)
+ floor_rekeyed, floor_rekeyed_from_path = _normalize_floor_rekeyed_association(
+ criterion.requested_output_floor_rekeyed,
+ criterion.floor_rekeyed_from_path,
+ )
+ floor_rekeyed_key = (
+ f"\x1ffloor_rekeyed:{str(floor_rekeyed).lower()}\x1ffloor_rekeyed_from_path:{floor_rekeyed_from_path or ''}"
+ )
if criterion.output_path:
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
+ f"\x1fantecedent_family:{antecedent_family_key}"
f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1foutput_path:{criterion.output_path}"
f"\x1fexpected_output_value:{expected_output_value_key}"
@@ -347,21 +428,26 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
f"\x1fterminal_action_verification_mode:{terminal_action_verification_mode_key}"
f"\x1fclassification_output_key:{classification_output_key}"
f"\x1fexpected_classification:{expected_classification_key}"
+ f"{floor_rekeyed_key}"
)
if criterion.kind == "validation_classification":
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
+ f"\x1fantecedent_family:{antecedent_family_key}"
f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1fkind:{criterion.kind}"
f"\x1fclassification_output_key:{classification_output_key}"
f"\x1fexpected_classification:{expected_classification_key}"
+ f"{floor_rekeyed_key}"
)
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
+ f"\x1fantecedent_family:{antecedent_family_key}"
f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1fkind:{criterion.kind}"
f"\x1fterminal_action_verification_mode:{terminal_action_verification_mode_key}"
f"\x1foutcome:{normalized_criterion_outcome_key(criterion.outcome)}"
+ f"{floor_rekeyed_key}"
)
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index cb29a91ca..78c207dab 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -120,6 +120,15 @@ class PlainOutcomeNoEvidenceAbstentionDecision:
confirming_deliverable_sources: tuple[AuthoritativeDeliverableSource, ...]
+@dataclass(frozen=True)
+class RegisteredBlockerEvidence:
+ block_label: str
+ output_path: str
+ registered_output_key: str
+ registered_output_id: str | None
+ value: Any
+
+
@dataclass(frozen=True)
class CompletionVerificationResult:
status: VerificationStatus
@@ -129,6 +138,7 @@ class CompletionVerificationResult:
contingent_criterion_ids: list[str] = field(default_factory=list)
contingent_on_by_criterion_id: dict[str, str] = field(default_factory=dict)
contingent_antecedent_output_path_by_criterion_id: dict[str, str] = field(default_factory=dict)
+ antecedent_family_by_criterion_id: dict[str, str] = field(default_factory=dict)
structural_unfired_criterion_ids: list[str] = field(default_factory=list)
degraded_criterion_ids: list[str] = field(default_factory=list)
floor_rekeyed_criterion_ids: list[str] = field(default_factory=list)
@@ -142,6 +152,9 @@ class CompletionVerificationResult:
criterion_deliverable_kind_by_id: dict[str, Literal["registered_download"]] = field(default_factory=dict)
criterion_requested_output_mint_source_by_id: dict[str, RequestedOutputPathMintSource] = field(default_factory=dict)
deliverable_confirmation_criterion_id_by_id: dict[str, str] = field(default_factory=dict)
+ registered_blocker_evidence_by_request_slot_id: dict[str, tuple[RegisteredBlockerEvidence, ...]] = field(
+ default_factory=dict
+ )
def is_fully_satisfied(self) -> bool:
if self.status != "evaluated" or not self.criterion_ids:
@@ -226,16 +239,23 @@ class CompletionVerificationResult:
detail = verdict_missing_evidence(verdict)
if detail:
missing_evidence.append(f"{verdict.criterion_id}: {detail}")
+ trace_state_by_criterion_id = {
+ verdict.criterion_id: self._trace_state_and_reason(verdict) for verdict in self.verdicts
+ }
data: dict[str, Any] = {
"status": self.status,
"criterion_count": len(self.criterion_ids),
- "satisfied_count": sum(1 for verdict in self.verdicts if verdict.state == "satisfied"),
- "unsatisfied_count": sum(1 for verdict in self.verdicts if verdict.state == "unsatisfied"),
- "unknown_count": sum(1 for verdict in self.verdicts if verdict.state == "unknown"),
+ "satisfied_count": sum(
+ 1 for state, _reason in trace_state_by_criterion_id.values() if state == "satisfied"
+ ),
+ "unsatisfied_count": sum(
+ 1 for state, _reason in trace_state_by_criterion_id.values() if state == "unsatisfied"
+ ),
+ "unknown_count": sum(1 for state, _reason in trace_state_by_criterion_id.values() if state == "unknown"),
"fully_satisfied": self.is_fully_satisfied(),
"no_gradeable_run_plane": self.no_gradeable_run_plane,
"requested_output_criteria_count": self.requested_output_criteria_count,
- "reason_codes": [verdict.reason_code for verdict in self.verdicts],
+ "reason_codes": [trace_state_by_criterion_id[verdict.criterion_id][1] for verdict in self.verdicts],
"unmet_criterion_ids": [verdict.criterion_id for verdict in unmet],
"missing_evidence": missing_evidence,
}
@@ -269,15 +289,22 @@ class CompletionVerificationResult:
data["structural_unfired_criterion_ids"] = list(self.structural_unfired_criterion_ids)
if self.contingent_degraded_criterion_ids:
data["contingent_degraded_criterion_ids"] = list(self.contingent_degraded_criterion_ids)
+ if self.antecedent_family_by_criterion_id:
+ data["antecedent_family_by_criterion_id"] = dict(self.antecedent_family_by_criterion_id)
contingent_id_set = set(self.contingent_criterion_ids)
structural_unfired_id_set = set(self.structural_unfired_criterion_ids)
plain_outcome_abstained_id_set = set(plain_outcome_decision.criterion_ids) if plain_outcome_decision else set()
for index, verdict in enumerate(self.verdicts[:_MAX_TRACE_VERDICTS]):
prefix = f"verdict_{index}"
+ structural_abstention = self.is_structural_contingent_abstention(verdict)
data[f"{prefix}_criterion_id"] = verdict.criterion_id
- data[f"{prefix}_state"] = verdict.state
- data[f"{prefix}_satisfied"] = verdict.satisfied
- data[f"{prefix}_reason_code"] = verdict.reason_code
+ data[f"{prefix}_state"] = "unknown" if structural_abstention else verdict.state
+ data[f"{prefix}_satisfied"] = False if structural_abstention else verdict.satisfied
+ data[f"{prefix}_reason_code"] = (
+ _STRUCTURAL_ABSTENTION_REASON_CODE if structural_abstention else verdict.reason_code
+ )
+ if structural_abstention:
+ data[f"{prefix}_abstention_reason_code"] = "unfired_contingent_antecedent"
if verdict.output_path:
data[f"{prefix}_output_path"] = verdict.output_path
if verdict.grounding_mode:
@@ -303,6 +330,19 @@ class CompletionVerificationResult:
data[f"{prefix}_contingent_on"] = contingent_on
if contingent_path := self.contingent_antecedent_output_path_by_criterion_id.get(verdict.criterion_id):
data[f"{prefix}_contingent_antecedent_output_path"] = contingent_path
+ if antecedent_family := self.antecedent_family_by_criterion_id.get(verdict.criterion_id):
+ data[f"{prefix}_antecedent_family"] = antecedent_family
+ registered_blocker_evidence = self.registered_blocker_evidence_by_request_slot_id.get(
+ verdict.criterion_id, ()
+ )
+ if len(registered_blocker_evidence) == 1:
+ registered_source = registered_blocker_evidence[0]
+ data[f"{prefix}_antecedent_evidence_source"] = "registered_output_parameter"
+ data[f"{prefix}_antecedent_producer_block_label"] = registered_source.block_label
+ data[f"{prefix}_antecedent_output_path"] = registered_source.output_path
+ data[f"{prefix}_antecedent_registered_output_key"] = registered_source.registered_output_key
+ if registered_source.registered_output_id is not None:
+ data[f"{prefix}_antecedent_registered_output_id"] = registered_source.registered_output_id
if verdict.criterion_id in contingent_id_set or verdict.criterion_id in structural_unfired_id_set:
data[f"{prefix}_structural_unfired"] = verdict.criterion_id in structural_unfired_id_set
evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)
@@ -318,11 +358,17 @@ class CompletionVerificationResult:
data[f"{prefix}_missing_evidence"] = detail
return data
+ def _trace_state_and_reason(self, verdict: CriterionVerdict) -> tuple[CriterionState, str]:
+ if self.is_structural_contingent_abstention(verdict):
+ return "unknown", _STRUCTURAL_ABSTENTION_REASON_CODE
+ return verdict.state, verdict.reason_code
+
def verdict_state_counts(self) -> dict[str, int]:
+ trace_states = [self._trace_state_and_reason(verdict)[0] for verdict in self.verdicts]
return {
- "satisfied": sum(1 for verdict in self.verdicts if verdict.state == "satisfied"),
- "unsatisfied": sum(1 for verdict in self.verdicts if verdict.state == "unsatisfied"),
- "unknown": sum(1 for verdict in self.verdicts if verdict.state == "unknown"),
+ "satisfied": trace_states.count("satisfied"),
+ "unsatisfied": trace_states.count("unsatisfied"),
+ "unknown": trace_states.count("unknown"),
}
@@ -336,6 +382,10 @@ class RunEvidenceSnapshot:
workflow_run_id: str | None = None
block_outputs: dict[str, Any] = field(default_factory=dict)
block_output_sources: dict[str, EvidenceSourceKind] = field(default_factory=dict)
+ registered_output_values: dict[str, Any] = field(default_factory=dict)
+ registered_blocker_evidence_by_request_slot_id: dict[str, tuple[RegisteredBlockerEvidence, ...]] = field(
+ default_factory=dict
+ )
current_url: str | None = None
page_title: str | None = None
run_terminal_status: str | None = None
@@ -522,6 +572,7 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:
if criterion.contingent_antecedent_output_path
else ""
)
+ antecedent_family = f" [antecedent_family={criterion.antecedent_family}]" if criterion.antecedent_family else ""
deliverable_kind = f" [deliverable_kind={criterion.deliverable_kind}]" if criterion.deliverable_kind else ""
output_path = f" [required_output_path={criterion.output_path}]" if criterion.output_path else ""
classification_output_key = (
@@ -537,7 +588,7 @@ def _render_criteria(criteria: list[CompletionCriterion]) -> str:
parts.append(
f"- {criterion.id}: {criterion.outcome}"
f" [kind={criterion.kind}]"
- f"{contingent_on}{antecedent_output_path}{deliverable_kind}{output_path}"
+ f"{contingent_on}{antecedent_output_path}{antecedent_family}{deliverable_kind}{output_path}"
f"{classification_output_key}{expected_classification}{flags}"
)
return "\n".join(parts)
@@ -554,7 +605,12 @@ def _contingent_metadata_for_criteria(
for criterion in criteria
if criterion.contingent_antecedent_output_path is not None
}
- contingent_ids = list(dict.fromkeys([*contingent_on_by_id, *contingent_antecedent_output_path_by_id]))
+ family_contingent_ids = [
+ criterion.id for criterion in criteria if criterion.antecedent_family in {"blocker", "undecidable"}
+ ]
+ contingent_ids = list(
+ dict.fromkeys([*contingent_on_by_id, *contingent_antecedent_output_path_by_id, *family_contingent_ids])
+ )
return contingent_ids, contingent_on_by_id, contingent_antecedent_output_path_by_id
@@ -579,7 +635,6 @@ def _find_structured_field_values(value: Any, field_names: Collection[str]) -> I
_BLOCKER_PATH_FIELDS = frozenset({"blocker", "manual_service_blocker"})
-_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})
def structural_unfired_contingent_criterion_ids(
@@ -588,12 +643,24 @@ def structural_unfired_contingent_criterion_ids(
) -> list[str]:
unfired_ids: list[str] = []
for criterion in criteria:
+ if criterion.antecedent_family == "blocker":
+ if _blocker_criterion_may_structurally_abstain(criterion, snapshot):
+ unfired_ids.append(criterion.id)
+ continue
+ if criterion.antecedent_family == "undecidable":
+ unfired_ids.append(criterion.id)
+ continue
+ if criterion.antecedent_family == "unconditional":
+ continue
if is_turn_unsatisfiable_fallback_degraded(criterion):
continue
if is_contingent_missing_antecedent_degraded(criterion):
# The degrade value is itself the source determination that the antecedent never fired,
# so only affirmative evidence that a real blocker fired can keep the criterion in play.
- if not (_contingent_antecedent_blocker_shaped(criterion) and _has_real_blocker_evidence(snapshot)):
+ if not (
+ _contingent_antecedent_blocker_shaped(criterion)
+ and _has_real_legacy_criterion_antecedent_evidence(criterion, snapshot)
+ ):
unfired_ids.append(criterion.id)
continue
path = criterion.contingent_antecedent_output_path
@@ -606,9 +673,9 @@ def structural_unfired_contingent_criterion_ids(
values.append(value)
values.extend(_find_structured_field_values(value, field_names))
if _is_blocker_contingent_criterion(criterion):
- if _has_real_blocker_evidence(snapshot):
+ if _has_real_legacy_criterion_antecedent_evidence(criterion, snapshot):
continue
- if _has_structural_no_blocker_evidence(snapshot):
+ if _has_structural_no_blocker_evidence(criterion, snapshot):
unfired_ids.append(criterion.id)
continue
if not values:
@@ -635,27 +702,57 @@ def _contingent_antecedent_blocker_shaped(criterion: CompletionCriterion) -> boo
return path is not None and _output_path_field(path) in _BLOCKER_PATH_FIELDS
-def _blocker_family_values(snapshot: RunEvidenceSnapshot) -> list[Any]:
+def _legacy_criterion_antecedent_values(
+ criterion: CompletionCriterion,
+ snapshot: RunEvidenceSnapshot,
+) -> list[Any]:
+ path = criterion.contingent_antecedent_output_path or criterion.output_path
+ if path is None:
+ return []
+ field_names = set(_field_aliases_for_output_path(path))
+ if _output_path_field(path) in _BLOCKER_PATH_FIELDS:
+ field_names.update(_BLOCKER_PATH_FIELDS)
+ field_names.update(f"{field}_output" for field in _BLOCKER_PATH_FIELDS)
values: list[Any] = []
for key, value in snapshot.block_outputs.items():
- if key in _BLOCKER_FIELDS:
+ if key in field_names:
values.append(value)
- values.extend(_find_structured_field_values(value, _BLOCKER_FIELDS))
+ values.extend(_find_structured_field_values(value, field_names))
return values
-def _has_real_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:
- return any(_is_real_blocker_evidence(value) for value in _blocker_family_values(snapshot))
+def _blocker_criterion_may_structurally_abstain(
+ criterion: CompletionCriterion,
+ snapshot: RunEvidenceSnapshot,
+) -> bool:
+ if criterion.request_slot_id is None:
+ return not snapshot.registered_output_values
+ evidence = snapshot.registered_blocker_evidence_by_request_slot_id.get(criterion.request_slot_id, ())
+ if len(evidence) == 1:
+ return _is_structural_no_blocker_marker(evidence[0].value)
+ if evidence:
+ return False
+ # Registered values with no accepted association fail closed: they may contain the fired
+ # blocker whose consequent would otherwise be waived. Another slot's association cannot
+ # license this criterion to abstain.
+ return not snapshot.registered_output_values
+
+
+def _has_real_legacy_criterion_antecedent_evidence(
+ criterion: CompletionCriterion,
+ snapshot: RunEvidenceSnapshot,
+) -> bool:
+ return any(_is_real_blocker_evidence(value) for value in _legacy_criterion_antecedent_values(criterion, snapshot))
def _is_real_blocker_evidence(value: Any) -> bool:
return _is_meaningful_contingent_antecedent_value(value) and not _is_structural_no_blocker_marker(value)
-def _has_structural_no_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:
+def _has_structural_no_blocker_evidence(criterion: CompletionCriterion, snapshot: RunEvidenceSnapshot) -> bool:
"""A registered blocker-family key is an explicit negative record whatever its value shape
- (None/""/false/"none"); only an absent key is an absent record, which ``_blocker_family_values`` guarantees."""
- for value in _blocker_family_values(snapshot):
+ (None/""/false/"none"); only an absent key is an absent record."""
+ for value in _legacy_criterion_antecedent_values(criterion, snapshot):
if _is_structural_no_blocker_marker(value):
return True
return False
@@ -2719,15 +2816,28 @@ def carry_degraded_criterion_ids(
contingent_degraded = [
criterion.id for criterion in criteria if is_contingent_missing_antecedent_degraded(criterion)
]
+ antecedent_family_by_id = {
+ criterion.id: criterion.antecedent_family for criterion in criteria if criterion.antecedent_family is not None
+ }
+ family_contingent_ids = [
+ criterion.id for criterion in criteria if criterion.antecedent_family in {"blocker", "undecidable"}
+ ]
if (
not degraded
and not contingent_degraded
+ and not antecedent_family_by_id
and not result.degraded_criterion_ids
and not result.contingent_degraded_criterion_ids
+ and not result.antecedent_family_by_criterion_id
):
return result
return replace(
result,
+ contingent_criterion_ids=list(dict.fromkeys([*result.contingent_criterion_ids, *family_contingent_ids])),
+ antecedent_family_by_criterion_id={
+ **result.antecedent_family_by_criterion_id,
+ **antecedent_family_by_id,
+ },
degraded_criterion_ids=list(dict.fromkeys([*result.degraded_criterion_ids, *degraded])),
contingent_degraded_criterion_ids=list(
dict.fromkeys([*result.contingent_degraded_criterion_ids, *contingent_degraded])
@@ -2885,7 +2995,7 @@ async def evaluate_completion_criteria(
structural_unfired_criterion_ids=structural_unfired_ids,
)
- return _coerce_result(
+ result = _coerce_result(
raw,
criterion_ids,
contingent_criterion_ids=contingent_ids,
@@ -2894,3 +3004,7 @@ async def evaluate_completion_criteria(
structural_unfired_criterion_ids=structural_unfired_ids,
block_output_sources=snapshot.block_output_sources,
)
+ return replace(
+ result,
+ registered_blocker_evidence_by_request_slot_id=dict(snapshot.registered_blocker_evidence_by_request_slot_id),
+ )
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 929848d92..d41627ec7 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -220,6 +220,7 @@ RequestedOutputEvidenceSource = Literal[
JudgmentPredicate = Literal["login_gate_blocks_target"]
MintDisposition = Literal["pending", "decidable", "degraded"]
Pinability = Literal["pinned", "shapeless_valid", "unpinnable"]
+AntecedentFamily = Literal["unconditional", "blocker", "undecidable"]
_JUDGMENT_PREDICATES: frozenset[str] = frozenset(get_args(JudgmentPredicate))
MintDegrade = Literal[
"turn_unsatisfiable_fallback",
@@ -227,6 +228,7 @@ MintDegrade = Literal[
"undecidable_judgment",
]
MINT_DEGRADE_VALUES: frozenset[str] = frozenset(get_args(MintDegrade))
+ANTECEDENT_FAMILY_VALUES: frozenset[str] = frozenset(get_args(AntecedentFamily))
_CRITERION_KINDS: frozenset[str] = frozenset({"outcome", "terminal_action", "validation_classification"})
_TERMINAL_ACTION_FAMILIES: frozenset[str] = frozenset({"request", "application", "form", "order"})
_EXPECTED_OUTPUT_SHAPES: frozenset[str] = frozenset(get_args(ExpectedOutputShape))
@@ -301,6 +303,7 @@ class CompletionCriterion:
outcome: str
contingent_on: str | None = None
contingent_antecedent_output_path: str | None = None
+ antecedent_family: AntecedentFamily | None = None
deliverable_kind: Literal["registered_download"] | None = None
# Author-time seam signal only: unlike ``deliverable_kind`` it survives canonicalization onto
# non-canonical output paths, so it is never rendered to the completion verifier.
@@ -433,6 +436,14 @@ class RequestPolicy:
data["requested_output_criteria_count"] = len(requested_output_criteria)
if self.request_slot_failure_kind is not None:
data["request_slot_failure_kind"] = self.request_slot_failure_kind
+ family_criteria = [
+ criterion for criterion in self.graded_completion_criteria() if criterion.antecedent_family is not None
+ ]
+ data["antecedent_family_criterion_count"] = len(family_criteria)
+ for index, criterion in enumerate(family_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):
+ prefix = f"antecedent_family_criterion_{index}"
+ data[f"{prefix}_id"] = criterion.id
+ data[f"{prefix}_antecedent_family"] = criterion.antecedent_family
for index, criterion in enumerate(requested_output_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):
prefix = f"requested_output_criterion_{index}"
data[f"{prefix}_id"] = criterion.id
@@ -1087,6 +1098,7 @@ def _degrade_unbound_request_slot_criterion(criterion: CompletionCriterion) -> C
expected_classification=None,
judgment_truth_condition=None,
kind="outcome",
+ antecedent_family="undecidable",
mint_disposition="degraded",
mint_degrade="undecidable_judgment",
requested_output_floor_rekeyed=True,
@@ -1126,7 +1138,12 @@ def _bind_criterion_to_request_slot(
) -> CompletionCriterion:
pinability = cast(Pinability, slot.pinability.value)
has_exact_requirement = _request_slot_has_exact_requirement(criterion)
- degraded = pinability == "unpinnable" or (pinability == "pinned" and not has_exact_requirement)
+ antecedent_family = cast(AntecedentFamily, slot.antecedent_family.value)
+ degraded = (
+ pinability == "unpinnable"
+ or (pinability == "pinned" and not has_exact_requirement)
+ or antecedent_family == "undecidable"
+ )
rekeyed = degraded or pinability == "shapeless_valid"
expected_output_value = criterion.expected_output_value
@@ -1168,6 +1185,7 @@ def _bind_criterion_to_request_slot(
classification_output_key=classification_output_key,
expected_classification=expected_classification,
judgment_truth_condition=judgment_truth_condition,
+ antecedent_family=antecedent_family,
request_slot_id=slot.slot_id,
pinability=pinability,
mint_disposition="degraded" if degraded else ("pending" if pending else "decidable"),
@@ -2946,20 +2964,10 @@ async def _classify_request(
"request-policy request-slot anchor correction failed",
failure_kind=correction_failure_kind,
)
- declared_request_slots = (
- request_slot_failure_kind is None
- and isinstance(raw_criteria, list)
- and any(
- isinstance(item, dict)
- and _item_claims_request_slot(item)
- and _request_slot_anchor_is_admissible(item, request_slot_request=request_slot_request)
- for item in raw_criteria
- )
- )
- if declared_request_slots:
- # This validation hop is deliberately sequential: the request-policy
- # classifier first declares which criteria are request-owned slots, then
- # the independent producer types their identity, plane, and pinability.
+ if request_slot_failure_kind is None:
+ # The independent producer owns slot membership as well as type. Run it even
+ # when the primary classifier declares no slots so an omitted conditional
+ # fallback cannot disappear from the completion contract.
request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)
if request_slot_result.status == "success":
request_slot_contract = request_slot_result.contract
diff --git a/skyvern/forge/sdk/copilot/request_slots.py b/skyvern/forge/sdk/copilot/request_slots.py
index 85174bd56..5a0055726 100644
--- a/skyvern/forge/sdk/copilot/request_slots.py
+++ b/skyvern/forge/sdk/copilot/request_slots.py
@@ -46,6 +46,12 @@ class RequestSlotPinability(StrEnum):
UNPINNABLE = "unpinnable"
+class RequestSlotAntecedentFamily(StrEnum):
+ UNCONDITIONAL = "unconditional"
+ BLOCKER = "blocker"
+ UNDECIDABLE = "undecidable"
+
+
class RequestSlotProducerFailureKind(StrEnum):
MISSING_HANDLER = "missing_handler"
PROMPT_RENDER_ERROR = "prompt_render_error"
@@ -97,6 +103,7 @@ class RequestSlotDeclarationV1(BaseModel):
source_quote: str = Field(min_length=1, max_length=_MAX_SOURCE_QUOTE_CHARS)
plane: RequestSlotPlane
pinability: RequestSlotPinability
+ antecedent_family: RequestSlotAntecedentFamily
@field_validator("source_quote")
@classmethod
@@ -110,7 +117,7 @@ class RequestSlotEnvelopeV1(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
version: Literal["1"]
- slots: tuple[RequestSlotDeclarationV1, ...] = Field(min_length=1, max_length=64)
+ slots: tuple[RequestSlotDeclarationV1, ...] = Field(max_length=64)
def _request_digest(version: str, sources: tuple[RequestSlotSourceV1, ...]) -> str:
@@ -148,6 +155,7 @@ class CanonicalRequestSlotV1(BaseModel):
source_end: int = Field(ge=1, le=_MAX_REQUEST_PROMPT_CHARS)
plane: RequestSlotPlane
pinability: RequestSlotPinability
+ antecedent_family: RequestSlotAntecedentFamily
@field_validator("path_segments")
@classmethod
@@ -172,8 +180,8 @@ class RequestSlotContractV1(BaseModel):
version: Literal["1"]
request_digest: str = Field(min_length=64, max_length=64, pattern=r"^[0-9a-f]{64}$")
- slots: tuple[CanonicalRequestSlotV1, ...] = Field(min_length=1, max_length=64)
- count: int = Field(ge=1, le=64)
+ slots: tuple[CanonicalRequestSlotV1, ...] = Field(max_length=64)
+ count: int = Field(ge=0, le=64)
@model_validator(mode="after")
def _validate_derived_membership(self) -> RequestSlotContractV1:
@@ -340,6 +348,7 @@ def canonicalize_request_slots(
source_end=end,
plane=declaration.plane,
pinability=declaration.pinability,
+ antecedent_family=declaration.antecedent_family,
)
)
return RequestSlotContractV1(
@@ -389,6 +398,7 @@ def request_slot_contracts_agree(
and first_slot.source_id == second_slot.source_id
and first_slot.plane == second_slot.plane
and first_slot.pinability == second_slot.pinability
+ and first_slot.antecedent_family == second_slot.antecedent_family
and first_slot.source_start < second_slot.source_end
and second_slot.source_start < first_slot.source_end
for first_slot, second_slot in zip(first.slots, second.slots, strict=True)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 84c3c38a9..6bc9c81c1 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -9,6 +9,7 @@ from skyvern.config import settings
from skyvern.forge.sdk.copilot.challenge_evidence import carrier_backed_anti_bot_categories
from skyvern.forge.sdk.copilot.completion_criteria_store import note_adjudication_on_turn_state
from skyvern.forge.sdk.copilot.completion_output_grounding import (
+ _artifact_contract_paths,
_GroundingCtx,
floor_rekeyed_path_backing,
grade_requested_output_criteria,
@@ -20,6 +21,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
CriterionVerdict,
EvidenceSourceKind,
+ RegisteredBlockerEvidence,
RunEvidenceSnapshot,
_contingent_metadata_for_criteria,
_is_structural_requested_output_abstention,
@@ -300,6 +302,98 @@ def _accepted_staged_output_contract_metadata(copilot_ctx: Any) -> object:
return getattr(copilot_ctx, "code_artifact_metadata", None)
+def _exact_registered_output_path_value(value: object, output_path: str) -> tuple[bool, object]:
+ parts = output_path.removeprefix("output.").split(".")
+ if not parts or any(not part for part in parts):
+ return False, None
+ current = value
+ for part in parts:
+ if not isinstance(current, Mapping) or part not in current:
+ return False, None
+ current = current[part]
+ return True, current
+
+
+def _registered_blocker_evidence_by_request_slot_id(
+ copilot_ctx: Any,
+ run_data: Mapping[str, Any],
+) -> dict[str, tuple[RegisteredBlockerEvidence, ...]]:
+ """Bind blocker evidence through exact criterion -> artifact owner -> run row identity."""
+ metadata = _accepted_staged_output_contract_metadata(copilot_ctx)
+ if not isinstance(metadata, Mapping):
+ return {}
+ formed_criteria = _formed_completion_criteria(copilot_ctx)
+ blocker_criteria = [
+ criterion
+ for criterion in formed_criteria
+ if criterion.antecedent_family == "blocker" and criterion.request_slot_id is not None
+ ]
+ criterion_paths: dict[str, list[CompletionCriterion]] = {}
+ for criterion in formed_criteria:
+ output_path = criterion.output_path or criterion.floor_rekeyed_from_path
+ if isinstance(output_path, str) and output_path.startswith("output."):
+ criterion_paths.setdefault(output_path, []).append(criterion)
+
+ metadata_labels_by_path: dict[str, list[str]] = {}
+ for metadata_key, artifact in metadata.items():
+ if not isinstance(artifact, Mapping):
+ continue
+ block_label = str(artifact.get("block_label") or metadata_key).strip()
+ if not block_label:
+ continue
+ for path in _artifact_contract_paths(artifact):
+ metadata_labels_by_path.setdefault(path, []).append(block_label)
+
+ registered_rows = _registered_output_parameter_payloads(run_data)
+ candidates_by_slot_id: dict[str, list[RegisteredBlockerEvidence]] = {}
+ identity_owners: dict[tuple[str, str, str], list[str]] = {}
+ for criterion in blocker_criteria:
+ slot_id = criterion.request_slot_id
+ output_path = criterion.output_path or criterion.floor_rekeyed_from_path
+ if slot_id is None or not isinstance(output_path, str) or len(criterion_paths.get(output_path, ())) != 1:
+ continue
+ metadata_labels = metadata_labels_by_path.get(output_path.removeprefix("output."), ())
+ if len(metadata_labels) != 1:
+ continue
+ metadata_label = metadata_labels[0]
+ candidates: list[RegisteredBlockerEvidence] = []
+ for registered in registered_rows:
+ registered_block_label = registered.get("block_label")
+ registered_output_key = registered.get("output_parameter_key")
+ if (
+ not isinstance(registered_block_label, str)
+ or registered_block_label != metadata_label
+ or not isinstance(registered_output_key, str)
+ or not registered_output_key
+ ):
+ continue
+ found, value = _exact_registered_output_path_value(registered.get("value"), output_path)
+ if not found:
+ continue
+ registered_output_id = registered.get("output_parameter_id")
+ candidates.append(
+ RegisteredBlockerEvidence(
+ block_label=registered_block_label,
+ output_path=output_path,
+ registered_output_key=registered_output_key,
+ registered_output_id=(registered_output_id if isinstance(registered_output_id, str) else None),
+ value=value,
+ )
+ )
+ identity_owners.setdefault((registered_block_label, registered_output_key, output_path), []).append(slot_id)
+ candidates_by_slot_id[slot_id] = candidates
+
+ associations: dict[str, tuple[RegisteredBlockerEvidence, ...]] = {}
+ for slot_id, candidates in candidates_by_slot_id.items():
+ if len(candidates) != 1:
+ continue
+ candidate = candidates[0]
+ identity = (candidate.block_label, candidate.registered_output_key, candidate.output_path)
+ if len(identity_owners.get(identity, ())) == 1:
+ associations[slot_id] = (candidate,)
+ return associations
+
+
def _authored_output_contract_metadata_paths(metadata: object) -> set[str]:
paths: set[str] = set()
if isinstance(metadata, Mapping):
@@ -1025,6 +1119,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
blocks = data.get("blocks")
block_outputs: dict[str, Any] = {}
block_output_sources: dict[str, EvidenceSourceKind] = {}
+ registered_output_values: dict[str, Any] = {}
if isinstance(blocks, list):
for block in blocks:
if not isinstance(block, dict):
@@ -1049,12 +1144,23 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
block_output_sources[output_key] = "registered_output_parameter"
for registered in _registered_output_parameter_payloads(data):
registered_output_key = registered.get("output_parameter_key")
+ registered_output_value = _completion_evidence_payload(registered.get("value"))
+ registered_block_label = registered.get("block_label")
+ if isinstance(registered_output_key, str) and registered_output_key:
+ registered_output_values[registered_output_key] = registered_output_value
+ if isinstance(registered_block_label, str) and registered_block_label in current_labels:
+ if isinstance(registered_output_key, str) and registered_output_key:
+ registered_existing = registered_output_values.get(registered_block_label)
+ if isinstance(registered_existing, dict):
+ registered_existing.setdefault(registered_output_key, registered_output_value)
+ else:
+ registered_output_values[registered_block_label] = {registered_output_key: registered_output_value}
+ else:
+ registered_output_values[registered_block_label] = registered_output_value
if not _is_meaningful_extracted_data(
_registered_output_payload_view(registered.get("value"), registered.get("block_type"))
):
continue
- registered_output_value = _completion_evidence_payload(registered.get("value"))
- registered_block_label = registered.get("block_label")
if isinstance(registered_output_key, str) and registered_output_key:
block_outputs[registered_output_key] = registered_output_value
block_output_sources[registered_output_key] = "registered_output_parameter"
@@ -1095,6 +1201,10 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
workflow_run_id=run_id if isinstance(run_id, str) else None,
block_outputs=block_outputs,
block_output_sources=block_output_sources,
+ registered_output_values=registered_output_values,
+ registered_blocker_evidence_by_request_slot_id=_registered_blocker_evidence_by_request_slot_id(
+ copilot_ctx, data
+ ),
current_url=_valid_runtime_anchor_url(data.get("current_url")),
page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
run_terminal_status=run_terminal_status
@@ -1382,6 +1492,12 @@ async def _maybe_run_completion_verification(
verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)
if verification is None:
return None
+ verification = replace(
+ verification,
+ registered_blocker_evidence_by_request_slot_id=dict(
+ _build_run_evidence_snapshot(copilot_ctx, result).registered_blocker_evidence_by_request_slot_id
+ ),
+ )
verification = carry_criterion_metadata(verification, criteria)
verification = _carry_degraded_ids(copilot_ctx, verification)
verification = carry_floor_rekeyed_criterion_ids(verification, criteria)
@@ -1715,12 +1831,18 @@ def _outcome_failure_warrants_repair(
if only_degraded_blocking(completion_verification):
return False
unmet_verdicts = effective_unmet_verdicts(completion_verification)
- if any(verdict.reason_code == "evidence_contradicts" for verdict in unmet_verdicts):
+ if any(
+ verdict.reason_code == "evidence_contradicts"
+ and not completion_verification.is_structural_contingent_abstention(verdict)
+ for verdict in unmet_verdicts
+ ):
return True
# Repair needs at least one affirmatively unsatisfied criterion; unknown alone
# (absent judge signal, unmappable definition checks) never warrants repair.
if not any(
- verdict.state == "unsatisfied" and not _is_structural_requested_output_abstention(verdict)
+ verdict.state == "unsatisfied"
+ and not _is_structural_requested_output_abstention(verdict)
+ and not completion_verification.is_structural_contingent_abstention(verdict)
for verdict in unmet_verdicts
):
return False
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index c1254e69e..a80476e5a 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
+from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import Any
@@ -8,7 +9,7 @@ import structlog
from sqlalchemy import select
from skyvern.config import settings
-from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json
+from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criterion_authority_projection
from skyvern.forge.sdk.copilot.context import TurnNarrativePayload
from skyvern.forge.sdk.db._error_handling import db_operation
from skyvern.forge.sdk.db._sentinels import _UNSET
@@ -76,35 +77,64 @@ from skyvern.webeye.actions.actions import Action
LOG = structlog.get_logger()
+def _floor_rekeyed_association_is_coherent(item: Mapping[str, object]) -> bool:
+ marker_present = "requested_output_floor_rekeyed" in item
+ path_present = "floor_rekeyed_from_path" in item
+ if not marker_present and not path_present:
+ return True
+ return (
+ marker_present
+ and path_present
+ and item["requested_output_floor_rekeyed"] is True
+ and isinstance(item["floor_rekeyed_from_path"], str)
+ )
+
+
def _decode_completion_criteria_set(
row: WorkflowCopilotCompletionCriteriaSetModel,
) -> WorkflowCopilotCompletionCriteriaSet | NonAdoptableCriteriaSet:
- """A v1 dict-envelope row adopts only when every recorded criterion decodes."""
+ """A current or v1 row adopts only when every recorded criterion decodes without loss."""
raw_criteria = row.criteria
try:
if isinstance(raw_criteria, list):
- return WorkflowCopilotCompletionCriteriaSet.model_validate(row)
- if (
+ current = WorkflowCopilotCompletionCriteriaSet.model_validate(row)
+ inner = current.criteria
+ is_v1_envelope = False
+ elif (
isinstance(raw_criteria, dict)
and raw_criteria.get("contract_version") == 1
and isinstance(raw_criteria.get("criteria"), list)
):
inner = raw_criteria["criteria"]
- decoded = criteria_from_json(inner)
- if (
- not inner
- or len(decoded) != len(inner)
+ current = None
+ is_v1_envelope = True
+ else:
+ inner = None
+ current = None
+ is_v1_envelope = False
+
+ decoded = criteria_from_json(inner)
+ if inner is not None and (
+ (is_v1_envelope and not inner)
+ or len(decoded) != len(inner)
+ or any(
+ not _floor_rekeyed_association_is_coherent(item)
or any(
- (isinstance(item.get("kind"), str) and item["kind"] != criterion.kind)
- or (isinstance(item.get("level"), str) and item["level"] != criterion.level)
- for item, criterion in zip(inner, decoded)
- )
- ):
- return NonAdoptableCriteriaSet(
- reason="undecodable_v1_criteria",
- completion_criteria_set_id=row.completion_criteria_set_id,
- goal_epoch=row.goal_epoch,
+ field in item and item[field] != canonical_value
+ for field, canonical_value in criterion_authority_projection(criterion).items()
)
+ or ("antecedent_family" in item and item["antecedent_family"] is None)
+ for item, criterion in zip(inner, decoded)
+ )
+ ):
+ return NonAdoptableCriteriaSet(
+ reason="undecodable_v1_criteria",
+ completion_criteria_set_id=row.completion_criteria_set_id,
+ goal_epoch=row.goal_epoch,
+ )
+ if current is not None:
+ return current
+ if is_v1_envelope:
return WorkflowCopilotCompletionCriteriaSet(
completion_criteria_set_id=row.completion_criteria_set_id,
organization_id=row.organization_id,
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 0ea489b71..43cd4198a 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -169,6 +169,7 @@ def make_completion_criterion(
output_path: str | None = None,
contingent_on: str | None = None,
contingent_antecedent_output_path: str | None = None,
+ antecedent_family: str | None = None,
kind: str = "outcome",
terminal_action_family: str | None = None,
deliverable_kind: str | None = None,
@@ -192,6 +193,7 @@ def make_completion_criterion(
output_path=output_path,
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
+ antecedent_family=antecedent_family, # type: ignore[arg-type]
kind=kind, # type: ignore[arg-type]
terminal_action_family=terminal_action_family, # type: ignore[arg-type]
deliverable_kind=deliverable_kind, # type: ignore[arg-type]
diff --git a/tests/unit/fixtures/copilot/p7_run2_zero_signal_packet.json b/tests/unit/fixtures/copilot/p7_run2_zero_signal_packet.json
new file mode 100644
index 000000000..6447e385c
--- /dev/null
+++ b/tests/unit/fixtures/copilot/p7_run2_zero_signal_packet.json
@@ -0,0 +1,119 @@
+{
+ "provenance": {
+ "artifact_root_sha256": "841e1ac5e9f3f3f415e019ba434dba48ca234ac3fb5d761f1327efdd1b227692",
+ "backend_log_sha256": "be9f4880b3e8411fb3bfaab84e09ba15e2737a3c03d71143042dfa763e8b635c",
+ "proposed_workflow_sha256": "d89d27d9e809914f745c02430f9fdf7c264670b95bd8535e04e485c2166de8a8",
+ "result_sha256": "7c2794223e422eccdba84b2f800944f9d2ad584071ca47f113f0b4b96964c123",
+ "source_lines": [4, 560, 616, 782],
+ "workflow_run_id": "wr_551768603334229250"
+ },
+ "request_slot": {
+ "latest_request": "I need a reusable workflow that starts a commercial water service request in the mock Citrus County Utilities portal at http://localhost:8900/utility_services/citrus_water/. It should use the business name, contact email, service address, and desired start date as reusable inputs. For this eval run use Example Realty Labs Inc, [email protected], 1842 Harbor Vista Dr, Orlando, FL 32801, and 2026-06-22. The workflow should submit the request when the reviewable form is complete, then return a safe result with the confirmation number, account number, selected start date, deposit amount, and next owner. If the site only exposes an email/manual-service path instead of an online form, report that as the blocker rather than inventing values.",
+ "source_quotes": [
+ "confirmation number",
+ "account number",
+ "selected start date",
+ "deposit amount",
+ "next owner",
+ "If the site only exposes an email/manual-service path instead of an online form, report that as the blocker rather than inventing values."
+ ]
+ },
+ "criterion": {
+ "outcome": "the blocker is reported if the site only exposes an email/manual-service path instead of an online form",
+ "output_path": "output.blocker"
+ },
+ "accepted_artifact_metadata": {
+ "submit_commercial_water_request": {
+ "artifact_id": "code_artifact:submit_commercial_water_request",
+ "block_label": "submit_commercial_water_request",
+ "claimed_outcomes": [
+ {
+ "covered_criteria": ["online_submission_complete"],
+ "criteria_ids": ["online_submission_complete"],
+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"submitted\":{\"type\":\"boolean\"},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"submitted\",\"evidence_text\"]}",
+ "extraction_schema_provenance": "self_authored",
+ "goal_value_paths": [
+ "confirmation_number",
+ "account_number",
+ "selected_start_date",
+ "deposit_amount",
+ "next_owner",
+ "submitted"
+ ],
+ "id": "online_submission_result",
+ "scope": "terminal",
+ "status": "observed_not_verified",
+ "text": "For the tested online-form path, the request is submitted and all five requested page-grounded result values are returned with blocker null."
+ }
+ ],
+ "completion_criteria": [
+ {
+ "id": "online_submission_complete",
+ "level": "terminal",
+ "outcome": "Online request submitted with complete result",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal": true,
+ "text": "On the tested online path, submitted is true; confirmation_number, account_number, selected_start_date, deposit_amount, and next_owner are non-empty and visibly supported; blocker is null."
+ }
+ ],
+ "declared_goal": "Submit the online commercial water request when available and safely classify the result.",
+ "terminal_verifier_expectations": [
+ {
+ "claimed_outcome_ids": ["online_submission_result"],
+ "criteria_ids": ["online_submission_complete"],
+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"submitted\":{\"type\":\"boolean\"},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"submitted\",\"evidence_text\"]}",
+ "extraction_schema_provenance": "self_authored",
+ "goal_value_paths": [
+ "confirmation_number",
+ "account_number",
+ "selected_start_date",
+ "deposit_amount",
+ "next_owner",
+ "submitted"
+ ],
+ "id": "verify_online_submission",
+ "text": "Confirm submitted is true and each requested result field appears verbatim in evidence_text; blocker must be null for this online-form run."
+ }
+ ]
+ }
+ },
+ "registered_output_row": {
+ "workflow_run_id": "wr_551768603334229250",
+ "output_parameter_id": "op_551768603334229240",
+ "output_parameter_key": "submit_commercial_water_request_output",
+ "block_label": "submit_commercial_water_request",
+ "block_type": "code",
+ "value": {
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ "deposit_amount": "$41.00 plus initiation fee",
+ "next_owner": "Provider",
+ "blocker": null,
+ "submitted": true,
+ "evidence_text": "Contact Us | Support\nCC\nCitrus County Utilities\nWater Service\nOverview\nSupport Center\nStart/Stop/Move Service\nSubmitted confirmation page is visible.\nWater Service Request Submitted\nConfirmation Number\nWTR-1842-DEMO\nService Address\n1842 Harbor Vista Dr, Orlando, FL 32801\nSelected Start Date\n2026-06-22\nAccount Number\n100245\nDeposit Amount\n$41.00 plus initiation fee\nNext Owner\nProvider\nResidential Customers\nMy Account\nStart Service\nBusiness Customers\nTurn On Service\nService Requests\nHelp & Support\nContact Us\nPayment Locations\nCompany Information\nSafety\nRates\nTerms"
+ }
+ },
+ "recorded_criteria": [
+ {"id": "c303cb0527a5020e7c6c1df0973bd8706d7860af2f59c0bd4a6483870cb8df00", "outcome": "the confirmation number is returned", "output_path": "output.confirmation_number"},
+ {"id": "1005ea901a0dece2b2d7b1831d94084d1109d055737dda722d0eb6f462fb8994", "outcome": "the account number is returned", "output_path": "output.account_number"},
+ {"id": "68e587b4e594b09b28bc70e2dd43aad4c120e577ef245bdf90bfefca1bea2385", "outcome": "the selected start date is returned", "output_path": "output.selected_start_date"},
+ {"id": "fb6a01d4803a2a54403060d691f91bc8979d9df3427dfe547cda77e411f3e5c4", "outcome": "the deposit amount is returned", "output_path": "output.deposit_amount"},
+ {"id": "5e137f7b0e047880023c409114f147c0d8dda7ab04b47ebfc15192ff905e811f", "outcome": "the next owner is returned", "output_path": "output.next_owner"}
+ ],
+ "recorded_verdicts": [
+ {"criterion_id": "c303cb0527a5020e7c6c1df0973bd8706d7860af2f59c0bd4a6483870cb8df00", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "submit_commercial_water_request_output.confirmation_number"},
+ {"criterion_id": "1005ea901a0dece2b2d7b1831d94084d1109d055737dda722d0eb6f462fb8994", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "submit_commercial_water_request_output.account_number"},
+ {"criterion_id": "68e587b4e594b09b28bc70e2dd43aad4c120e577ef245bdf90bfefca1bea2385", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "submit_commercial_water_request_output.selected_start_date"},
+ {"criterion_id": "fb6a01d4803a2a54403060d691f91bc8979d9df3427dfe547cda77e411f3e5c4", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "submit_commercial_water_request_output.deposit_amount"},
+ {"criterion_id": "5e137f7b0e047880023c409114f147c0d8dda7ab04b47ebfc15192ff905e811f", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "submit_commercial_water_request_output.next_owner"}
+ ],
+ "recorded_failure": {
+ "criterion_id": "3362dc25307750ce8cfcc05e3459f621874ba48d0a4500fbdd67f95d84cace37",
+ "reason_code": "evidence_contradicts",
+ "evidence_ref": "submit_commercial_water_request_output.blocker",
+ "missing_evidence": "produced evidence contradicted this criterion",
+ "warrants_repair": true,
+ "turn_halt_kind": "repair_ceiling_reached"
+ }
+}
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 124fdad54..d2260f7de 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -82,6 +82,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
is_fallback_floor_criterion,
redact_raw_secrets_for_prompt,
)
+from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOTS_PROMPT_NAME
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.completion import (
@@ -118,6 +119,17 @@ from tests.unit.copilot_test_helpers import make_verified_goal_contract as _veri
_HISTORY_SENTINEL_TS = datetime(2026, 1, 1, tzinfo=timezone.utc)
+def _with_empty_request_slots(handler):
+ """Keep request-policy test doubles explicit about the independent slot producer."""
+
+ async def wrapped(*, prompt: str, prompt_name: str):
+ if prompt_name == REQUEST_SLOTS_PROMPT_NAME:
+ return {"version": "1", "slots": []}
+ return await handler(prompt=prompt, prompt_name=prompt_name)
+
+ return wrapped
+
+
def test_prompt_offer_compiles_plan_and_refreshes_when_observation_changes() -> None:
criterion = CompletionCriterion(
id="record_id",
@@ -3844,11 +3856,13 @@ class TestRequestPolicyCredentialResolution:
workflow_yaml="",
chat_history=[],
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert policy.classifier_status == "success"
- assert observed_timeouts == [default_timeout]
+ # The primary policy classifier uses the configured budget. The independent
+ # request-slot producer then performs its two matching deterministic reads.
+ assert observed_timeouts == [default_timeout, 30.0, 30.0]
@pytest.mark.asyncio
async def test_request_policy_classifier_timeout_does_not_retry(self, monkeypatch) -> None:
@@ -3868,7 +3882,7 @@ class TestRequestPolicyCredentialResolution:
workflow_yaml="",
chat_history=[],
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert calls == 1
@@ -3900,7 +3914,7 @@ class TestRequestPolicyCredentialResolution:
workflow_yaml="",
chat_history=[],
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert calls == 1
@@ -3978,7 +3992,7 @@ class TestRequestPolicyCredentialResolution:
workflow_yaml="",
chat_history=[],
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert calls == 2
@@ -4293,7 +4307,7 @@ workflow_definition:
chat_history=[],
global_llm_context="",
organization_id="org-1",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert policy.raw_secret_detected is True
@@ -4390,7 +4404,7 @@ workflow_definition:
chat_history=[],
global_llm_context="",
organization_id="org-1",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
assert policy.raw_secret_detected is True
@@ -6118,7 +6132,7 @@ class TestRequestPolicyPromptRendering:
("ai", "Which saved credential should I use?"),
),
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
prompt = captured["prompt"]
@@ -6144,7 +6158,7 @@ class TestRequestPolicyPromptRendering:
workflow_yaml="",
chat_history=[],
global_llm_context="",
- handler=handler,
+ handler=_with_empty_request_slots(handler),
)
prompt = captured["prompt"]
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index cc2882859..c8923e296 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -311,6 +311,18 @@ def test_reconcile_no_criteria_anywhere_is_noop() -> None:
),
id="request-slot-contract-fields",
),
+ pytest.param(
+ (
+ CompletionCriterion(
+ id="blocker-family",
+ outcome="The blocker is reported only when manual service is required.",
+ antecedent_family="blocker",
+ request_slot_id="b" * 64,
+ pinability="shapeless_valid",
+ ),
+ ),
+ id="antecedent-family",
+ ),
],
)
def test_criteria_json_round_trip_preserves_fields(criteria: tuple[CompletionCriterion, ...]) -> None:
@@ -472,6 +484,45 @@ def test_typed_criteria_list_round_trip_preserves_grading_metadata() -> None:
assert criteria_from_json(raw) == (criterion,)
+def test_floor_rekeyed_association_round_trip_preserves_coherent_pair() -> None:
+ criterion = CompletionCriterion(
+ id="c0",
+ outcome="The blocker is reported.",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.blocker",
+ )
+
+ raw = criteria_to_json([criterion])
+
+ assert raw[0]["requested_output_floor_rekeyed"] is True
+ assert raw[0]["floor_rekeyed_from_path"] == "output.blocker"
+ assert criteria_from_json(raw) == (criterion,)
+
+
+def test_floor_rekeyed_association_legacy_omission_defaults_false() -> None:
+ (criterion,) = criteria_from_json([{"id": "c0", "outcome": "done"}])
+
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+
+
+ "stored_pair",
+ [
+ {"requested_output_floor_rekeyed": True},
+ {"floor_rekeyed_from_path": "output.blocker"},
+ {"requested_output_floor_rekeyed": False, "floor_rekeyed_from_path": "output.blocker"},
+ {"requested_output_floor_rekeyed": True, "floor_rekeyed_from_path": "blocker"},
+ {"requested_output_floor_rekeyed": "true", "floor_rekeyed_from_path": "output.blocker"},
+ ],
+)
+def test_floor_rekeyed_association_decode_fails_closed(stored_pair: dict[str, object]) -> None:
+ (criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", **stored_pair}])
+
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+
+
def test_typed_boolean_validation_classification_round_trip_preserves_shape_and_evidence() -> None:
"""Typed boolean classifications persist goal_judgment_boolean / independent_run_evidence."""
criterion = CompletionCriterion(
@@ -533,6 +584,55 @@ def test_typed_metadata_is_self_describing_without_a_storage_version() -> None:
mint_disposition="degraded",
),
)
+ assert "antecedent_family" not in criteria_to_json(criteria_from_json([legacy]))[0]
+
+
+def test_antecedent_family_is_reconciliation_identity() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_1",
+ goal_epoch=1,
+ criteria=(CompletionCriterion(id="s0", outcome="The blocker is reported.", antecedent_family="blocker"),),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [CompletionCriterion(id="f0", outcome="The blocker is reported.", antecedent_family="unconditional")],
+ actionable=True,
+ )
+
+ assert decision.action == "create"
+ assert decision.reason == "not_subset"
+
+
+def test_floor_rekeyed_association_is_reconciliation_identity() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_1",
+ goal_epoch=1,
+ criteria=(
+ CompletionCriterion(
+ id="s0",
+ outcome="The blocker is reported.",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.blocker",
+ ),
+ ),
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ [CompletionCriterion(id="f0", outcome="The blocker is reported.")],
+ actionable=True,
+ )
+
+ assert decision.action == "create"
+ assert decision.reason == "not_subset"
+
+
[email protected]("invalid_family", ["conditional", [], {"family": "blocker"}])
+def test_criteria_from_json_ignores_invalid_antecedent_family(invalid_family: object) -> None:
+ (criterion,) = criteria_from_json([{"id": "c0", "outcome": "done", "antecedent_family": invalid_family}])
+
+ assert criterion.antecedent_family is None
def test_reconciliation_identity_ignores_pinability_and_derived_mint_state() -> None:
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 798625580..682ddab06 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -1,10 +1,13 @@
from __future__ import annotations
import asyncio
+import hashlib
+import json
import textwrap
import time
from collections.abc import Awaitable, Callable
from dataclasses import replace
+from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -41,6 +44,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
EvidenceSourceKind,
FloorRekeyedDeliverableCredit,
FloorRekeyedEmissionWithhold,
+ RegisteredBlockerEvidence,
RunEvidenceSnapshot,
_coerce_result,
_structured_record_has_identifier,
@@ -100,12 +104,22 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestPolicy,
_apply_classifier_typed_requested_output_corroborators,
_apply_requested_output_completion_criteria,
+ _bind_criterion_to_request_slot,
_degrade_unbound_request_slot_criterion,
_parse_completion_criteria,
build_classifier_fallback_floor,
is_contingent_missing_antecedent_degraded,
is_turn_unsatisfiable_fallback_degraded,
)
+from skyvern.forge.sdk.copilot.request_slots import (
+ RequestSlotAntecedentFamily,
+ RequestSlotDeclarationV1,
+ RequestSlotEnvelopeV1,
+ RequestSlotPinability,
+ RequestSlotPlane,
+ RequestSlotProducerInputV1,
+ canonicalize_request_slots,
+)
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
from skyvern.forge.sdk.copilot.tools import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
@@ -2624,6 +2638,7 @@ async def test_evaluate_happy_path_returns_evaluated() -> None:
def test_snapshot_has_evidence() -> None:
assert RunEvidenceSnapshot().has_evidence() is False
+ assert RunEvidenceSnapshot(registered_output_values={"empty_output": {"blocker": None}}).has_evidence() is False
assert RunEvidenceSnapshot(run_terminal_status="failed").has_evidence() is False
assert RunEvidenceSnapshot(current_url="https://example.com").has_evidence() is True
assert RunEvidenceSnapshot(block_outputs={"a": 1}).has_evidence() is True
@@ -13456,6 +13471,536 @@ def test_real_blocker_evidence_keeps_pathless_contingent_criterion_blocking() ->
assert result.is_fully_satisfied() is False
+def test_declared_blocker_family_uses_registered_output_only() -> None:
+ criterion = replace(
+ _criterion(
+ "a" * 64,
+ "The blocker is reported.",
+ antecedent_family="blocker",
+ ),
+ request_slot_id="a" * 64,
+ )
+ incidental = RunEvidenceSnapshot(block_outputs={"runtime": {"blocker": "Incidental runtime value"}})
+ registered = RunEvidenceSnapshot(
+ block_outputs={"runtime": {"blocker": "Incidental runtime value"}},
+ registered_output_values={"submit_output": {"blocker": "Provider requires a phone call."}},
+ registered_blocker_evidence_by_request_slot_id={
+ "a" * 64: (
+ RegisteredBlockerEvidence(
+ block_label="submit",
+ output_path="output.blocker",
+ registered_output_key="submit_output",
+ registered_output_id=None,
+ value="Provider requires a phone call.",
+ ),
+ )
+ },
+ )
+
+ incidental_result = _graded_result([*_registered_output_criteria(), criterion], incidental, {criterion.id})
+ registered_result = _graded_result([*_registered_output_criteria(), criterion], registered, {criterion.id})
+
+ assert incidental_result.structural_unfired_criterion_ids == [criterion.id]
+ assert incidental_result.is_fully_satisfied() is True
+ assert registered_result.structural_unfired_criterion_ids == []
+ assert registered_result.is_fully_satisfied() is False
+
+
+def test_blocker_family_associated_abstention_does_not_waive_unassociated_real_blocker() -> None:
+ licensed = replace(
+ _criterion(
+ "a" * 64,
+ "The first blocker is reported.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id="a" * 64,
+ )
+ unassociated = replace(
+ _criterion(
+ "b" * 64,
+ "The second blocker is reported.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id="b" * 64,
+ )
+ snapshot = RunEvidenceSnapshot(
+ registered_output_values={
+ "first_output": {"blocker": None},
+ "second_output": {"blocker": "Second producer is blocked."},
+ },
+ registered_blocker_evidence_by_request_slot_id={
+ "a" * 64: (
+ RegisteredBlockerEvidence(
+ block_label="first_producer",
+ output_path="output.blocker",
+ registered_output_key="first_output",
+ registered_output_id=None,
+ value=None,
+ ),
+ )
+ },
+ )
+
+ result = _graded_result([licensed, unassociated], snapshot, {licensed.id, unassociated.id})
+
+ assert result.structural_unfired_criterion_ids == [licensed.id]
+ assert result.is_fully_satisfied() is False
+
+
+def test_snapshot_binds_blocker_evidence_to_exact_artifact_owner_and_request_slot() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "first_producer", "second_producer")
+ criteria = [
+ replace(
+ _criterion(
+ slot_id,
+ f"Blocker {index} is reported.",
+ antecedent_family="blocker",
+ output_path=f"output.{['first', 'second'][index - 1]}_blocker",
+ ),
+ request_slot_id=slot_id,
+ )
+ for index, slot_id in enumerate(("a" * 64, "b" * 64), start=1)
+ ]
+ ctx.request_policy = RequestPolicy(completion_contract_status="present", completion_criteria=criteria)
+ ctx.code_artifact_metadata = {
+ "first_producer": {
+ "block_label": "first_producer",
+ "claimed_outcomes": [{"goal_value_paths": ["first_blocker"]}],
+ "completion_criteria": [{"id": "criterion:first_blocker"}],
+ },
+ "second_producer": {
+ "block_label": "second_producer",
+ "claimed_outcomes": [{"goal_value_paths": ["second_blocker"]}],
+ "completion_criteria": [{"id": "criterion:second_blocker"}],
+ },
+ }
+ result = {
+ "data": {
+ "workflow_run_id": "wr_association",
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_association",
+ "output_parameter_key": "first_output",
+ "block_label": "first_producer",
+ "block_type": "code",
+ "value": {"first_blocker": "First producer is blocked."},
+ },
+ {
+ "workflow_run_id": "wr_association",
+ "output_parameter_key": "second_output",
+ "block_label": "second_producer",
+ "block_type": "code",
+ "value": {"second_blocker": None},
+ },
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+
+ assert snapshot.registered_blocker_evidence_by_request_slot_id["a" * 64][0].value == ("First producer is blocked.")
+ assert snapshot.registered_blocker_evidence_by_request_slot_id["b" * 64][0].value is None
+ assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == ["b" * 64]
+
+
+def test_persisted_blocker_slot_binds_artifact_local_criterion_to_exact_registered_row() -> None:
+ slot_id = "a" * 64
+ criterion = replace(
+ _criterion(
+ slot_id,
+ "The provider blocker is reported.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id=slot_id,
+ )
+ persisted = criteria_from_json(criteria_to_json([criterion]))
+ assert len(persisted) == 1
+ assert persisted[0].request_slot_id == slot_id
+ assert persisted[0].antecedent_family == "blocker"
+
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_request")
+ ctx.request_policy = RequestPolicy(completion_contract_status="present", completion_criteria=list(persisted))
+ ctx.code_artifact_metadata = {
+ "submit_request": {
+ "block_label": "submit_request",
+ "claimed_outcomes": [{"goal_value_paths": ["confirmation_number", "blocker"]}],
+ "completion_criteria": [{"id": "criterion:artifact_local_blocker"}],
+ }
+ }
+ run_result = {
+ "data": {
+ "workflow_run_id": "wr_normalized_binding",
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_normalized_binding",
+ "output_parameter_key": "submit_request_output",
+ "block_label": "submit_request",
+ "block_type": "code",
+ "value": {"confirmation_number": "WTR-123", "blocker": "Provider requires a phone call."},
+ }
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, run_result)
+ assert snapshot.registered_blocker_evidence_by_request_slot_id[slot_id][0].value == (
+ "Provider requires a phone call."
+ )
+ assert structural_unfired_contingent_criterion_ids(list(persisted), snapshot) == []
+
+ association_absent = replace(snapshot, registered_blocker_evidence_by_request_slot_id={})
+ result = _graded_result(
+ [*_registered_output_criteria(), *persisted],
+ association_absent,
+ {persisted[0].id},
+ )
+ assert result.structural_unfired_criterion_ids == []
+ assert result.is_fully_satisfied() is False
+
+
+def test_snapshot_rejects_ambiguous_artifact_owner_for_blocker_request_slot() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "first_producer", "second_producer")
+ criterion = replace(
+ _criterion(
+ "a" * 64,
+ "The blocker is reported.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id="a" * 64,
+ )
+ ctx.request_policy = RequestPolicy(completion_contract_status="present", completion_criteria=[criterion])
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["blocker"]}],
+ "completion_criteria": [{"id": f"criterion:{label}_blocker"}],
+ }
+ for label in ("first_producer", "second_producer")
+ }
+ result = {
+ "data": {
+ "workflow_run_id": "wr_ambiguous",
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_ambiguous",
+ "output_parameter_key": f"{label}_output",
+ "block_label": label,
+ "block_type": "code",
+ "value": {"blocker": "A blocker exists."},
+ }
+ for label in ("first_producer", "second_producer")
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+
+ assert snapshot.registered_blocker_evidence_by_request_slot_id == {}
+ assert structural_unfired_contingent_criterion_ids([criterion], snapshot) == []
+
+
+def test_snapshot_does_not_cross_credit_unconditional_blocker_metadata_owner_to_blocker_slot() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "shared_producer")
+ unconditional = replace(
+ _criterion(
+ "a" * 64,
+ "The blocker-shaped output is returned unconditionally.",
+ antecedent_family="unconditional",
+ output_path="output.blocker",
+ ),
+ request_slot_id="a" * 64,
+ )
+ blocker = replace(
+ _criterion(
+ "b" * 64,
+ "The blocker is reported when the contingent path fires.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id="b" * 64,
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_contract_status="present",
+ completion_criteria=[unconditional, blocker],
+ )
+ ctx.code_artifact_metadata = {
+ "shared_producer": {
+ "block_label": "shared_producer",
+ "claimed_outcomes": [{"goal_value_paths": ["blocker"]}],
+ "completion_criteria": [{"id": "criterion:unconditional_blocker"}],
+ }
+ }
+ result = {
+ "data": {
+ "workflow_run_id": "wr_unconditional_owner",
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_unconditional_owner",
+ "output_parameter_key": "shared_output",
+ "block_label": "shared_producer",
+ "block_type": "code",
+ "value": {"blocker": "The unconditional producer emitted this value."},
+ }
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+
+ assert snapshot.registered_blocker_evidence_by_request_slot_id == {}
+ assert structural_unfired_contingent_criterion_ids([unconditional, blocker], snapshot) == []
+
+
+def test_structural_blocker_abstention_trace_never_emits_contradiction_semantics() -> None:
+ criterion = replace(
+ _criterion(
+ "a" * 64,
+ "The blocker is reported.",
+ antecedent_family="blocker",
+ output_path="output.blocker",
+ ),
+ request_slot_id="a" * 64,
+ )
+ result = _graded_result([*_registered_output_criteria(), criterion], RunEvidenceSnapshot(), {criterion.id})
+
+ trace = result.to_trace_data()
+
+ assert trace["verdict_5_reason_code"] == "structurally_abstained"
+ assert trace["verdict_5_state"] == "unknown"
+ assert trace["verdict_5_satisfied"] is False
+ assert trace["verdict_5_abstention_reason_code"] == "unfired_contingent_antecedent"
+ assert trace["satisfied_count"] == 5
+ assert trace["unsatisfied_count"] == 0
+ assert trace["unknown_count"] == 1
+ assert trace["reason_codes"] == [*(["evidence_confirms"] * 5), "structurally_abstained"]
+ assert result.verdict_state_counts() == {"satisfied": 5, "unsatisfied": 0, "unknown": 1}
+
+
+def test_explicit_unconditional_blocker_shaped_criterion_remains_gradeable() -> None:
+ criterion = _criterion(
+ "c_unconditional",
+ "The blocker field is always returned.",
+ output_path="output.blocker",
+ antecedent_family="unconditional",
+ )
+ snapshot = RunEvidenceSnapshot(registered_output_values={"submit_output": {"blocker": None}})
+
+ result = _graded_result([criterion], snapshot, {criterion.id})
+
+ assert gradeable_completion_criteria([criterion]) == [criterion]
+ assert result.structural_unfired_criterion_ids == []
+ assert result.is_fully_satisfied() is False
+
+
+async def test_p7_run2_zero_signal_packet_replays_to_typed_abstention() -> None:
+ fixture_path = Path(__file__).parent / "fixtures/copilot/p7_run2_zero_signal_packet.json"
+ fixture_bytes = fixture_path.read_bytes()
+ assert (
+ hashlib.sha256(fixture_bytes).hexdigest() == "ed29e5de80532766c141a96f67603913bbb087d8626e101fb49e48275cd5f26b"
+ )
+ fixture = json.loads(fixture_bytes)
+ assert fixture["provenance"] == {
+ "artifact_root_sha256": "841e1ac5e9f3f3f415e019ba434dba48ca234ac3fb5d761f1327efdd1b227692",
+ "backend_log_sha256": "be9f4880b3e8411fb3bfaab84e09ba15e2737a3c03d71143042dfa763e8b635c",
+ "proposed_workflow_sha256": "d89d27d9e809914f745c02430f9fdf7c264670b95bd8535e04e485c2166de8a8",
+ "result_sha256": "7c2794223e422eccdba84b2f800944f9d2ad584071ca47f113f0b4b96964c123",
+ "source_lines": [4, 560, 616, 782],
+ "workflow_run_id": "wr_551768603334229250",
+ }
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request=fixture["request_slot"]["latest_request"],
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=tuple(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=(
+ RequestSlotAntecedentFamily.BLOCKER
+ if index == len(fixture["request_slot"]["source_quotes"]) - 1
+ else RequestSlotAntecedentFamily.UNCONDITIONAL
+ ),
+ )
+ for index, source_quote in enumerate(fixture["request_slot"]["source_quotes"])
+ ),
+ ),
+ )
+ assert [slot.slot_id for slot in contract.slots] == [
+ *[item["id"] for item in fixture["recorded_criteria"]],
+ fixture["recorded_failure"]["criterion_id"],
+ ]
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(id=fixture["recorded_failure"]["criterion_id"], **fixture["criterion"]),
+ slot=contract.slots[-1],
+ source_quote=fixture["request_slot"]["source_quotes"][-1],
+ )
+ criteria = [
+ _bind_criterion_to_request_slot(
+ CompletionCriterion(**item),
+ slot=slot,
+ source_quote=source_quote,
+ )
+ for item, slot, source_quote in zip(
+ fixture["recorded_criteria"],
+ contract.slots[:-1],
+ fixture["request_slot"]["source_quotes"][:-1],
+ strict=True,
+ )
+ ] + [criterion]
+ persisted_criteria = criteria_from_json(criteria_to_json(criteria))
+ assert persisted_criteria[-1].request_slot_id == contract.slots[-1].slot_id
+ assert persisted_criteria[-1].antecedent_family == "blocker"
+ assert persisted_criteria[-1].requested_output_floor_rekeyed is True
+ assert persisted_criteria[-1].floor_rekeyed_from_path == fixture["criterion"]["output_path"]
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, fixture["registered_output_row"]["block_label"])
+ ctx.request_policy = RequestPolicy(
+ completion_contract_status="present", completion_criteria=list(persisted_criteria)
+ )
+ ctx.code_artifact_metadata = fixture["accepted_artifact_metadata"]
+ snapshot = _build_run_evidence_snapshot(
+ ctx,
+ {
+ "data": {
+ "workflow_run_id": fixture["provenance"]["workflow_run_id"],
+ "workflow_run_output_parameters": [fixture["registered_output_row"]],
+ }
+ },
+ )
+ blocker_association = snapshot.registered_blocker_evidence_by_request_slot_id[criterion.id]
+ assert len(blocker_association) == 1
+ assert blocker_association[0].output_path == fixture["criterion"]["output_path"]
+
+ async def recorded_verifier(**_: object) -> dict[str, object]:
+ blocker_verdict = {
+ "criterion_id": criterion.id,
+ "satisfied": False,
+ "reason_code": fixture["recorded_failure"]["reason_code"],
+ "evidence_ref": fixture["recorded_failure"]["evidence_ref"],
+ "missing_evidence": fixture["recorded_failure"]["missing_evidence"],
+ }
+ return {"verdicts": [*fixture["recorded_verdicts"], blocker_verdict]}
+
+ for _ in range(10):
+ result = carry_degraded_criterion_ids(
+ await evaluate_completion_criteria(persisted_criteria, snapshot, recorded_verifier), persisted_criteria
+ )
+ trace = result.to_trace_data()
+
+ assert result.structural_unfired_criterion_ids == [criterion.id]
+ assert result.is_fully_satisfied() is True
+ assert criterion.id not in trace["unmet_criterion_ids"]
+ assert trace["antecedent_family_by_criterion_id"][criterion.id] == "blocker"
+ assert trace["verdict_5_antecedent_family"] == "blocker"
+ assert trace["verdict_5_structural_unfired"] is True
+ assert trace["verdict_5_reason_code"] == "structurally_abstained"
+ assert trace["verdict_5_abstention_reason_code"] == "unfired_contingent_antecedent"
+ assert trace["verdict_5_antecedent_evidence_source"] == "registered_output_parameter"
+ assert trace["verdict_5_antecedent_producer_block_label"] == fixture["registered_output_row"]["block_label"]
+ assert (
+ trace["verdict_5_antecedent_registered_output_key"]
+ == (fixture["registered_output_row"]["output_parameter_key"])
+ )
+ assert trace["verdict_5_antecedent_output_path"] == fixture["criterion"]["output_path"]
+ assert (
+ trace["verdict_5_antecedent_registered_output_id"]
+ == fixture["registered_output_row"]["output_parameter_id"]
+ )
+ assert trace["unsatisfied_count"] == 0
+ assert trace["unknown_count"] == 1
+ assert "evidence_contradicts" not in trace["reason_codes"]
+ assert result.verdict_state_counts() == {"satisfied": 5, "unsatisfied": 0, "unknown": 1}
+ assert _outcome_failure_warrants_repair(_run_ctx(), result) is False
+
+
+def test_run_evidence_snapshot_preserves_registered_null_for_family_routing() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_commercial_water_request")
+ result = {
+ "data": {
+ "workflow_run_id": "wr_family",
+ "blocks": [
+ {
+ "label": "submit_commercial_water_request",
+ "block_type": "code",
+ "extracted_data": {"blocker": "incidental runtime value"},
+ }
+ ],
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_family",
+ "output_parameter_key": "submit_commercial_water_request_output",
+ "block_label": "submit_commercial_water_request",
+ "block_type": "code",
+ "value": {"blocker": None, "submitted": True},
+ }
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+
+ assert snapshot.registered_output_values["submit_commercial_water_request_output"] == {
+ "blocker": None,
+ "submitted": True,
+ }
+ assert snapshot.block_outputs["submit_commercial_water_request"]["blocker"] == "incidental runtime value"
+
+
+def test_runtime_output_named_like_registered_output_cannot_fire_blocker_family() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "submit_commercial_water_request")
+ result = {
+ "data": {
+ "workflow_run_id": "wr_family",
+ "blocks": [
+ {
+ "label": "submit_commercial_water_request",
+ "block_type": "code",
+ "extracted_data": {
+ "submit_commercial_water_request_output": {
+ "blocker": "Incidental runtime value",
+ }
+ },
+ }
+ ],
+ }
+ }
+ criterion = _criterion(
+ "c_family",
+ "The blocker is reported.",
+ antecedent_family="blocker",
+ )
+
+ snapshot = _build_run_evidence_snapshot(ctx, result)
+ verification = _graded_result([*_registered_output_criteria(), criterion], snapshot, {criterion.id})
+
+ assert snapshot.registered_output_values == {}
+ assert verification.structural_unfired_criterion_ids == [criterion.id]
+ assert verification.is_fully_satisfied() is True
+
+
def test_missing_blocker_family_output_abstains_pathless_contingent_criterion() -> None:
contingent = _blocker_contingent_criterion()
criteria = [*_registered_output_criteria(), contingent]
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 31d47c38d..51dbc5393 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -43,6 +43,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
)
from skyvern.forge.sdk.copilot.request_slots import (
CanonicalRequestSlotV1,
+ RequestSlotAntecedentFamily,
RequestSlotPinability,
RequestSlotPlane,
)
@@ -2071,6 +2072,7 @@ def test_canonically_bound_shapeless_valid_criterion_stays_dispatchable() -> Non
source_end=10,
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
)
bound = _bind_criterion_to_request_slot(
CompletionCriterion(
diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.py
index a439a50d9..250c3bf58 100644
--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py
+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py
@@ -22,6 +22,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
)
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_PROMPT_NAME
from skyvern.forge.sdk.copilot.request_slots import (
+ RequestSlotAntecedentFamily,
RequestSlotDeclarationV1,
RequestSlotEnvelopeV1,
RequestSlotPinability,
@@ -87,6 +88,7 @@ def _anchor_only_envelope(
source_quote=source_quote,
plane=RequestSlotPlane.RUN,
pinability=pinability,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
)
for source_quote in ("confirmation number", "account number", "selected start date", "deposit amount")
),
@@ -103,6 +105,45 @@ _EXAMPLE_WATER_SERVICE_SLOT_OUTCOMES = (
)
+async def test_independent_request_slot_producer_recovers_primary_classifier_omission() -> None:
+ source_quote = "email/manual-service path"
+ primary = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": [],
+ }
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.BLOCKER,
+ ),
+ ),
+ )
+ calls: list[str] = []
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ calls.append(prompt_name)
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return envelope.model_dump(mode="json")
+ return primary
+
+ policy = await _classify_request(EXAMPLE_WATER_SERVICE_REQUEST, "", [], "", handler)
+
+ assert calls == ["workflow-copilot-request-policy", REQUEST_SLOT_PROMPT_NAME, REQUEST_SLOT_PROMPT_NAME]
+ assert len(policy.completion_criteria) == 1
+ criterion = policy.completion_criteria[0]
+ assert criterion.outcome == source_quote
+ assert criterion.antecedent_family == "blocker"
+ assert criterion.request_slot_id is not None
+
+
def _synthetic_eight_row_anchor_payload(*, valid_anchors: bool) -> dict[str, Any]:
# The live classifier packet was not retained. This labeled reconstruction mirrors
# the observed eight-row decision and six degraded request-slot claims without
@@ -176,6 +217,7 @@ def _example_water_service_envelope() -> RequestSlotEnvelopeV1:
source_quote=source_quote,
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
)
for _outcome, source_quote in _EXAMPLE_WATER_SERVICE_SLOT_OUTCOMES
),
@@ -204,18 +246,21 @@ def _envelope() -> RequestSlotEnvelopeV1:
source_quote="completion status",
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.PINNED,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
RequestSlotDeclarationV1(
source_id="u0",
source_quote="the visible path label",
plane=RequestSlotPlane.DEFINITION,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
RequestSlotDeclarationV1(
source_id="u0",
source_quote="whether the path is login-only",
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.UNPINNABLE,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
),
)
@@ -280,6 +325,11 @@ def test_fresh_contract_consumes_declared_pinability_without_guessed_polarity()
"shapeless_valid",
"unpinnable",
]
+ assert [criterion.antecedent_family for criterion in policy.completion_criteria] == [
+ "unconditional",
+ "unconditional",
+ "unconditional",
+ ]
pinned, shapeless, unpinnable = policy.completion_criteria
assert pinned.expected_output_value == "complete"
@@ -382,6 +432,72 @@ def test_producer_only_slots_are_authoritative_contract_members() -> None:
]
+ ("request_text", "source_quote", "antecedent_family"),
+ [
+ (
+ "Submit online. If the site only exposes a manual path, report that as the blocker.",
+ "report that as the blocker",
+ RequestSlotAntecedentFamily.BLOCKER,
+ ),
+ (
+ "Always return the blocker field as part of the audit record.",
+ "blocker field",
+ RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ],
+)
+async def test_independent_producer_binds_family_when_primary_classifier_emits_zero_contingent_signal(
+ request_text: str,
+ source_quote: str,
+ antecedent_family: RequestSlotAntecedentFamily,
+) -> None:
+ calls: list[str] = []
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ calls.append(prompt_name)
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return {
+ "version": "1",
+ "slots": [
+ {
+ "source_id": "u0",
+ "source_quote": source_quote,
+ "plane": "run",
+ "pinability": "shapeless_valid",
+ "antecedent_family": antecedent_family.value,
+ }
+ ],
+ }
+ return {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": [
+ {
+ "outcome": "The blocker is reported.",
+ "output_path": "output.blocker",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ }
+ ],
+ }
+
+ policy = await _classify_request(request_text, "", [], "", handler)
+
+ criterion = policy.completion_criteria[0]
+ trace = policy.to_trace_data()
+ assert criterion.contingent_on is None
+ assert criterion.contingent_antecedent_output_path is None
+ assert criterion.mint_degrade is None
+ assert criterion.antecedent_family == antecedent_family.value
+ assert trace["antecedent_family_criterion_0_id"] == criterion.id
+ assert trace["antecedent_family_criterion_0_antecedent_family"] == antecedent_family.value
+ assert calls.count("workflow-copilot-request-policy") == 1
+ assert calls.count(REQUEST_SLOT_PROMPT_NAME) == 2
+
+
def test_duplicate_classifier_bindings_keep_one_criterion_per_producer_slot() -> None:
request = _request()
contract = canonicalize_request_slots(request=request, envelope=_envelope())
@@ -469,6 +585,7 @@ def test_unbound_typed_criteria_fail_safe_without_entering_legacy_minting() -> N
assert all(criterion.expected_output_value is None for criterion in policy.completion_criteria)
assert all(criterion.expected_classification is None for criterion in policy.completion_criteria)
assert all(criterion.mint_degrade == "undecidable_judgment" for criterion in policy.completion_criteria)
+ assert all(criterion.antecedent_family == "undecidable" for criterion in policy.completion_criteria)
def test_fresh_slot_failure_preserves_non_output_outcome() -> None:
@@ -536,11 +653,13 @@ async def test_request_slot_producer_failure_degrades_without_legacy_guessing()
@pytest.mark.asyncio
-async def test_classifier_payload_without_request_slots_does_not_invoke_request_slot_producer() -> None:
+async def test_classifier_payload_without_request_slots_requires_agreed_empty_producer_contract() -> None:
calls: list[str] = []
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return RequestSlotEnvelopeV1(version="1", slots=()).model_dump(mode="json")
return {
"testing_intent": "require_test",
"credential_input_kind": "none",
@@ -550,7 +669,7 @@ async def test_classifier_payload_without_request_slots_does_not_invoke_request_
await _classify_request("Build the workflow.", "", [], "", handler)
- assert calls == ["workflow-copilot-request-policy"]
+ assert calls == ["workflow-copilot-request-policy", REQUEST_SLOT_PROMPT_NAME, REQUEST_SLOT_PROMPT_NAME]
@pytest.mark.asyncio
@@ -1016,6 +1135,7 @@ async def test_unanchored_correction_tied_to_its_structured_datum_mints_decidabl
source_quote="confirmation code",
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
),
)
@@ -1064,6 +1184,7 @@ async def test_source_valid_wrong_datum_anchor_gets_one_correction_and_fails_clo
source_quote="account number",
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
),
)
@@ -1116,6 +1237,7 @@ async def test_anchor_correction_preserves_original_criterion_datum_quote() -> N
source_quote="confirmation code",
plane=RequestSlotPlane.RUN,
pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
),
),
)
@@ -1253,11 +1375,13 @@ async def test_anchor_correction_still_requires_slot_producer_agreement() -> Non
@pytest.mark.asyncio
-async def test_fresh_non_output_request_does_not_invoke_request_slot_producer() -> None:
+async def test_fresh_non_output_request_accepts_agreed_empty_request_slot_contract() -> None:
calls: list[str] = []
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return RequestSlotEnvelopeV1(version="1", slots=()).model_dump(mode="json")
return {
"testing_intent": "require_test",
"credential_input_kind": "none",
@@ -1270,4 +1394,4 @@ async def test_fresh_non_output_request_does_not_invoke_request_slot_producer()
assert policy.request_slot_failure_kind is None
assert policy.completion_criteria[0].outcome == "The application is submitted."
assert policy.completion_criteria[0].mint_degrade is None
- assert calls == ["workflow-copilot-request-policy"]
+ assert calls == ["workflow-copilot-request-policy", REQUEST_SLOT_PROMPT_NAME, REQUEST_SLOT_PROMPT_NAME]
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 4ffacfde5..b59e32e3a 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -21,6 +21,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
is_fallback_floor_criterion,
redact_raw_secrets_for_prompt,
)
+from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_PROMPT_NAME
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotChatHistoryMessage,
WorkflowCopilotChatSender,
@@ -566,6 +567,8 @@ class TestMalformedCredentialIdExtraction:
def _capture_handler(captured: dict[str, str]):
async def handler(prompt: str, prompt_name: str) -> dict[str, object]:
+ if prompt_name == REQUEST_SLOT_PROMPT_NAME:
+ return {"version": "1", "slots": []}
captured["prompt"] = prompt
return {"testing_intent": "unspecified", "credential_input_kind": "none"}
diff --git a/tests/unit/test_copilot_request_slots.py b/tests/unit/test_copilot_request_slots.py
index bdc5268b5..531ff5621 100644
--- a/tests/unit/test_copilot_request_slots.py
+++ b/tests/unit/test_copilot_request_slots.py
@@ -10,6 +10,7 @@ from skyvern.forge.sdk.api.llm.exceptions import EmptyLLMResponseError, InvalidL
from skyvern.forge.sdk.copilot import request_slots as request_slots_module
from skyvern.forge.sdk.copilot.request_slots import (
CanonicalRequestSlotV1,
+ RequestSlotAntecedentFamily,
RequestSlotContractV1,
RequestSlotDeclarationV1,
RequestSlotEnvelopeV1,
@@ -60,6 +61,7 @@ def _response(
"source_quote": quote,
"plane": plane,
"pinability": pinability,
+ "antecedent_family": "unconditional",
}
for quote, plane, pinability in slots
],
@@ -73,9 +75,16 @@ def _envelope(
return RequestSlotEnvelopeV1.model_validate_json(json.dumps(_response(request, *slots)))
-def _identity(contract: RequestSlotContractV1) -> tuple[tuple[str, str, str, str], ...]:
+def _identity(contract: RequestSlotContractV1) -> tuple[tuple[str, str, str, str, str], ...]:
return tuple(
- (slot.slot_id, slot.canonical_path, slot.plane.value, slot.pinability.value) for slot in contract.slots
+ (
+ slot.slot_id,
+ slot.canonical_path,
+ slot.plane.value,
+ slot.pinability.value,
+ slot.antecedent_family.value,
+ )
+ for slot in contract.slots
)
@@ -91,6 +100,7 @@ def test_request_slot_models_are_strict_bounded_and_model_cannot_name_paths() ->
"source_quote": "public form",
"plane": "run",
"pinability": "shapeless_valid",
+ "antecedent_family": "unconditional",
}
with pytest.raises(ValidationError):
@@ -103,6 +113,10 @@ def test_request_slot_models_are_strict_bounded_and_model_cannot_name_paths() ->
RequestSlotDeclarationV1.model_validate(
{key: value for key, value in valid_declaration.items() if key != "pinability"}
)
+ with pytest.raises(ValidationError):
+ RequestSlotDeclarationV1.model_validate(
+ {key: value for key, value in valid_declaration.items() if key != "antecedent_family"}
+ )
with pytest.raises(ValidationError):
RequestSlotDeclarationV1.model_validate({**valid_declaration, "source_id": "latest_request"})
with pytest.raises(ValidationError):
@@ -112,9 +126,11 @@ def test_request_slot_models_are_strict_bounded_and_model_cannot_name_paths() ->
with pytest.raises(ValidationError):
RequestSlotDeclarationV1.model_validate({**valid_declaration, "pinability": "maybe"})
with pytest.raises(ValidationError):
- RequestSlotSourceV1.model_validate({"source_id": "u1", "order": 0, "text": "Return status."})
+ RequestSlotDeclarationV1.model_validate({**valid_declaration, "antecedent_family": "conditional"})
with pytest.raises(ValidationError):
- RequestSlotEnvelopeV1.model_validate({"version": "1", "slots": []})
+ RequestSlotSourceV1.model_validate({"source_id": "u1", "order": 0, "text": "Return status."})
+ empty = RequestSlotEnvelopeV1.model_validate_json('{"version":"1","slots":[]}')
+ assert empty.slots == ()
with pytest.raises(ValidationError):
RequestSlotProducerResult.model_validate({"status": "success", "attempts": 1})
@@ -219,6 +235,7 @@ def test_canonicalization_derives_server_owned_paths_ids_count_and_source_spans(
RequestSlotPinability.PINNED,
RequestSlotPinability.SHAPELESS_VALID,
]
+ assert all(slot.antecedent_family == RequestSlotAntecedentFamily.UNCONDITIONAL for slot in contract.slots)
assert _anchor_texts(request, contract) == ["public form status", "recommended next action"]
assert contract == RequestSlotContractV1.model_validate(contract.model_dump())
@@ -248,6 +265,7 @@ def test_canonicalization_rejects_missing_ambiguous_and_overlapping_anchors(
"source_quote": quote,
"plane": "run",
"pinability": "shapeless_valid",
+ "antecedent_family": "unconditional",
}
for quote in quotes
],
@@ -272,6 +290,7 @@ def test_canonicalization_rejects_unknown_source_and_context_only_anchor() -> No
"source_quote": source_quote,
"plane": "run",
"pinability": "shapeless_valid",
+ "antecedent_family": "unconditional",
}
],
}
@@ -339,6 +358,43 @@ def test_contract_agreement_detects_membership_that_opaque_identity_does_not_enc
assert not request_slot_contracts_agree(alpha, beta)
+def test_contract_agreement_requires_antecedent_family_consensus() -> None:
+ request = _input(latest_request="Report the blocker if online submission is unavailable.")
+ unconditional_envelope = _envelope(request, ("blocker", "run", "shapeless_valid"))
+ blocker_envelope = unconditional_envelope.model_copy(
+ update={
+ "slots": (
+ unconditional_envelope.slots[0].model_copy(
+ update={"antecedent_family": RequestSlotAntecedentFamily.BLOCKER}
+ ),
+ )
+ }
+ )
+ unconditional = canonicalize_request_slots(request=request, envelope=unconditional_envelope)
+ blocker = canonicalize_request_slots(request=request, envelope=blocker_envelope)
+
+ assert unconditional.slots[0].slot_id == blocker.slots[0].slot_id
+ assert not request_slot_contracts_agree(unconditional, blocker)
+
+
+async def test_producer_accepts_two_agreeing_empty_contracts() -> None:
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal calls
+ calls += 1
+ return {"version": "1", "slots": []}
+
+ result = await produce_request_slots(request=_input(latest_request="Submit the form."), handler=handler)
+
+ assert result.status == "success"
+ assert result.attempts == 2
+ assert result.contract is not None
+ assert result.contract.count == 0
+ assert calls == 2
+
+
def test_pinability_is_not_identity_but_plane_is() -> None:
request = _input(latest_request="Return status.")
pinned = canonicalize_request_slots(
diff --git a/tests/unit/test_workflow_copilot_completion_criteria_schema.py b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
index f53ddedc0..958be5ce8 100644
--- a/tests/unit/test_workflow_copilot_completion_criteria_schema.py
+++ b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
@@ -100,6 +100,104 @@ async def test_loader_adopts_current_bare_list_row_unchanged() -> None:
assert loaded.criteria == criteria
+async def test_loader_adopts_current_bare_list_with_known_antecedent_family() -> None:
+ criteria = [{"id": "c0", "outcome": "done", "antecedent_family": "blocker"}]
+
+ loaded = await _load_latest(_make_row(criteria=criteria))
+
+ assert isinstance(loaded, WorkflowCopilotCompletionCriteriaSet)
+ assert loaded.criteria == criteria
+
+
[email protected]("enveloped", [False, True], ids=["bare_list", "v1_envelope"])
+async def test_loader_adopts_coherent_floor_rekeyed_association(enveloped: bool) -> None:
+ criteria = [
+ {
+ "id": "c0",
+ "outcome": "done",
+ "requested_output_floor_rekeyed": True,
+ "floor_rekeyed_from_path": "output.blocker",
+ }
+ ]
+
+ stored: object = {"contract_version": 1, "criteria": criteria} if enveloped else criteria
+ loaded = await _load_latest(_make_row(criteria=stored))
+
+ assert isinstance(loaded, WorkflowCopilotCompletionCriteriaSet)
+ assert loaded.criteria == criteria
+
+
[email protected]("enveloped", [False, True], ids=["bare_list", "v1_envelope"])
+ "stored_pair",
+ [
+ {"requested_output_floor_rekeyed": False},
+ {"requested_output_floor_rekeyed": None},
+ {"requested_output_floor_rekeyed": True},
+ {"floor_rekeyed_from_path": None},
+ {"floor_rekeyed_from_path": "output.blocker"},
+ {"requested_output_floor_rekeyed": False, "floor_rekeyed_from_path": None},
+ {"requested_output_floor_rekeyed": False, "floor_rekeyed_from_path": "output.blocker"},
+ {"requested_output_floor_rekeyed": True, "floor_rekeyed_from_path": None},
+ {"requested_output_floor_rekeyed": True, "floor_rekeyed_from_path": "blocker"},
+ ],
+)
+async def test_loader_rejects_malformed_floor_rekeyed_association(
+ enveloped: bool,
+ stored_pair: dict[str, object],
+) -> None:
+ criteria = [{"id": "c0", "outcome": "done", **stored_pair}]
+
+ stored: object = {"contract_version": 1, "criteria": criteria} if enveloped else criteria
+ loaded = await _load_latest(_make_row(criteria=stored))
+
+ assert isinstance(loaded, NonAdoptableCriteriaSet)
+ assert loaded.reason == "undecodable_v1_criteria"
+
+
+async def test_loader_rejects_current_bare_list_with_unknown_antecedent_family() -> None:
+ criteria = [{"id": "c0", "outcome": "done", "antecedent_family": "speculative_future_family"}]
+
+ loaded = await _load_latest(_make_row(criteria=criteria))
+
+ assert isinstance(loaded, NonAdoptableCriteriaSet)
+ assert loaded.reason == "undecodable_v1_criteria"
+
+
[email protected]("enveloped", [False, True], ids=["bare_list", "v1_envelope"])
+ ("field", "value"),
+ [
+ ("antecedent_family", None),
+ ("requested_output_evidence_source", "future_evidence_source"),
+ ("mint_disposition", "future_mint_disposition"),
+ ("terminal_action_verification_mode", "future_verification_mode"),
+ ],
+)
+async def test_loader_rejects_present_authority_field_that_decode_would_normalize(
+ enveloped: bool,
+ field: str,
+ value: object,
+) -> None:
+ criterion: dict[str, object] = {
+ "id": "c0",
+ "outcome": "done",
+ "kind": "terminal_action",
+ field: value,
+ }
+ criteria: object = {"contract_version": 1, "criteria": [criterion]} if enveloped else [criterion]
+
+ loaded = await _load_latest(_make_row(criteria=criteria))
+
+ assert isinstance(loaded, NonAdoptableCriteriaSet)
+ assert loaded.reason == "undecodable_v1_criteria"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"criteria_value",
@@ -144,6 +242,7 @@ async def test_loader_rejects_v1_envelope_whose_criteria_do_not_fully_decode(inn
[
{"id": "c0", "outcome": "done", "kind": "speculative_future_kind"},
{"id": "c0", "outcome": "done", "level": "epic"},
+ {"id": "c0", "outcome": "done", "antecedent_family": "speculative_future_family"},
],
)
async def test_loader_rejects_v1_envelope_whose_enum_value_was_coerced(criterion: dict[str, Any]) -> None:
@@ -153,6 +252,16 @@ async def test_loader_rejects_v1_envelope_whose_enum_value_was_coerced(criterion
assert loaded.reason == "undecodable_v1_criteria"
+async def test_loader_adopts_v1_envelope_with_known_antecedent_family() -> None:
+ criterion = {"id": "c0", "outcome": "done", "antecedent_family": "blocker"}
+
+ loaded = await _load_latest(_make_row(criteria={"contract_version": 1, "criteria": [criterion]}))
+
+ assert isinstance(loaded, WorkflowCopilotCompletionCriteriaSet)
+ assert loaded.criteria == [criterion]
+
+
@pytest.mark.asyncio
async def test_loader_returns_highest_epoch_row_over_lower_epoch_non_adoptable(sqlite_engine: AsyncEngine) -> None:
db = BaseAlchemyDB(sqlite_engine)