Skip to content

Automate browser based workflows with AI

SKY-12163: Preserve Copilot completion checks when workflows are saved (#7455)

Andrew Neilson88fa07c13e96
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index d3f09e365..c9dc03f01 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -1,5 +1,5 @@
Classify the latest Workflow Copilot request. Return only JSON with keys:
-contract_version=1; testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
+testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.
testing_intent:
- skip_test: the user asks for an untested draft, says they will test it themselves, says draft only, just sketch it, no validation run, or skip testing.
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 66ff5c8cd..868667018 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -10,7 +10,7 @@ from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
-from typing import Any, Literal, cast
+from typing import Any, Literal, cast, get_args
from skyvern.forge.sdk.copilot.completion_output_grounding import split_requested_output_criteria
from skyvern.forge.sdk.copilot.completion_verification import (
@@ -56,8 +56,8 @@ TRIPWIRE_CONSECUTIVE_ALL_NO_EVIDENCE = 2
_CRITERION_LEVELS = ("definition", "run")
_CRITERION_KINDS = ("outcome", "terminal_action", "validation_classification")
_TERMINAL_ACTION_FAMILIES = ("request", "application", "form", "order")
-_PINABILITY_VALUES = ("pinned", "shapeless_valid", "unpinnable")
-_MINT_DISPOSITION_VALUES = ("pending", "decidable", "degraded")
+_MINT_DISPOSITIONS = frozenset(get_args(MintDisposition))
+_PINABILITIES = frozenset(get_args(Pinability))
@dataclass(frozen=True)
@@ -221,9 +221,9 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
else None
)
pinability_raw = item.get("pinability")
- pinability = pinability_raw if pinability_raw in _PINABILITY_VALUES else None
+ pinability = pinability_raw if pinability_raw in _PINABILITIES else None
mint_disposition_raw = item.get("mint_disposition")
- mint_disposition = mint_disposition_raw if mint_disposition_raw in _MINT_DISPOSITION_VALUES else "decidable"
+ mint_disposition = mint_disposition_raw if mint_disposition_raw in _MINT_DISPOSITIONS else "decidable"
stored_output_path = output_path.strip() if isinstance(output_path, str) and output_path.strip() else None
stored_expected_output_value = _coerce_expected_output_value(expected_output_value)
stored_expected_output_shape = cast(ExpectedOutputShape | None, expected_output_shape)
@@ -237,8 +237,18 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
if kind == "validation_classification":
stored_output_path = None
stored_expected_output_value = None
- stored_expected_output_shape = None
- requested_output_evidence_source = "runtime_output"
+ # Typed boolean classifications carry their own validated mint metadata. Rows without
+ # it carry no reliable shape, so retain the historical normalization for those rows.
+ is_typed_boolean_classification = (
+ stored_expected_output_shape == "goal_judgment_boolean"
+ and pinability == "pinned"
+ and mint_disposition == "pending"
+ )
+ if is_typed_boolean_classification:
+ requested_output_evidence_source = "independent_run_evidence"
+ else:
+ stored_expected_output_shape = None
+ requested_output_evidence_source = "runtime_output"
elif isinstance(stored_expected_output_value, bool) or stored_expected_output_shape == "goal_judgment_boolean":
requested_output_evidence_source = "independent_run_evidence"
criteria.append(
@@ -281,7 +291,6 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_path_key = criterion.contingent_antecedent_output_path or ""
deliverable_kind_key = (
f"{criterion.deliverable_kind or ''}\x1fdeclared:{criterion.declared_deliverable_kind or ''}"
- f"\x1fmint_degrade:{criterion.mint_degrade or ''}"
f"\x1fjudgment:{judgment_truth_condition_key(criterion.judgment_truth_condition)}"
)
expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)
@@ -480,14 +489,25 @@ def reconcile_completion_criteria(
return ReconcileDecision(action="none", reason="no_criteria", epoch=0, criteria=())
if not actionable:
return ReconcileDecision(action="none", reason="not_actionable", epoch=0, criteria=tuple(fresh))
- return ReconcileDecision(action="create", reason="first", epoch=next_epoch, criteria=tuple(fresh))
+ return ReconcileDecision(
+ action="create",
+ reason="first",
+ epoch=next_epoch,
+ criteria=tuple(fresh),
+ )
if not fresh:
return ReconcileDecision(
- action="adopt_stored", reason="empty_fresh", epoch=stored.goal_epoch, criteria=stored.criteria
+ action="adopt_stored",
+ reason="empty_fresh",
+ epoch=stored.goal_epoch,
+ criteria=stored.criteria,
)
if _outcome_key_set(fresh) <= _outcome_key_set(stored.criteria):
return ReconcileDecision(
- action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria
+ action="adopt_stored",
+ reason="kept",
+ epoch=stored.goal_epoch,
+ criteria=stored.criteria,
)
if _fresh_generic_rephrase_lacks_stored_requested_outputs(
stored.criteria,
@@ -495,11 +515,17 @@ def reconcile_completion_criteria(
requested_output_path_aliases=requested_output_path_aliases,
):
return ReconcileDecision(
- action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria
+ action="adopt_stored",
+ reason="kept",
+ epoch=stored.goal_epoch,
+ criteria=stored.criteria,
)
if not actionable:
return ReconcileDecision(
- action="adopt_stored", reason="not_actionable", epoch=stored.goal_epoch, criteria=stored.criteria
+ action="adopt_stored",
+ reason="not_actionable",
+ epoch=stored.goal_epoch,
+ criteria=stored.criteria,
)
return ReconcileDecision(
action="create",
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index ad337a091..f2bd44476 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -28,8 +28,8 @@ from skyvern.forge.sdk.copilot.request_policy import (
TerminalActionFamily,
is_contingent_missing_antecedent_degraded,
is_fallback_floor_base_criterion,
+ is_judgment_finalization_candidate,
is_turn_unsatisfiable_fallback_degraded,
- is_undecidable_judgment_degraded,
redact_raw_secrets_for_prompt,
)
from skyvern.utils.strings import escape_code_fences
@@ -2457,6 +2457,21 @@ def floor_rekeyed_deliverable_credit(
)
+def gradeable_completion_criteria(criteria: Iterable[CompletionCriterion]) -> list[CompletionCriterion]:
+ return [
+ criterion
+ for criterion in criteria
+ if criterion.mint_degrade != "undecidable_judgment"
+ and not (
+ is_judgment_finalization_candidate(criterion) and criterion.mint_disposition in {"pending", "degraded"}
+ )
+ and not (
+ criterion.requested_output_corroborator
+ and criterion.requested_output_evidence_source == "independent_run_evidence"
+ )
+ ]
+
+
def carry_degraded_criterion_ids(
result: CompletionVerificationResult,
criteria: Iterable[CompletionCriterion],
@@ -2465,7 +2480,7 @@ def carry_degraded_criterion_ids(
degraded = [
criterion.id
for criterion in criteria
- if is_turn_unsatisfiable_fallback_degraded(criterion) or is_undecidable_judgment_degraded(criterion)
+ if criterion.mint_degrade is not None and not is_contingent_missing_antecedent_degraded(criterion)
]
contingent_degraded = [
criterion.id for criterion in criteria if is_contingent_missing_antecedent_degraded(criterion)
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index a759dfbee..50e5d3719 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -2318,6 +2318,17 @@ def _active_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion,
return turn_state.decision.criteria
+def _coverage_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion, ...]:
+ criteria = list(_active_completion_criteria(ctx))
+ policy = getattr(ctx, "request_policy", None)
+ if isinstance(policy, RequestPolicy):
+ known = {(criterion.id, criterion.output_path) for criterion in criteria}
+ criteria.extend(
+ criterion for criterion in policy.completion_criteria if (criterion.id, criterion.output_path) not in known
+ )
+ return tuple(criteria)
+
+
def _pre_run_gated_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion, ...]:
"""Completion criteria whose requested output is observable before a run. A criterion whose
evidence comes from an independent run, registered output parameter, or artifact content is
@@ -2326,7 +2337,7 @@ def _pre_run_gated_completion_criteria(ctx: AgentContext) -> tuple[CompletionCri
is SKY-11591's."""
return tuple(
criterion
- for criterion in _active_completion_criteria(ctx)
+ for criterion in _coverage_completion_criteria(ctx)
if criterion.requested_output_evidence_source not in _PRE_RUN_UNGATED_EVIDENCE_SOURCES
)
@@ -2341,15 +2352,15 @@ def _requested_output_paths_for_ctx(ctx: AgentContext) -> set[str]:
for raw in repair_context.required_goal_value_paths
if isinstance(raw, str) and raw
)
- active_criteria = _active_completion_criteria(ctx)
+ coverage_criteria = _coverage_completion_criteria(ctx)
independent_run_evidence_paths = {
_canonical_output_path(criterion.output_path)
- for criterion in active_criteria
+ for criterion in coverage_criteria
if criterion.requested_output_evidence_source == "independent_run_evidence" and criterion.output_path
}
non_independent_evidence_paths = {
_canonical_output_path(criterion.output_path)
- for criterion in active_criteria
+ for criterion in coverage_criteria
if criterion.requested_output_evidence_source != "independent_run_evidence" and criterion.output_path
}
independent_only_paths = independent_run_evidence_paths - non_independent_evidence_paths
@@ -2412,9 +2423,10 @@ def _requested_output_labels_by_path(ctx: AgentContext) -> dict[str, tuple[str,
requested_paths = _requested_output_paths_for_ctx(ctx)
labels_by_path: dict[str, tuple[str, ...]] = {}
for criterion in _pre_run_gated_completion_criteria(ctx):
- if criterion.output_path in requested_paths and criterion.outcome.strip():
+ outcome = criterion.outcome.strip()
+ if criterion.output_path in requested_paths and outcome:
labels_by_path.setdefault(criterion.output_path, ())
- labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)
+ labels_by_path[criterion.output_path] += (outcome,)
return labels_by_path
@@ -2440,9 +2452,10 @@ def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutpu
return None
labels_by_path: dict[str, tuple[str, ...]] = {}
for criterion in _pre_run_gated_completion_criteria(ctx):
- if criterion.output_path in requested_paths and criterion.outcome.strip():
+ outcome = criterion.outcome.strip()
+ if criterion.output_path in requested_paths and outcome:
labels_by_path.setdefault(criterion.output_path, ())
- labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)
+ labels_by_path[criterion.output_path] += (outcome,)
if set(labels_by_path) != requested_paths:
return None
return derive_requested_output_extraction_plan(
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 6b4d30fe6..cbbcc8fff 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -323,8 +323,8 @@ class CompletionCriterion:
judgment_truth_condition: JudgmentTruthCondition | None = None
requested_output_floor_rekeyed: bool = False
floor_rekeyed_from_path: str | None = None
- # Fresh request-slot contract metadata. Defaults keep versionless criteria on
- # the byte-identical legacy path and are excluded from identity/dedup keys.
+ # Typed request-slot metadata. Assertion identity, deduplication, and reconciliation
+ # intentionally exclude these fields.
request_slot_id: str | None = None
pinability: Pinability | None = None
mint_disposition: MintDisposition = "decidable"
@@ -366,7 +366,6 @@ class RequestPolicy:
classifier_retry_count: int = 0
classifier_non_runtime_requested_output_evidence_sources: list[str] = field(default_factory=list)
completion_contract_status: str = "absent"
- classifier_contract_version: int | None = None
request_slot_failure_kind: str | None = None
def graded_completion_criteria(self) -> list[CompletionCriterion]:
@@ -413,10 +412,8 @@ class RequestPolicy:
criterion for criterion in self.graded_completion_criteria() if criterion.output_path is not None
]
data["requested_output_criteria_count"] = len(requested_output_criteria)
- if self.classifier_contract_version is not None:
- data["classifier_contract_version"] = self.classifier_contract_version
- if self.request_slot_failure_kind is not None:
- data["request_slot_failure_kind"] = self.request_slot_failure_kind
+ if self.request_slot_failure_kind is not None:
+ data["request_slot_failure_kind"] = self.request_slot_failure_kind
for index, criterion in enumerate(requested_output_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):
prefix = f"requested_output_criterion_{index}"
data[f"{prefix}_id"] = criterion.id
@@ -425,13 +422,12 @@ class RequestPolicy:
data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None
if criterion.mint_degrade is not None:
data[f"{prefix}_mint_degrade"] = criterion.mint_degrade
+ data[f"{prefix}_mint_disposition"] = criterion.mint_disposition
data[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_source
- if self.classifier_contract_version is not None:
- data[f"{prefix}_mint_disposition"] = criterion.mint_disposition
- if criterion.request_slot_id is not None:
- data[f"{prefix}_request_slot_id"] = criterion.request_slot_id
- if criterion.pinability is not None:
- data[f"{prefix}_pinability"] = criterion.pinability
+ if criterion.request_slot_id is not None:
+ data[f"{prefix}_request_slot_id"] = criterion.request_slot_id
+ if criterion.pinability is not None:
+ data[f"{prefix}_pinability"] = criterion.pinability
if criterion.expected_output_shape:
data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape
mint_degraded_criteria = [
@@ -529,6 +525,21 @@ def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:
)
+def is_judgment_finalization_candidate(criterion: CompletionCriterion) -> bool:
+ """Whether accepted-artifact evidence must decide or degrade this criterion."""
+ if criterion.pinability in {"shapeless_valid", "unpinnable"}:
+ return False
+ return (
+ criterion.judgment_truth_condition is not None
+ or _is_judgment_boolean_criterion(criterion)
+ or (
+ criterion.kind == "validation_classification"
+ and criterion.classification_output_key is not None
+ and (isinstance(criterion.expected_classification, bool) or criterion.mint_disposition == "pending")
+ )
+ )
+
+
def typed_expected_output_value_key(value: ExpectedOutputValue | None) -> str:
if isinstance(value, bool):
return f"bool:{value}"
@@ -1025,22 +1036,21 @@ def _classification_from_raw(
completion_contract = completion_contract_raw.strip() if isinstance(completion_contract_raw, str) else None
evidence_raw = raw.get("raw_secret_evidence")
raw_secret_evidence = evidence_raw if isinstance(evidence_raw, str) and evidence_raw.strip() else None
- has_contract_version = "contract_version" in raw
- version_raw = raw.get("contract_version")
- contract_version = (
- version_raw if isinstance(version_raw, int) and not isinstance(version_raw, bool) and version_raw == 1 else None
+ raw_criteria = raw.get("completion_criteria")
+ claims_request_slots = isinstance(raw_criteria, list) and any(
+ isinstance(item, dict) and _item_claims_request_slot(item) for item in raw_criteria
)
- if contract_version == 1 and request_slot_request is not None and request_slot_contract is not None:
+ if request_slot_request is not None and request_slot_contract is not None:
completion_criteria = _parse_fresh_request_slot_criteria(
- raw.get("completion_criteria"),
+ raw_criteria,
request_slot_request=request_slot_request,
request_slot_contract=request_slot_contract,
)
- elif has_contract_version:
- completion_criteria = _fresh_request_slot_failure_criteria(raw.get("completion_criteria"))
- request_slot_failure_kind = request_slot_failure_kind or "unsupported_contract_version"
+ elif claims_request_slots:
+ completion_criteria = _fresh_request_slot_failure_criteria(raw_criteria)
+ request_slot_failure_kind = request_slot_failure_kind or "missing_request_slot_contract"
else:
- completion_criteria = _parse_completion_criteria(raw.get("completion_criteria"))
+ completion_criteria = _parse_completion_criteria(raw_criteria)
policy = RequestPolicy(
testing_intent=testing_intent if testing_intent in _TESTING_INTENTS else "unspecified",
authoring_intent=authoring_intent if authoring_intent in _AUTHORING_INTENTS else "author_now",
@@ -1055,9 +1065,6 @@ def _classification_from_raw(
clarification_reason=_coerce_clarification_reason(raw.get("clarification_reason")),
classifier_status="success",
classifier_failure_kind="none",
- classifier_contract_version=contract_version
- if contract_version is not None
- else (-1 if has_contract_version else None),
request_slot_failure_kind=request_slot_failure_kind,
)
if policy.credential_input_kind == "raw_secret":
@@ -1157,11 +1164,20 @@ def _parse_completion_criterion_entries(raw: Any) -> list[tuple[dict[str, Any],
if kind == "validation_classification"
else None
)
+ boolean_classification = kind == "validation_classification" and (
+ isinstance(expected_classification, bool)
+ or (expected_classification is None and expected_output_shape == "goal_judgment_boolean")
+ )
if kind == "validation_classification":
+ if boolean_classification and classification_output_key is not None:
+ outcome = f"The run classifies whether {classification_output_key.replace('_', ' ')}."
+ expected_classification = None
output_path = None
expected_output_value = None
- expected_output_shape = None
- requested_output_evidence_source = "runtime_output"
+ expected_output_shape = "goal_judgment_boolean" if boolean_classification else None
+ requested_output_evidence_source = (
+ "independent_run_evidence" if boolean_classification else "runtime_output"
+ )
elif isinstance(expected_output_value, bool) or expected_output_shape == "goal_judgment_boolean":
requested_output_evidence_source = "independent_run_evidence"
requested_output_path_mint_source: RequestedOutputPathMintSource | None = None
@@ -1221,6 +1237,15 @@ def _parse_completion_criterion_entries(raw: Any) -> list[tuple[dict[str, Any],
classification_output_key=classification_output_key,
expected_classification=expected_classification,
judgment_truth_condition=judgment_truth_condition,
+ mint_disposition=(
+ "pending"
+ if (
+ isinstance(expected_output_value, bool)
+ or expected_output_shape == "goal_judgment_boolean"
+ or boolean_classification
+ )
+ else "decidable"
+ ),
),
)
)
@@ -1617,6 +1642,18 @@ def _requested_output_criterion_id(output_path: str) -> str:
return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"
+def _requested_output_mint_state(
+ expected_value: ExpectedOutputValue | None,
+ expected_shape: ExpectedOutputShape | None,
+ judgment_condition: JudgmentTruthCondition | None,
+) -> tuple[MintDisposition, MintDegrade | None]:
+ if isinstance(expected_value, bool) or expected_shape == "goal_judgment_boolean" or judgment_condition is not None:
+ return "pending", None
+ if expected_shape == "status_label" or (expected_value is None and expected_shape is not None):
+ return "degraded", "undecidable_judgment"
+ return "decidable", None
+
+
def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:
key = normalized_criterion_outcome_key(criterion.outcome)
if is_fallback_floor_criterion(criterion):
@@ -1734,21 +1771,31 @@ def _source_requested_output_corroborator(
requested_fields: list[str],
requested_output_paths: set[str],
) -> CompletionCriterion | None:
- for criterion in criteria:
- if criterion.level != "run" or criterion.kind != "outcome" or criterion.method_mandated:
- continue
- if _validation_classification_output_path(criterion.output_path):
- continue
- if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(
- criterion, requested_fields
- ):
- return replace(
- criterion,
- output_path=None,
- expected_output_value=None,
- expected_output_shape=None,
- requested_output_corroborator=True,
- )
+ if len(requested_output_paths) == 1:
+ for criterion in criteria:
+ if (
+ criterion.level != "run"
+ or criterion.kind != "outcome"
+ or criterion.method_mandated
+ or criterion.output_path is None
+ ):
+ continue
+ if is_judgment_finalization_candidate(criterion):
+ continue
+ if criterion.requested_output_evidence_source == "independent_run_evidence":
+ continue
+ if _validation_classification_output_path(criterion.output_path):
+ continue
+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(
+ criterion, requested_fields
+ ):
+ return replace(
+ criterion,
+ output_path=None,
+ expected_output_value=None,
+ expected_output_shape=None,
+ requested_output_corroborator=True,
+ )
for criterion in criteria:
if is_fallback_floor_base_criterion(criterion):
return replace(criterion, requested_output_corroborator=True)
@@ -1775,7 +1822,9 @@ def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicy
and criterion.kind == "outcome"
and not criterion.method_mandated
and criterion.output_path is not None
+ and criterion.requested_output_evidence_source != "independent_run_evidence"
and not _validation_classification_output_path(criterion.output_path)
+ and not is_judgment_finalization_candidate(criterion)
and not criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)
]
if not source_criteria:
@@ -1885,23 +1934,34 @@ def _apply_requested_output_completion_criteria(
forbidden_output_paths,
)
]
- canonical_requested_criteria = [
- CompletionCriterion(
- id=_requested_output_criterion_id(output_path),
- outcome=f"The returned record includes {field_label}.",
- level="run",
- output_path=output_path,
- expected_output_value=value_by_output_path.get(output_path),
- expected_output_shape=shape_by_output_path.get(output_path),
- requested_output_evidence_source=source_by_output_path.get(output_path, "runtime_output"),
- contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],
- contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],
- deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],
- declared_deliverable_kind=declared_kind_by_output_path.get(output_path),
- judgment_truth_condition=judgment_condition_by_output_path.get(output_path),
+ canonical_requested_criteria: list[CompletionCriterion] = []
+ for _field_name, output_path, field_label in requested_specs:
+ expected_value = value_by_output_path.get(output_path)
+ expected_shape = shape_by_output_path.get(output_path)
+ judgment_condition = judgment_condition_by_output_path.get(output_path)
+ mint_disposition, mint_degrade = _requested_output_mint_state(
+ expected_value,
+ expected_shape,
+ judgment_condition,
+ )
+ canonical_requested_criteria.append(
+ CompletionCriterion(
+ id=_requested_output_criterion_id(output_path),
+ outcome=f"The returned record includes {field_label}.",
+ level="run",
+ output_path=output_path,
+ expected_output_value=expected_value,
+ expected_output_shape=expected_shape,
+ requested_output_evidence_source=source_by_output_path.get(output_path, "runtime_output"),
+ contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],
+ contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],
+ deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],
+ declared_deliverable_kind=declared_kind_by_output_path.get(output_path),
+ judgment_truth_condition=judgment_condition,
+ mint_degrade=mint_degrade,
+ mint_disposition=mint_disposition,
+ )
)
- for _field_name, output_path, field_label in requested_specs
- ]
criteria = preserved_criteria + canonical_requested_criteria
if not any(_non_requested_output_run_corroborator(criterion) for criterion in criteria):
corroborator = _source_requested_output_corroborator(
@@ -1942,18 +2002,22 @@ def _apply_validation_classification_completion_criteria(policy: RequestPolicy)
target = _validation_classification_target_for_legacy_criterion(criterion)
if target is not None:
output_key, expected = target
+ boolean_classification = isinstance(expected, bool)
criterion = replace(
criterion,
kind="validation_classification",
output_path=None,
expected_output_value=None,
- expected_output_shape=None,
- requested_output_evidence_source="runtime_output",
+ expected_output_shape="goal_judgment_boolean" if boolean_classification else None,
+ requested_output_evidence_source=(
+ "independent_run_evidence" if boolean_classification else "runtime_output"
+ ),
deliverable_kind=None,
declared_deliverable_kind=None,
terminal_action_family=None,
classification_output_key=output_key,
expected_classification=expected,
+ mint_disposition="pending" if boolean_classification else "decidable",
)
if criterion.kind == "validation_classification":
target_key = (
@@ -2023,10 +2087,6 @@ def is_contingent_missing_antecedent_degraded(criterion: CompletionCriterion) ->
return criterion.mint_degrade == "contingent_missing_antecedent"
-def is_undecidable_judgment_degraded(criterion: CompletionCriterion) -> bool:
- return criterion.mint_degrade == "undecidable_judgment"
-
-
def resolve_mint_degrade(
stored_value: object,
contingent_on: str | None,
@@ -2422,7 +2482,7 @@ async def _classify_request(
return _classifier_fallback_policy(
ids,
raw_secret_present=raw_secret_present,
- failure_kind="provider_error",
+ failure_kind=failure_kind if failure_kind != "none" else "provider_error",
retry_count=retry_count,
user_message=user_message,
requested_output_path_aliases=requested_output_path_aliases,
@@ -2430,32 +2490,26 @@ async def _classify_request(
request_slot_contract: RequestSlotContractV1 | None = None
request_slot_failure_kind: str | None = None
- version_raw = raw_payload.get("contract_version")
- if isinstance(version_raw, int) and not isinstance(version_raw, bool) and version_raw == 1:
- raw_criteria = raw_payload.get("completion_criteria")
- declared_request_slots = isinstance(raw_criteria, list) and any(
- isinstance(item, dict) and _request_slot_anchor(item) is not None for item in raw_criteria
- )
- if declared_request_slots:
- # This validation hop is deliberately sequential: the request-policy
- # classifier first declares which criteria are request-owned slots, then
- # the independent producer types their identity, plane, and pinability.
- request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)
- if request_slot_result.status == "success":
- request_slot_contract = request_slot_result.contract
- else:
- request_slot_failure_kind = (
- request_slot_result.failure_kind.value
- if request_slot_result.failure_kind is not None
- else "unknown"
- )
- LOG.warning(
- "request-policy request-slot producer failed",
- failure_kind=request_slot_failure_kind,
- attempts=request_slot_result.attempts,
- )
+ raw_criteria = raw_payload.get("completion_criteria")
+ declared_request_slots = isinstance(raw_criteria, list) and any(
+ isinstance(item, dict) and _request_slot_anchor(item) is not None for item in raw_criteria
+ )
+ if declared_request_slots:
+ # This validation hop is deliberately sequential: the request-policy
+ # classifier first declares which criteria are request-owned slots, then
+ # the independent producer types their identity, plane, and pinability.
+ request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)
+ if request_slot_result.status == "success":
+ request_slot_contract = request_slot_result.contract
else:
- request_slot_failure_kind = "no_declared_slots"
+ request_slot_failure_kind = (
+ request_slot_result.failure_kind.value if request_slot_result.failure_kind is not None else "unknown"
+ )
+ LOG.warning(
+ "request-policy request-slot producer failed",
+ failure_kind=request_slot_failure_kind,
+ attempts=request_slot_result.attempts,
+ )
policy = _classification_from_raw(
raw_payload,
request_slot_request=request_slot_request,
@@ -2514,13 +2568,6 @@ async def _classify_request(
policy.clarification_reason = "none"
policy.requires_user_clarification = False
policy.raw_secret_evidence = None
- if policy.classifier_contract_version is None:
- # Legacy corroborators are inference-based compatibility behavior. A fresh
- # contract remains fail-closed even when its request-slot producer fails:
- # fresh criteria degrade above instead of re-entering legacy guessing.
- _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)
- _apply_validation_classification_completion_criteria(policy)
- _apply_classifier_typed_requested_output_corroborators(policy)
_degrade_pathless_contingent_criteria(policy)
policy.completion_contract_status = (
"present" if policy.completion_contract or policy.graded_completion_criteria() else "absent"
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 1053bddd3..954b4d7bb 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_terminal_goal_record_corroboration,
grade_terminal_goal_record_criteria,
grade_validation_classification_criteria,
+ gradeable_completion_criteria,
is_fallback_floor_base_criterion,
is_registered_download_completion_criterion,
only_degraded_blocking,
@@ -237,17 +238,26 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriter
policy = _completion_request_policy(copilot_ctx)
# A method-mandated criterion asserts HOW the goal was reached; the outcome
# judge sees only end-state evidence.
- criteria = policy.graded_completion_criteria() if policy is not None else []
+ formed_criteria = policy.graded_completion_criteria() if policy is not None else []
+ criteria = gradeable_completion_criteria(formed_criteria)
authored_output_criteria = _authored_output_contract_criteria(copilot_ctx)
- if authored_output_criteria and (not criteria or all(is_fallback_floor_criterion(c) for c in criteria)):
+ if authored_output_criteria and (
+ not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)
+ ):
return authored_output_criteria
if _accepted_staged_output_contract_missing(copilot_ctx) and (
- not criteria or all(is_fallback_floor_criterion(c) for c in criteria)
+ not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)
):
return [_authored_output_contract_missing_criterion()]
return criteria
+def _carry_degraded_ids(copilot_ctx: Any, verification: CompletionVerificationResult) -> CompletionVerificationResult:
+ policy = _completion_request_policy(copilot_ctx)
+ formed_criteria = policy.graded_completion_criteria() if policy is not None else []
+ return carry_degraded_criterion_ids(verification, formed_criteria)
+
+
def _authored_output_contract_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:
paths = _authored_output_contract_paths(copilot_ctx)
return [
@@ -459,7 +469,7 @@ async def _maybe_run_completion_verification_from_page_observation(
)
if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:
verification = _no_gradeable_run_plane_result(criterion_ids)
- verification = carry_degraded_criterion_ids(verification, criteria)
+ verification = _carry_degraded_ids(copilot_ctx, verification)
verification = carry_floor_rekeyed_criterion_ids(verification, criteria)
copilot_ctx.completion_verification_result = verification
record_completion_verification(copilot_ctx, verification)
@@ -616,7 +626,7 @@ async def _maybe_run_completion_verification_from_page_observation(
requested_output_criteria_count=len(requested_output_criteria),
)
- verification = carry_degraded_criterion_ids(verification, criteria)
+ verification = _carry_degraded_ids(copilot_ctx, verification)
verification = carry_floor_rekeyed_criterion_ids(verification, criteria)
if (
isinstance(existing, CompletionVerificationResult)
@@ -1355,7 +1365,7 @@ async def _maybe_run_completion_verification(
verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)
if verification is None:
return None
- verification = carry_degraded_criterion_ids(verification, criteria)
+ verification = _carry_degraded_ids(copilot_ctx, verification)
verification = carry_floor_rekeyed_criterion_ids(verification, criteria)
run_data = result.get("data")
return _carry_floor_rekeyed_backing(
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index 0ee848b39..7c2cd9d6f 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -718,7 +718,7 @@ class WorkflowParametersRepository(BaseRepository):
organization_id: str,
workflow_copilot_chat_id: str,
goal_epoch: int,
- criteria: list[dict],
+ criteria: list[dict[str, Any]],
source_turn_id: str | None = None,
source_goal_text: str | None = None,
consecutive_all_no_evidence: int = 0,
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 89cc8ac9b..ff8059cb6 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -29,7 +29,7 @@ class WorkflowCopilotCompletionCriteriaSet(BaseModel):
workflow_copilot_chat_id: str
goal_epoch: int
status: str
- criteria: list[dict]
+ criteria: list[dict[str, Any]]
source_turn_id: str | None = None
source_goal_text: str | None = None
consecutive_all_no_evidence: int = 0
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index ed3962076..87710324b 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -17,6 +17,7 @@ from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot import request_policy as request_policy_module
from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import (
_VERIFIED_WORKFLOW_SUCCESS_REPLY,
@@ -72,6 +73,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
_REDACTED_REFUSED_SECRET_TURN,
TRANSCRIPT_ANCHOR_CHAR_CAP,
CompletionCriterion,
+ JudgmentTruthCondition,
RequestPolicy,
_classifier_fallback_policy,
_classify_request,
@@ -84,6 +86,7 @@ from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REA
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.completion import (
_authored_output_contract_criteria,
+ _carry_degraded_ids,
_completion_verification_criteria,
_maybe_run_completion_verification,
)
@@ -3910,6 +3913,28 @@ class TestRequestPolicyCredentialResolution:
assert policy.classifier_retry_count == 0
assert policy.completion_contract_status == "present"
+ @pytest.mark.asyncio
+ async def test_request_policy_classifier_malformed_payload_preserves_timeout_failure(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ async def malformed_after_timeout(*_args: object, **_kwargs: object) -> tuple[str, str, int]:
+ return "not-json", "timeout", 1
+
+ monkeypatch.setattr(request_policy_module, "_run_request_policy_classifier", malformed_after_timeout)
+
+ policy = await _classify_request(
+ user_message="Build a workflow for https://example.com.",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ handler=AsyncMock(),
+ )
+
+ assert policy.classifier_status == "fallback"
+ assert policy.classifier_failure_kind == "timeout"
+ assert policy.classifier_retry_count == 1
+ assert policy.completion_contract_status == "present"
+
@pytest.mark.asyncio
async def test_request_policy_classifier_retries_transient_error_then_succeeds(self) -> None:
class RateLimitError(Exception):
@@ -6512,6 +6537,89 @@ class TestClassifierFallbackCompletionFloor:
assert policy.completion_criteria
assert _completion_verification_criteria(ctx) == [policy.completion_criteria[0]]
+ def test_degraded_judgment_does_not_hide_contradictory_required_outputs(self) -> None:
+ reached = CompletionCriterion(id="c0", outcome="the requested page is reached")
+ degraded_judgment = CompletionCriterion(
+ id="public_form_exists",
+ outcome="the public-form judgment is returned",
+ mint_degrade="undecidable_judgment",
+ )
+ required_output = CompletionCriterion(id="visible_page_path_label", outcome="the visible path is returned")
+ contradiction = "evidence_contradicts"
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[reached.id, required_output.id],
+ verdicts=[
+ CriterionVerdict(criterion_id=reached.id, state="satisfied", reason_code="evidence_confirms"),
+ CriterionVerdict(criterion_id=required_output.id, state="unsatisfied", reason_code=contradiction),
+ ],
+ )
+
+ carried = _carry_degraded_ids(
+ SimpleNamespace(
+ request_policy=RequestPolicy(completion_criteria=[reached, degraded_judgment, required_output])
+ ),
+ verification,
+ )
+ assert carried.degraded_criterion_ids == [degraded_judgment.id]
+ assert carried.is_fully_satisfied() is False
+
+ judgment_only = _carry_degraded_ids(
+ SimpleNamespace(request_policy=RequestPolicy(completion_criteria=[reached, degraded_judgment])),
+ CompletionVerificationResult(
+ status="evaluated", criterion_ids=[reached.id], verdicts=verification.verdicts[:1]
+ ),
+ )
+ assert judgment_only.degraded_criterion_ids == [degraded_judgment.id]
+ assert judgment_only.is_fully_satisfied() is True
+
+ def test_p9_degraded_judgments_suppress_their_generic_corroborator(self) -> None:
+ reached = CompletionCriterion(id="c2", outcome="the visible page path label is returned")
+ degraded = [
+ CompletionCriterion(
+ id=criterion_id,
+ outcome=outcome,
+ kind="validation_classification",
+ classification_output_key=output_key,
+ judgment_truth_condition=truth_condition,
+ mint_degrade="undecidable_judgment",
+ mint_disposition="degraded",
+ )
+ for criterion_id, outcome, output_key, truth_condition in (
+ ("c0", "whether a public form exists", "public_form_exists", None),
+ (
+ "c1",
+ "whether the path is login-only",
+ "login_only",
+ JudgmentTruthCondition(predicate="login_gate_blocks_target", polarity_when_holds=True),
+ ),
+ )
+ ]
+ judgment_corroborator = CompletionCriterion(
+ id="c3",
+ outcome="the recommended next action is returned",
+ requested_output_evidence_source="independent_run_evidence",
+ requested_output_corroborator=True,
+ )
+ policy = RequestPolicy(completion_criteria=[*degraded, reached, judgment_corroborator])
+ ctx = SimpleNamespace(request_policy=policy)
+
+ assert _completion_verification_criteria(ctx) == [reached]
+
+ carried = _carry_degraded_ids(
+ ctx,
+ CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[reached.id],
+ verdicts=[
+ CriterionVerdict(criterion_id=reached.id, state="satisfied", reason_code="evidence_confirms"),
+ ],
+ ),
+ )
+
+ assert carried.degraded_criterion_ids == ["c0", "c1"]
+ assert carried.is_fully_satisfied() is True
+
def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:
base = build_classifier_fallback_floor([])
credentialed = build_classifier_fallback_floor(["cred_1"])
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 3c609de18..751d025ae 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -81,6 +81,8 @@ from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ CriterionKind,
+ CriterionLevel,
JudgmentTruthCondition,
RequestedOutputEvidenceSource,
RequestPolicy,
@@ -112,6 +114,24 @@ from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
+def _typed_completion_criterion(
+ *,
+ id: str,
+ output_path: str,
+ level: CriterionLevel,
+ method_mandated: bool,
+ kind: CriterionKind,
+) -> CompletionCriterion:
+ return CompletionCriterion(
+ id=id,
+ outcome=f"The run returns {output_path}.",
+ output_path=output_path,
+ level=level,
+ method_mandated=method_mandated,
+ kind=kind,
+ )
+
+
def _yaml(body: str) -> str:
return textwrap.dedent(body).strip() + "\n"
@@ -3846,21 +3866,21 @@ class TestCodeRepairProgressClassification:
label = "lookup_entry"
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.npi",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_locations",
output_path="output.locations",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_statuses",
output_path="output.statuses",
level="run",
@@ -3931,7 +3951,7 @@ class TestCodeRepairProgressClassification:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -4110,14 +4130,14 @@ class TestCodeRepairProgressClassification:
required_paths = {"output.account_number", "output.confirmation_number"}
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_account",
output_path="output.account_number",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_confirmation",
output_path="output.confirmation_number",
level="run",
@@ -4451,7 +4471,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -4641,7 +4661,7 @@ class TestCodeRepairProgressClassification:
ctx.turn_id = "stable-contract-key"
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -4701,7 +4721,7 @@ class TestCodeRepairProgressClassification:
ctx.turn_id = turn_id
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path=output_path,
level="run",
@@ -4732,14 +4752,14 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_flags",
output_path="output.flags",
level="run",
@@ -4785,7 +4805,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.npi",
level="run",
@@ -4849,7 +4869,7 @@ class TestCodeRepairProgressClassification:
child_paths = ["output.npi"]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.npi",
level="run",
@@ -4978,7 +4998,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_account",
output_path="output.account_number",
level="run",
@@ -5178,7 +5198,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.requested_value",
level="run",
@@ -6175,7 +6195,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.requested_value",
level="run",
@@ -6603,14 +6623,14 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_flags",
output_path="output.flags",
level="run",
@@ -6658,7 +6678,7 @@ class TestCodeRepairProgressClassification:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -6717,7 +6737,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_npi",
output_path="output.npi",
level="run",
@@ -6792,7 +6812,7 @@ class TestCodeRepairProgressClassification:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -6862,7 +6882,7 @@ class TestCodeRepairProgressClassification:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -6932,7 +6952,7 @@ class TestCodeRepairProgressClassification:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -7001,7 +7021,7 @@ class TestCodeRepairProgressClassification:
ctx.scout_trajectory = []
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -7058,7 +7078,7 @@ class TestCodeRepairProgressClassification:
)
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id=f"requested_{index}",
output_path=path,
level="run",
@@ -7108,14 +7128,14 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.npi",
level="run",
method_mandated=False,
kind="outcome",
),
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_statuses",
output_path="output.statuses",
level="run",
@@ -7263,7 +7283,7 @@ class TestCodeRepairProgressClassification:
ctx = _code_only_ctx()
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.requested_only",
level="run",
@@ -8018,7 +8038,7 @@ def _spine_actuation_ctx() -> CopilotContext:
]
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_value",
output_path="output.record_id",
level="run",
@@ -13293,7 +13313,7 @@ class TestWholeTrajectoryImposition:
)
ctx.request_policy = RequestPolicy(
completion_criteria=[
- SimpleNamespace(
+ _typed_completion_criterion(
id="requested_npi",
output_path="output.npi",
level="run",
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index acc0dacab..628db0951 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
)
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ JudgmentTruthCondition,
RequestPolicy,
_parse_completion_criteria,
normalized_criterion_outcome_key,
@@ -315,6 +316,118 @@ def test_criteria_json_round_trip_preserves_fields(criteria: tuple[CompletionCri
assert criteria_from_json(criteria_to_json(criteria)) == criteria
+def test_typed_criteria_list_round_trip_preserves_grading_metadata() -> None:
+ criterion = CompletionCriterion(
+ id="c0",
+ outcome="The visible page path label is returned.",
+ output_path="output.visible_page_path_label",
+ expected_output_value="Public start-service path",
+ pinability="unpinnable",
+ mint_degrade="undecidable_judgment",
+ mint_disposition="degraded",
+ )
+
+ raw = criteria_to_json([criterion])
+
+ assert isinstance(raw, list)
+ assert raw[0]["pinability"] == "unpinnable"
+ assert criteria_from_json(raw) == (criterion,)
+
+
+def test_typed_boolean_validation_classification_round_trip_preserves_shape_and_evidence() -> None:
+ """Typed boolean classifications persist goal_judgment_boolean / independent_run_evidence."""
+ criterion = CompletionCriterion(
+ id="c0",
+ outcome="The run classifies whether a public form exists.",
+ kind="validation_classification",
+ classification_output_key="public_form_exists",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ judgment_truth_condition=JudgmentTruthCondition(
+ predicate="login_gate_blocks_target", polarity_when_holds=False
+ ),
+ pinability="pinned",
+ mint_disposition="pending",
+ )
+
+ raw = criteria_to_json([criterion])
+
+ assert raw[0]["expected_output_shape"] == "goal_judgment_boolean"
+ assert raw[0]["requested_output_evidence_source"] == "independent_run_evidence"
+ reloaded = criteria_from_json(raw)
+ assert reloaded == (criterion,)
+ assert reloaded[0].expected_output_shape == "goal_judgment_boolean"
+ assert reloaded[0].requested_output_evidence_source == "independent_run_evidence"
+
+
+def test_untyped_boolean_validation_classification_reload_normalizes_shape_and_evidence() -> None:
+ """Rows without typed mint metadata carry no reliable shape/evidence markers."""
+ legacy = {
+ "id": "c0",
+ "outcome": "The run classifies whether a public form exists.",
+ "kind": "validation_classification",
+ "classification_output_key": "public_form_exists",
+ "expected_output_shape": "goal_judgment_boolean",
+ "requested_output_evidence_source": "independent_run_evidence",
+ }
+
+ reloaded = criteria_from_json([legacy])
+
+ assert reloaded[0].expected_output_shape is None
+ assert reloaded[0].requested_output_evidence_source == "runtime_output"
+
+
+def test_typed_metadata_is_self_describing_without_a_storage_version() -> None:
+ legacy = {"id": "c0", "outcome": "done", "output_path": "output.done"}
+ decorated = {
+ **legacy,
+ "pinability": "unpinnable",
+ "mint_disposition": "degraded",
+ }
+
+ assert criteria_from_json([legacy]) == (CompletionCriterion(id="c0", outcome="done", output_path="output.done"),)
+ assert criteria_from_json([decorated]) == (
+ CompletionCriterion(
+ id="c0",
+ outcome="done",
+ output_path="output.done",
+ pinability="unpinnable",
+ mint_disposition="degraded",
+ ),
+ )
+
+
+def test_reconciliation_identity_ignores_pinability_and_derived_mint_state() -> None:
+ stored = StoredCriteriaSet(
+ set_id="wccs_1",
+ goal_epoch=1,
+ criteria=(
+ CompletionCriterion(
+ id="s0",
+ outcome="done",
+ output_path="output.done",
+ expected_output_value=True,
+ pinability="pinned",
+ ),
+ ),
+ )
+ fresh = CompletionCriterion(
+ id="f0",
+ outcome="done",
+ output_path="output.done",
+ expected_output_value=True,
+ pinability="unpinnable",
+ mint_degrade="undecidable_judgment",
+ mint_disposition="degraded",
+ )
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2), [fresh], actionable=True
+ )
+
+ assert decision.action == "adopt_stored"
+
+
def test_criteria_from_json_degrades_stored_pathless_contingent() -> None:
(criterion,) = criteria_from_json(
[
@@ -1159,6 +1272,17 @@ def test_reconcile_on_context_adopts_stored_criteria_onto_policy() -> None:
assert ctx.completion_criteria_turn_state.decision.reason == "kept"
+def test_reconcile_on_context_creates_typed_criteria_without_a_contract_version() -> None:
+ policy = RequestPolicy(completion_criteria=[_criterion("c0", "the item is in the cart")])
+ ctx = _ctx()
+
+ _reconcile_completion_criteria_on_context(ctx, policy, _policy_inputs(StoredCriteriaSnapshot()))
+
+ plan = plan_persistence(ctx.completion_criteria_turn_state)
+ assert plan is not None
+ assert plan.create_criteria == tuple(policy.completion_criteria)
+
+
def test_reconcile_on_context_skips_without_snapshot() -> None:
policy = RequestPolicy(completion_criteria=[_criterion("c0", "the item is in the cart")])
ctx = _ctx()
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 794df8dac..352c288c8 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -60,6 +60,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
grade_structured_record_criteria,
grade_terminal_goal_record_criteria,
grade_validation_classification_criteria,
+ gradeable_completion_criteria,
is_registered_download_completion_criterion,
only_degraded_blocking,
registered_download_completion_criterion,
@@ -3390,6 +3391,38 @@ def test_zero_requested_output_criteria_credit_fires_only_with_payload() -> None
assert zero_requested_output_criteria_credit(satisfied, has_meaningful_registered_output=False) is False
+def test_gradeable_completion_criteria_excludes_pending_and_degraded_judgments() -> None:
+ reached = CompletionCriterion(id="reached", outcome="The requested page is reached.")
+ pending_judgment = CompletionCriterion(
+ id="pending",
+ outcome="The pending requested judgment is returned.",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ )
+ degraded_judgment = CompletionCriterion(
+ id="judgment",
+ outcome="The requested judgment is returned.",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ mint_degrade="undecidable_judgment",
+ mint_disposition="degraded",
+ )
+ pending_classification = CompletionCriterion(
+ id="classification",
+ outcome="The requested classification is returned.",
+ kind="validation_classification",
+ classification_output_key="login_only",
+ mint_disposition="pending",
+ )
+
+ assert gradeable_completion_criteria([reached, pending_judgment, degraded_judgment, pending_classification]) == [
+ reached
+ ]
+ assert gradeable_completion_criteria([pending_judgment]) == []
+ assert gradeable_completion_criteria([degraded_judgment]) == []
+
+
def test_zero_requested_output_criteria_credit_ignored_when_criteria_formed() -> None:
with_criteria = replace(_evaluated(("bill", True)), requested_output_criteria_count=1)
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 64afac334..e9f854daa 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -46,6 +46,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_maybe_synthesized_block_offer_msg,
_needs_suspicious_success_nudge,
_prune_input_list,
+ _requested_output_paths_for_ctx,
_should_block_mutating_tool_after_synthesized_offer,
_should_force_advisory_run_dispatch,
_should_force_synthesized_block_persistence,
@@ -2476,17 +2477,18 @@ class TestScoutOutputCoverageGate:
def test_independent_run_evidence_is_exempt_from_repair_context(self) -> None:
independent = CompletionCriterion(
- id="output.login_gate_present",
- outcome="whether a login gate blocked the target is recorded",
- output_path="output.login_gate_present",
+ id="output.login_gate_blocks_target",
+ outcome="the login-gate judgment is independently observed after the run",
+ output_path="output.login_gate_blocks_target",
+ expected_output_shape="goal_judgment_boolean",
requested_output_evidence_source="independent_run_evidence",
)
- runtime = _criterion("output.document_name", "the order status document name is captured")
+ runtime = _criterion("output.document_name", "the document name is captured")
ctx = self._authoring_ctx(independent, runtime)
ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
block_label="extract_order",
reason_code="metadata_reject",
- required_goal_value_paths=["login_gate_present", "document_name"],
+ required_goal_value_paths=["login_gate_blocks_target", "document_name"],
)
assert uncovered_requested_output_paths(ctx) == {"output.document_name"}
@@ -2556,6 +2558,22 @@ class TestScoutOutputCoverageGate:
assert uncovered_requested_output_paths(ctx) == {"output.document_name"}
assert synthesized_trajectory_is_goal_complete(ctx) is False
+ def test_pathless_post_run_criterion_does_not_erase_repair_output_field(self) -> None:
+ independent = CompletionCriterion(
+ id="c_independent",
+ outcome="the judgment is independently observed after the run",
+ output_path=None,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ctx = self._authoring_ctx(independent)
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_order",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["field"],
+ )
+
+ assert _requested_output_paths_for_ctx(ctx) == {"output.field"}
+
def test_runtime_output_stays_gated_alongside_exempt_source(self) -> None:
registered = CompletionCriterion(
id="output.confirmation_number",
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index c08d310fe..e32ffdf48 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -443,6 +443,7 @@ class TestMCPFailedStepLoopDetection:
last_code_authoring_repair_context=None,
scouted_output_covered_paths=set(),
reached_download_target=None,
+ request_policy=None,
)
server = SkyvernOverlayMCPServer(
transport=MagicMock(),
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 52e077e24..ea3e72844 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -1954,6 +1954,24 @@ def test_non_minted_registered_download_path_still_requires_author_coverage() ->
assert wu._requested_output_child_paths(ctx) == {"output.downloaded_files"}
+def test_independent_validation_classification_is_post_run_only() -> None:
+ ctx = _antecedent_ctx(
+ CompletionCriterion(
+ id="c1",
+ outcome="The run classifies whether the path is login-only.",
+ kind="validation_classification",
+ classification_output_key="login_only",
+ expected_classification=True,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ )
+ )
+
+ assert wu._requested_output_child_paths(ctx) == set()
+ assert wu._output_contract_required_paths_source(ctx).observation_paths == set()
+
+
def test_runtime_repair_contract_carries_declaration_lane_with_stable_signature() -> None:
ctx = _antecedent_ctx(
CompletionCriterion(
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index 243521a41..fbd1c4648 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -20,10 +20,10 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
JudgmentTruthCondition,
RequestPolicy,
+ _apply_classifier_typed_requested_output_corroborators,
_apply_requested_output_completion_criteria,
_apply_validation_classification_completion_criteria,
_classifier_fallback_policy,
- _classify_request,
_criterion_grounding_mode,
_degrade_pathless_contingent_criteria,
_parse_completion_criteria,
@@ -41,22 +41,31 @@ async def _policy_for_message(
*,
config: CopilotConfig | None = None,
) -> RequestPolicy:
- async def _handler(**_: Any) -> dict[str, Any]:
- return {
- "testing_intent": "require_test",
- "credential_input_kind": "none",
- "requires_user_clarification": False,
- "completion_criteria": criteria,
- }
+ """Exercise the retained deterministic fallback canonicalizers directly.
- return await _classify_request(
- user_message,
- workflow_yaml="",
- chat_history=[],
- global_llm_context="",
- handler=_handler,
- config=config,
+ Fresh classifier responses use typed request-slot binding instead; those integration
+ semantics are covered by test_copilot_request_policy_pinability_consumer.py.
+ """
+ policy = RequestPolicy(
+ testing_intent="require_test",
+ classifier_status="success",
+ classifier_failure_kind="none",
+ completion_criteria=_parse_completion_criteria(criteria),
)
+ aliases = config.requested_output_path_aliases if config is not None else {}
+ _apply_requested_output_completion_criteria(policy, user_message, aliases)
+ _apply_validation_classification_completion_criteria(policy)
+ _apply_classifier_typed_requested_output_corroborators(policy)
+ _degrade_pathless_contingent_criteria(policy)
+ policy.classifier_non_runtime_requested_output_evidence_sources = sorted(
+ {
+ criterion.requested_output_evidence_source
+ for criterion in policy.completion_criteria
+ if criterion.output_path is not None and criterion.requested_output_evidence_source != "runtime_output"
+ }
+ )
+ policy.completion_contract_status = "present" if policy.graded_completion_criteria() else "absent"
+ return policy
def _stored(*criteria: CompletionCriterion) -> StoredCriteriaSet:
@@ -76,6 +85,63 @@ def _criteria_fingerprint(criteria: list[CompletionCriterion]) -> str:
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+def test_p9_exact_value_unpinnable_output_mint_degrades() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c0",
+ outcome="The visible page path label is returned.",
+ output_path="output.visible_page_path_label",
+ expected_output_value="Public start-service path",
+ expected_output_shape="status_label",
+ pinability="unpinnable",
+ )
+ ]
+ )
+
+ _apply_requested_output_completion_criteria(policy, "Return visible_page_path_label.")
+
+ criterion = _criteria_for_path(policy, "output.visible_page_path_label")[0]
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
+def test_p9_classifier_drift_normalizes_to_one_contract_shape() -> None:
+ prompt = "Return public_form_exists, visible_page_path_label, and recommended_next_action."
+ classifier_shapes = [
+ [
+ CompletionCriterion(id="c0", outcome="a public form exists for starting service"),
+ CompletionCriterion(id="c1", outcome="the visible page path label is returned"),
+ ],
+ [
+ CompletionCriterion(
+ id="c0",
+ outcome="a public form exists for starting service",
+ output_path="output.public_form_exists",
+ ),
+ CompletionCriterion(id="c1", outcome="the recommended next action is returned"),
+ ],
+ [
+ CompletionCriterion(id="c0", outcome="a public form exists for starting service"),
+ CompletionCriterion(id="c1", outcome="the visible page path label is returned"),
+ CompletionCriterion(
+ id="c2",
+ outcome="the public form result is returned",
+ output_path="output.public_form_exists",
+ ),
+ CompletionCriterion(id="c3", outcome="the recommended next action is returned"),
+ ],
+ ]
+ policies = [RequestPolicy(completion_criteria=criteria) for criteria in classifier_shapes]
+
+ for policy in policies:
+ _apply_requested_output_completion_criteria(policy, prompt)
+
+ serialized = [criteria_to_json(policy.completion_criteria) for policy in policies]
+ assert [len(policy.completion_criteria) for policy in policies] == [3, 3, 3]
+ assert serialized == [serialized[0], serialized[0], serialized[0]]
+
+
def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[str]) -> list[CompletionCriterion]:
return [
criterion
@@ -654,7 +720,7 @@ def test_requested_output_corroborator_respects_completion_criteria_cap() -> Non
}
-def test_requested_output_corroborator_survives_when_requested_outputs_exceed_cap() -> None:
+def test_requested_output_canonicalization_drops_redundant_corroborator_when_outputs_exceed_cap() -> None:
policy = RequestPolicy(
completion_criteria=[
CompletionCriterion(
@@ -686,9 +752,7 @@ def test_requested_output_corroborator_survives_when_requested_outputs_exceed_ca
assert {
criterion.output_path for criterion in _requested_output_subset(policy, requested_output_paths)
} == requested_output_paths
- assert len(corroborators) == 1
- assert corroborators[0].id == "name_source"
- assert corroborators[0].requested_output_corroborator is True
+ assert corroborators == []
@pytest.mark.asyncio
@@ -868,7 +932,8 @@ def test_parse_completion_criteria_preserves_validation_classification_contract(
assert len(parsed) == 2
assert parsed[0].kind == "validation_classification"
assert parsed[0].classification_output_key == "login_gated"
- assert parsed[0].expected_classification is True
+ assert parsed[0].expected_classification is None
+ assert parsed[0].mint_disposition == "pending"
assert parsed[1].kind == "outcome"
@@ -2160,7 +2225,8 @@ def test_scope_boundary_boolean_on_login_only_rekinds_and_drops_judgment_invaria
assert promoted.kind == "validation_classification"
assert promoted.classification_output_key == "login_only"
assert promoted.expected_output_value is None
- assert promoted.requested_output_evidence_source == "runtime_output"
+ assert promoted.requested_output_evidence_source == "independent_run_evidence"
+ assert promoted.mint_disposition == "pending"
def test_scope_boundary_boolean_on_non_enumerated_path_keeps_judgment_invariant() -> None:
diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.py
index 21a368aca..b7b5123f8 100644
--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py
+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py
@@ -68,7 +68,6 @@ def _envelope() -> RequestSlotEnvelopeV1:
def _fresh_payload() -> dict[str, Any]:
return {
- "contract_version": 1,
"testing_intent": "require_test",
"credential_input_kind": "none",
"requires_user_clarification": False,
@@ -109,7 +108,6 @@ def test_fresh_contract_consumes_declared_pinability_without_guessed_polarity()
request_slot_contract=contract,
)
- assert policy.classifier_contract_version == 1
assert [criterion.request_slot_id for criterion in policy.completion_criteria] == [
slot.slot_id for slot in contract.slots
]
@@ -276,14 +274,12 @@ def test_followup_source_digest_change_adopts_stored_semantic_criteria() -> None
assert decision.superseded_set_id is None
-def test_invalid_fresh_version_fails_safe_without_entering_legacy_minting() -> None:
+def test_unbound_typed_criteria_fail_safe_without_entering_legacy_minting() -> None:
payload = _fresh_payload()
- payload["contract_version"] = "1"
policy = _classification_from_raw(payload)
- assert policy.classifier_contract_version == -1
- assert policy.request_slot_failure_kind == "unsupported_contract_version"
+ assert policy.request_slot_failure_kind == "missing_request_slot_contract"
assert all(criterion.output_path is None for criterion in policy.completion_criteria)
assert all(criterion.expected_output_value is None for criterion in policy.completion_criteria)
assert all(criterion.expected_classification is None for criterion in policy.completion_criteria)
@@ -293,7 +289,6 @@ def test_invalid_fresh_version_fails_safe_without_entering_legacy_minting() -> N
def test_fresh_slot_failure_preserves_non_output_outcome() -> None:
policy = _classification_from_raw(
{
- "contract_version": 1,
"completion_criteria": [{"outcome": "The application is submitted."}],
},
request_slot_failure_kind="invalid_output",
@@ -348,7 +343,6 @@ async def test_request_slot_producer_failure_degrades_without_legacy_guessing()
policy = await _classify_request(P9_REQUEST, "", [], "", handler)
assert calls == ["workflow-copilot-request-policy", *([REQUEST_SLOT_PROMPT_NAME] * 4)]
- assert policy.classifier_contract_version == 1
assert policy.request_slot_failure_kind == "invalid_output"
assert all(criterion.output_path is None for criterion in policy.completion_criteria)
assert all(criterion.expected_output_value is None for criterion in policy.completion_criteria)
@@ -357,7 +351,7 @@ async def test_request_slot_producer_failure_degrades_without_legacy_guessing()
@pytest.mark.asyncio
-async def test_versionless_classifier_payload_never_invokes_request_slot_producer() -> None:
+async def test_classifier_payload_without_request_slots_does_not_invoke_request_slot_producer() -> None:
calls: list[str] = []
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
@@ -369,9 +363,8 @@ async def test_versionless_classifier_payload_never_invokes_request_slot_produce
"completion_criteria": [{"outcome": "The requested workflow is complete."}],
}
- policy = await _classify_request("Build the workflow.", "", [], "", handler)
+ await _classify_request("Build the workflow.", "", [], "", handler)
- assert policy.classifier_contract_version is None
assert calls == ["workflow-copilot-request-policy"]
@@ -382,7 +375,6 @@ async def test_fresh_non_output_request_does_not_invoke_request_slot_producer()
async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
calls.append(prompt_name)
return {
- "contract_version": 1,
"testing_intent": "require_test",
"credential_input_kind": "none",
"requires_user_clarification": False,
@@ -391,7 +383,7 @@ async def test_fresh_non_output_request_does_not_invoke_request_slot_producer()
policy = await _classify_request("Submit the application.", "", [], "", handler)
- assert policy.request_slot_failure_kind == "no_declared_slots"
+ assert policy.request_slot_failure_kind is None
assert policy.completion_criteria[0].outcome == "The application is submitted."
assert policy.completion_criteria[0].mint_degrade is None
assert calls == ["workflow-copilot-request-policy"]
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 894034aee..cf4203084 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -94,6 +94,8 @@ class TestRequestPolicyPromptStructure:
"expected_output_shape, requested_output_evidence_source, kind, terminal_action_family, "
"classification_output_key, expected_classification, judgment_predicate, judgment_polarity_when_holds}"
) in rendered
+ assert "typed request-slot producer owns its final identity, path, plane, and pinability" in rendered
+ assert "Do not infer or emit pinability here" in rendered
assert "never hide it in outcome prose" in rendered
assert (
"reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label, "
diff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
index 52e5eef72..3ebfd7d6b 100644
--- a/tests/unit/test_copilot_sky11865_mint_satisfiability.py
+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py
@@ -327,10 +327,10 @@ def test_mint_degrade_survives_json_round_trip() -> None:
assert restored[0].mint_degrade == _DEGRADE
-def test_mint_degrade_is_identity_bearing_in_reconcile_key() -> None:
+def test_mint_degrade_is_not_identity_bearing_in_reconcile_key() -> None:
base = CompletionCriterion(id="req", outcome="return the status", level="run", output_path="output.status")
degraded = replace(base, mint_degrade=_DEGRADE)
- assert _criterion_reconcile_key(base) != _criterion_reconcile_key(degraded)
+ assert _criterion_reconcile_key(base) == _criterion_reconcile_key(degraded)
def test_producer_floor_excludes_degraded_from_rekey() -> None:
diff --git a/tests/unit/test_workflow_copilot_completion_criteria_schema.py b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
new file mode 100644
index 000000000..c678f47f6
--- /dev/null
+++ b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
@@ -0,0 +1,23 @@
+from datetime import UTC, datetime
+
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotCompletionCriteriaSet
+
+
+def test_completion_criteria_set_keeps_list_storage_shape() -> None:
+ now = datetime.now(UTC)
+ criteria = [{"id": "c0", "outcome": "done", "pinability": "pinned"}]
+
+ stored = WorkflowCopilotCompletionCriteriaSet.model_validate(
+ {
+ "completion_criteria_set_id": "wccs_1",
+ "organization_id": "o_1",
+ "workflow_copilot_chat_id": "wcc_1",
+ "goal_epoch": 1,
+ "status": "active",
+ "criteria": criteria,
+ "created_at": now,
+ "modified_at": now,
+ }
+ )
+
+ assert stored.criteria == criteria