Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-12163: Preserve Copilot completion checks when workflows are saved (#7455)
Andrew Neilson88fa07c13e96
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index d3f09e365..c9dc03f01 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -1,5 +1,5 @@Classify the latest Workflow Copilot request. Return only JSON with keys:-contract_version=1; testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.+testing_intent=require_test|skip_test|unspecified; authoring_intent=author_now|defer_authoring; credential_input_kind=none|raw_secret|credential_id|credential_name|website_stored_credential|placeholder; credential_refs; login_page_urls; requires_user_clarification; clarification_reason; completion_contract; completion_criteria; raw_secret_evidence; raw_secret_handling=none|block|redacted_draft.testing_intent:- skip_test: the user asks for an untested draft, says they will test it themselves, says draft only, just sketch it, no validation run, or skip testing.diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex 66ff5c8cd..868667018 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -10,7 +10,7 @@ from __future__ import annotationsfrom collections.abc import Iterablefrom dataclasses import dataclass, field, replace-from typing import Any, Literal, cast+from typing import Any, Literal, cast, get_argsfrom skyvern.forge.sdk.copilot.completion_output_grounding import split_requested_output_criteriafrom skyvern.forge.sdk.copilot.completion_verification import (@@ -56,8 +56,8 @@ TRIPWIRE_CONSECUTIVE_ALL_NO_EVIDENCE = 2_CRITERION_LEVELS = ("definition", "run")_CRITERION_KINDS = ("outcome", "terminal_action", "validation_classification")_TERMINAL_ACTION_FAMILIES = ("request", "application", "form", "order")-_PINABILITY_VALUES = ("pinned", "shapeless_valid", "unpinnable")-_MINT_DISPOSITION_VALUES = ("pending", "decidable", "degraded")+_MINT_DISPOSITIONS = frozenset(get_args(MintDisposition))+_PINABILITIES = frozenset(get_args(Pinability))@dataclass(frozen=True)@@ -221,9 +221,9 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:else None)pinability_raw = item.get("pinability")- pinability = pinability_raw if pinability_raw in _PINABILITY_VALUES else None+ pinability = pinability_raw if pinability_raw in _PINABILITIES else Nonemint_disposition_raw = item.get("mint_disposition")- mint_disposition = mint_disposition_raw if mint_disposition_raw in _MINT_DISPOSITION_VALUES else "decidable"+ mint_disposition = mint_disposition_raw if mint_disposition_raw in _MINT_DISPOSITIONS else "decidable"stored_output_path = output_path.strip() if isinstance(output_path, str) and output_path.strip() else Nonestored_expected_output_value = _coerce_expected_output_value(expected_output_value)stored_expected_output_shape = cast(ExpectedOutputShape | None, expected_output_shape)@@ -237,8 +237,18 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:if kind == "validation_classification":stored_output_path = Nonestored_expected_output_value = None- stored_expected_output_shape = None- requested_output_evidence_source = "runtime_output"+ # Typed boolean classifications carry their own validated mint metadata. Rows without+ # it carry no reliable shape, so retain the historical normalization for those rows.+ is_typed_boolean_classification = (+ stored_expected_output_shape == "goal_judgment_boolean"+ and pinability == "pinned"+ and mint_disposition == "pending"+ )+ if is_typed_boolean_classification:+ requested_output_evidence_source = "independent_run_evidence"+ else:+ stored_expected_output_shape = None+ requested_output_evidence_source = "runtime_output"elif isinstance(stored_expected_output_value, bool) or stored_expected_output_shape == "goal_judgment_boolean":requested_output_evidence_source = "independent_run_evidence"criteria.append(@@ -281,7 +291,6 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:contingent_path_key = criterion.contingent_antecedent_output_path or ""deliverable_kind_key = (f"{criterion.deliverable_kind or ''}\x1fdeclared:{criterion.declared_deliverable_kind or ''}"- f"\x1fmint_degrade:{criterion.mint_degrade or ''}"f"\x1fjudgment:{judgment_truth_condition_key(criterion.judgment_truth_condition)}")expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)@@ -480,14 +489,25 @@ def reconcile_completion_criteria(return ReconcileDecision(action="none", reason="no_criteria", epoch=0, criteria=())if not actionable:return ReconcileDecision(action="none", reason="not_actionable", epoch=0, criteria=tuple(fresh))- return ReconcileDecision(action="create", reason="first", epoch=next_epoch, criteria=tuple(fresh))+ return ReconcileDecision(+ action="create",+ reason="first",+ epoch=next_epoch,+ criteria=tuple(fresh),+ )if not fresh:return ReconcileDecision(- action="adopt_stored", reason="empty_fresh", epoch=stored.goal_epoch, criteria=stored.criteria+ action="adopt_stored",+ reason="empty_fresh",+ epoch=stored.goal_epoch,+ criteria=stored.criteria,)if _outcome_key_set(fresh) <= _outcome_key_set(stored.criteria):return ReconcileDecision(- action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria+ action="adopt_stored",+ reason="kept",+ epoch=stored.goal_epoch,+ criteria=stored.criteria,)if _fresh_generic_rephrase_lacks_stored_requested_outputs(stored.criteria,@@ -495,11 +515,17 @@ def reconcile_completion_criteria(requested_output_path_aliases=requested_output_path_aliases,):return ReconcileDecision(- action="adopt_stored", reason="kept", epoch=stored.goal_epoch, criteria=stored.criteria+ action="adopt_stored",+ reason="kept",+ epoch=stored.goal_epoch,+ criteria=stored.criteria,)if not actionable:return ReconcileDecision(- action="adopt_stored", reason="not_actionable", epoch=stored.goal_epoch, criteria=stored.criteria+ action="adopt_stored",+ reason="not_actionable",+ epoch=stored.goal_epoch,+ criteria=stored.criteria,)return ReconcileDecision(action="create",diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex ad337a091..f2bd44476 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -28,8 +28,8 @@ from skyvern.forge.sdk.copilot.request_policy import (TerminalActionFamily,is_contingent_missing_antecedent_degraded,is_fallback_floor_base_criterion,+ is_judgment_finalization_candidate,is_turn_unsatisfiable_fallback_degraded,- is_undecidable_judgment_degraded,redact_raw_secrets_for_prompt,)from skyvern.utils.strings import escape_code_fences@@ -2457,6 +2457,21 @@ def floor_rekeyed_deliverable_credit()+def gradeable_completion_criteria(criteria: Iterable[CompletionCriterion]) -> list[CompletionCriterion]:+ return [+ criterion+ for criterion in criteria+ if criterion.mint_degrade != "undecidable_judgment"+ and not (+ is_judgment_finalization_candidate(criterion) and criterion.mint_disposition in {"pending", "degraded"}+ )+ and not (+ criterion.requested_output_corroborator+ and criterion.requested_output_evidence_source == "independent_run_evidence"+ )+ ]++def carry_degraded_criterion_ids(result: CompletionVerificationResult,criteria: Iterable[CompletionCriterion],@@ -2465,7 +2480,7 @@ def carry_degraded_criterion_ids(degraded = [criterion.idfor criterion in criteria- if is_turn_unsatisfiable_fallback_degraded(criterion) or is_undecidable_judgment_degraded(criterion)+ if criterion.mint_degrade is not None and not is_contingent_missing_antecedent_degraded(criterion)]contingent_degraded = [criterion.id for criterion in criteria if is_contingent_missing_antecedent_degraded(criterion)diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex a759dfbee..50e5d3719 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -2318,6 +2318,17 @@ def _active_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion,return turn_state.decision.criteria+def _coverage_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion, ...]:+ criteria = list(_active_completion_criteria(ctx))+ policy = getattr(ctx, "request_policy", None)+ if isinstance(policy, RequestPolicy):+ known = {(criterion.id, criterion.output_path) for criterion in criteria}+ criteria.extend(+ criterion for criterion in policy.completion_criteria if (criterion.id, criterion.output_path) not in known+ )+ return tuple(criteria)++def _pre_run_gated_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriterion, ...]:"""Completion criteria whose requested output is observable before a run. A criterion whoseevidence comes from an independent run, registered output parameter, or artifact content is@@ -2326,7 +2337,7 @@ def _pre_run_gated_completion_criteria(ctx: AgentContext) -> tuple[CompletionCriis SKY-11591's."""return tuple(criterion- for criterion in _active_completion_criteria(ctx)+ for criterion in _coverage_completion_criteria(ctx)if criterion.requested_output_evidence_source not in _PRE_RUN_UNGATED_EVIDENCE_SOURCES)@@ -2341,15 +2352,15 @@ def _requested_output_paths_for_ctx(ctx: AgentContext) -> set[str]:for raw in repair_context.required_goal_value_pathsif isinstance(raw, str) and raw)- active_criteria = _active_completion_criteria(ctx)+ coverage_criteria = _coverage_completion_criteria(ctx)independent_run_evidence_paths = {_canonical_output_path(criterion.output_path)- for criterion in active_criteria+ for criterion in coverage_criteriaif criterion.requested_output_evidence_source == "independent_run_evidence" and criterion.output_path}non_independent_evidence_paths = {_canonical_output_path(criterion.output_path)- for criterion in active_criteria+ for criterion in coverage_criteriaif criterion.requested_output_evidence_source != "independent_run_evidence" and criterion.output_path}independent_only_paths = independent_run_evidence_paths - non_independent_evidence_paths@@ -2412,9 +2423,10 @@ def _requested_output_labels_by_path(ctx: AgentContext) -> dict[str, tuple[str,requested_paths = _requested_output_paths_for_ctx(ctx)labels_by_path: dict[str, tuple[str, ...]] = {}for criterion in _pre_run_gated_completion_criteria(ctx):- if criterion.output_path in requested_paths and criterion.outcome.strip():+ outcome = criterion.outcome.strip()+ if criterion.output_path in requested_paths and outcome:labels_by_path.setdefault(criterion.output_path, ())- labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)+ labels_by_path[criterion.output_path] += (outcome,)return labels_by_path@@ -2440,9 +2452,10 @@ def requested_scalar_output_extraction_plan(ctx: AgentContext) -> RequestedOutpureturn Nonelabels_by_path: dict[str, tuple[str, ...]] = {}for criterion in _pre_run_gated_completion_criteria(ctx):- if criterion.output_path in requested_paths and criterion.outcome.strip():+ outcome = criterion.outcome.strip()+ if criterion.output_path in requested_paths and outcome:labels_by_path.setdefault(criterion.output_path, ())- labels_by_path[criterion.output_path] += (criterion.outcome.strip(),)+ labels_by_path[criterion.output_path] += (outcome,)if set(labels_by_path) != requested_paths:return Nonereturn derive_requested_output_extraction_plan(diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 6b4d30fe6..cbbcc8fff 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -323,8 +323,8 @@ class CompletionCriterion:judgment_truth_condition: JudgmentTruthCondition | None = Nonerequested_output_floor_rekeyed: bool = Falsefloor_rekeyed_from_path: str | None = None- # Fresh request-slot contract metadata. Defaults keep versionless criteria on- # the byte-identical legacy path and are excluded from identity/dedup keys.+ # Typed request-slot metadata. Assertion identity, deduplication, and reconciliation+ # intentionally exclude these fields.request_slot_id: str | None = Nonepinability: Pinability | None = Nonemint_disposition: MintDisposition = "decidable"@@ -366,7 +366,6 @@ class RequestPolicy:classifier_retry_count: int = 0classifier_non_runtime_requested_output_evidence_sources: list[str] = field(default_factory=list)completion_contract_status: str = "absent"- classifier_contract_version: int | None = Nonerequest_slot_failure_kind: str | None = Nonedef graded_completion_criteria(self) -> list[CompletionCriterion]:@@ -413,10 +412,8 @@ class RequestPolicy:criterion for criterion in self.graded_completion_criteria() if criterion.output_path is not None]data["requested_output_criteria_count"] = len(requested_output_criteria)- if self.classifier_contract_version is not None:- data["classifier_contract_version"] = self.classifier_contract_version- if self.request_slot_failure_kind is not None:- data["request_slot_failure_kind"] = self.request_slot_failure_kind+ if self.request_slot_failure_kind is not None:+ data["request_slot_failure_kind"] = self.request_slot_failure_kindfor index, criterion in enumerate(requested_output_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):prefix = f"requested_output_criterion_{index}"data[f"{prefix}_id"] = criterion.id@@ -425,13 +422,12 @@ class RequestPolicy:data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not Noneif criterion.mint_degrade is not None:data[f"{prefix}_mint_degrade"] = criterion.mint_degrade+ data[f"{prefix}_mint_disposition"] = criterion.mint_dispositiondata[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_source- if self.classifier_contract_version is not None:- data[f"{prefix}_mint_disposition"] = criterion.mint_disposition- if criterion.request_slot_id is not None:- data[f"{prefix}_request_slot_id"] = criterion.request_slot_id- if criterion.pinability is not None:- data[f"{prefix}_pinability"] = criterion.pinability+ if criterion.request_slot_id is not None:+ data[f"{prefix}_request_slot_id"] = criterion.request_slot_id+ if criterion.pinability is not None:+ data[f"{prefix}_pinability"] = criterion.pinabilityif criterion.expected_output_shape:data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shapemint_degraded_criteria = [@@ -529,6 +525,21 @@ def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:)+def is_judgment_finalization_candidate(criterion: CompletionCriterion) -> bool:+ """Whether accepted-artifact evidence must decide or degrade this criterion."""+ if criterion.pinability in {"shapeless_valid", "unpinnable"}:+ return False+ return (+ criterion.judgment_truth_condition is not None+ or _is_judgment_boolean_criterion(criterion)+ or (+ criterion.kind == "validation_classification"+ and criterion.classification_output_key is not None+ and (isinstance(criterion.expected_classification, bool) or criterion.mint_disposition == "pending")+ )+ )++def typed_expected_output_value_key(value: ExpectedOutputValue | None) -> str:if isinstance(value, bool):return f"bool:{value}"@@ -1025,22 +1036,21 @@ def _classification_from_raw(completion_contract = completion_contract_raw.strip() if isinstance(completion_contract_raw, str) else Noneevidence_raw = raw.get("raw_secret_evidence")raw_secret_evidence = evidence_raw if isinstance(evidence_raw, str) and evidence_raw.strip() else None- has_contract_version = "contract_version" in raw- version_raw = raw.get("contract_version")- contract_version = (- version_raw if isinstance(version_raw, int) and not isinstance(version_raw, bool) and version_raw == 1 else None+ raw_criteria = raw.get("completion_criteria")+ claims_request_slots = isinstance(raw_criteria, list) and any(+ isinstance(item, dict) and _item_claims_request_slot(item) for item in raw_criteria)- if contract_version == 1 and request_slot_request is not None and request_slot_contract is not None:+ if request_slot_request is not None and request_slot_contract is not None:completion_criteria = _parse_fresh_request_slot_criteria(- raw.get("completion_criteria"),+ raw_criteria,request_slot_request=request_slot_request,request_slot_contract=request_slot_contract,)- elif has_contract_version:- completion_criteria = _fresh_request_slot_failure_criteria(raw.get("completion_criteria"))- request_slot_failure_kind = request_slot_failure_kind or "unsupported_contract_version"+ elif claims_request_slots:+ completion_criteria = _fresh_request_slot_failure_criteria(raw_criteria)+ request_slot_failure_kind = request_slot_failure_kind or "missing_request_slot_contract"else:- completion_criteria = _parse_completion_criteria(raw.get("completion_criteria"))+ completion_criteria = _parse_completion_criteria(raw_criteria)policy = RequestPolicy(testing_intent=testing_intent if testing_intent in _TESTING_INTENTS else "unspecified",authoring_intent=authoring_intent if authoring_intent in _AUTHORING_INTENTS else "author_now",@@ -1055,9 +1065,6 @@ def _classification_from_raw(clarification_reason=_coerce_clarification_reason(raw.get("clarification_reason")),classifier_status="success",classifier_failure_kind="none",- classifier_contract_version=contract_version- if contract_version is not None- else (-1 if has_contract_version else None),request_slot_failure_kind=request_slot_failure_kind,)if policy.credential_input_kind == "raw_secret":@@ -1157,11 +1164,20 @@ def _parse_completion_criterion_entries(raw: Any) -> list[tuple[dict[str, Any],if kind == "validation_classification"else None)+ boolean_classification = kind == "validation_classification" and (+ isinstance(expected_classification, bool)+ or (expected_classification is None and expected_output_shape == "goal_judgment_boolean")+ )if kind == "validation_classification":+ if boolean_classification and classification_output_key is not None:+ outcome = f"The run classifies whether {classification_output_key.replace('_', ' ')}."+ expected_classification = Noneoutput_path = Noneexpected_output_value = None- expected_output_shape = None- requested_output_evidence_source = "runtime_output"+ expected_output_shape = "goal_judgment_boolean" if boolean_classification else None+ requested_output_evidence_source = (+ "independent_run_evidence" if boolean_classification else "runtime_output"+ )elif isinstance(expected_output_value, bool) or expected_output_shape == "goal_judgment_boolean":requested_output_evidence_source = "independent_run_evidence"requested_output_path_mint_source: RequestedOutputPathMintSource | None = None@@ -1221,6 +1237,15 @@ def _parse_completion_criterion_entries(raw: Any) -> list[tuple[dict[str, Any],classification_output_key=classification_output_key,expected_classification=expected_classification,judgment_truth_condition=judgment_truth_condition,+ mint_disposition=(+ "pending"+ if (+ isinstance(expected_output_value, bool)+ or expected_output_shape == "goal_judgment_boolean"+ or boolean_classification+ )+ else "decidable"+ ),),))@@ -1617,6 +1642,18 @@ def _requested_output_criterion_id(output_path: str) -> str:return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"+def _requested_output_mint_state(+ expected_value: ExpectedOutputValue | None,+ expected_shape: ExpectedOutputShape | None,+ judgment_condition: JudgmentTruthCondition | None,+) -> tuple[MintDisposition, MintDegrade | None]:+ if isinstance(expected_value, bool) or expected_shape == "goal_judgment_boolean" or judgment_condition is not None:+ return "pending", None+ if expected_shape == "status_label" or (expected_value is None and expected_shape is not None):+ return "degraded", "undecidable_judgment"+ return "decidable", None++def _generic_completion_criterion(criterion: CompletionCriterion) -> bool:key = normalized_criterion_outcome_key(criterion.outcome)if is_fallback_floor_criterion(criterion):@@ -1734,21 +1771,31 @@ def _source_requested_output_corroborator(requested_fields: list[str],requested_output_paths: set[str],) -> CompletionCriterion | None:- for criterion in criteria:- if criterion.level != "run" or criterion.kind != "outcome" or criterion.method_mandated:- continue- if _validation_classification_output_path(criterion.output_path):- continue- if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(- criterion, requested_fields- ):- return replace(- criterion,- output_path=None,- expected_output_value=None,- expected_output_shape=None,- requested_output_corroborator=True,- )+ if len(requested_output_paths) == 1:+ for criterion in criteria:+ if (+ criterion.level != "run"+ or criterion.kind != "outcome"+ or criterion.method_mandated+ or criterion.output_path is None+ ):+ continue+ if is_judgment_finalization_candidate(criterion):+ continue+ if criterion.requested_output_evidence_source == "independent_run_evidence":+ continue+ if _validation_classification_output_path(criterion.output_path):+ continue+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(+ criterion, requested_fields+ ):+ return replace(+ criterion,+ output_path=None,+ expected_output_value=None,+ expected_output_shape=None,+ requested_output_corroborator=True,+ )for criterion in criteria:if is_fallback_floor_base_criterion(criterion):return replace(criterion, requested_output_corroborator=True)@@ -1775,7 +1822,9 @@ def _apply_classifier_typed_requested_output_corroborators(policy: RequestPolicyand criterion.kind == "outcome"and not criterion.method_mandatedand criterion.output_path is not None+ and criterion.requested_output_evidence_source != "independent_run_evidence"and not _validation_classification_output_path(criterion.output_path)+ and not is_judgment_finalization_candidate(criterion)and not criterion.id.startswith(_REQUESTED_OUTPUT_CRITERION_ID_PREFIX)]if not source_criteria:@@ -1885,23 +1934,34 @@ def _apply_requested_output_completion_criteria(forbidden_output_paths,)]- canonical_requested_criteria = [- CompletionCriterion(- id=_requested_output_criterion_id(output_path),- outcome=f"The returned record includes {field_label}.",- level="run",- output_path=output_path,- expected_output_value=value_by_output_path.get(output_path),- expected_output_shape=shape_by_output_path.get(output_path),- requested_output_evidence_source=source_by_output_path.get(output_path, "runtime_output"),- contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],- contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],- deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],- declared_deliverable_kind=declared_kind_by_output_path.get(output_path),- judgment_truth_condition=judgment_condition_by_output_path.get(output_path),+ canonical_requested_criteria: list[CompletionCriterion] = []+ for _field_name, output_path, field_label in requested_specs:+ expected_value = value_by_output_path.get(output_path)+ expected_shape = shape_by_output_path.get(output_path)+ judgment_condition = judgment_condition_by_output_path.get(output_path)+ mint_disposition, mint_degrade = _requested_output_mint_state(+ expected_value,+ expected_shape,+ judgment_condition,+ )+ canonical_requested_criteria.append(+ CompletionCriterion(+ id=_requested_output_criterion_id(output_path),+ outcome=f"The returned record includes {field_label}.",+ level="run",+ output_path=output_path,+ expected_output_value=expected_value,+ expected_output_shape=expected_shape,+ requested_output_evidence_source=source_by_output_path.get(output_path, "runtime_output"),+ contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],+ contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],+ deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],+ declared_deliverable_kind=declared_kind_by_output_path.get(output_path),+ judgment_truth_condition=judgment_condition,+ mint_degrade=mint_degrade,+ mint_disposition=mint_disposition,+ ))- for _field_name, output_path, field_label in requested_specs- ]criteria = preserved_criteria + canonical_requested_criteriaif not any(_non_requested_output_run_corroborator(criterion) for criterion in criteria):corroborator = _source_requested_output_corroborator(@@ -1942,18 +2002,22 @@ def _apply_validation_classification_completion_criteria(policy: RequestPolicy)target = _validation_classification_target_for_legacy_criterion(criterion)if target is not None:output_key, expected = target+ boolean_classification = isinstance(expected, bool)criterion = replace(criterion,kind="validation_classification",output_path=None,expected_output_value=None,- expected_output_shape=None,- requested_output_evidence_source="runtime_output",+ expected_output_shape="goal_judgment_boolean" if boolean_classification else None,+ requested_output_evidence_source=(+ "independent_run_evidence" if boolean_classification else "runtime_output"+ ),deliverable_kind=None,declared_deliverable_kind=None,terminal_action_family=None,classification_output_key=output_key,expected_classification=expected,+ mint_disposition="pending" if boolean_classification else "decidable",)if criterion.kind == "validation_classification":target_key = (@@ -2023,10 +2087,6 @@ def is_contingent_missing_antecedent_degraded(criterion: CompletionCriterion) ->return criterion.mint_degrade == "contingent_missing_antecedent"-def is_undecidable_judgment_degraded(criterion: CompletionCriterion) -> bool:- return criterion.mint_degrade == "undecidable_judgment"--def resolve_mint_degrade(stored_value: object,contingent_on: str | None,@@ -2422,7 +2482,7 @@ async def _classify_request(return _classifier_fallback_policy(ids,raw_secret_present=raw_secret_present,- failure_kind="provider_error",+ failure_kind=failure_kind if failure_kind != "none" else "provider_error",retry_count=retry_count,user_message=user_message,requested_output_path_aliases=requested_output_path_aliases,@@ -2430,32 +2490,26 @@ async def _classify_request(request_slot_contract: RequestSlotContractV1 | None = Nonerequest_slot_failure_kind: str | None = None- version_raw = raw_payload.get("contract_version")- if isinstance(version_raw, int) and not isinstance(version_raw, bool) and version_raw == 1:- raw_criteria = raw_payload.get("completion_criteria")- declared_request_slots = isinstance(raw_criteria, list) and any(- isinstance(item, dict) and _request_slot_anchor(item) is not None for item in raw_criteria- )- if declared_request_slots:- # This validation hop is deliberately sequential: the request-policy- # classifier first declares which criteria are request-owned slots, then- # the independent producer types their identity, plane, and pinability.- request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)- if request_slot_result.status == "success":- request_slot_contract = request_slot_result.contract- else:- request_slot_failure_kind = (- request_slot_result.failure_kind.value- if request_slot_result.failure_kind is not None- else "unknown"- )- LOG.warning(- "request-policy request-slot producer failed",- failure_kind=request_slot_failure_kind,- attempts=request_slot_result.attempts,- )+ raw_criteria = raw_payload.get("completion_criteria")+ declared_request_slots = isinstance(raw_criteria, list) and any(+ isinstance(item, dict) and _request_slot_anchor(item) is not None for item in raw_criteria+ )+ if declared_request_slots:+ # This validation hop is deliberately sequential: the request-policy+ # classifier first declares which criteria are request-owned slots, then+ # the independent producer types their identity, plane, and pinability.+ request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)+ if request_slot_result.status == "success":+ request_slot_contract = request_slot_result.contractelse:- request_slot_failure_kind = "no_declared_slots"+ request_slot_failure_kind = (+ request_slot_result.failure_kind.value if request_slot_result.failure_kind is not None else "unknown"+ )+ LOG.warning(+ "request-policy request-slot producer failed",+ failure_kind=request_slot_failure_kind,+ attempts=request_slot_result.attempts,+ )policy = _classification_from_raw(raw_payload,request_slot_request=request_slot_request,@@ -2514,13 +2568,6 @@ async def _classify_request(policy.clarification_reason = "none"policy.requires_user_clarification = Falsepolicy.raw_secret_evidence = None- if policy.classifier_contract_version is None:- # Legacy corroborators are inference-based compatibility behavior. A fresh- # contract remains fail-closed even when its request-slot producer fails:- # fresh criteria degrade above instead of re-entering legacy guessing.- _apply_requested_output_completion_criteria(policy, user_message, requested_output_path_aliases)- _apply_validation_classification_completion_criteria(policy)- _apply_classifier_typed_requested_output_corroborators(policy)_degrade_pathless_contingent_criteria(policy)policy.completion_contract_status = ("present" if policy.completion_contract or policy.graded_completion_criteria() else "absent"diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.pyindex 1053bddd3..954b4d7bb 100644--- a/skyvern/forge/sdk/copilot/tools/completion.py+++ b/skyvern/forge/sdk/copilot/tools/completion.py@@ -37,6 +37,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_terminal_goal_record_corroboration,grade_terminal_goal_record_criteria,grade_validation_classification_criteria,+ gradeable_completion_criteria,is_fallback_floor_base_criterion,is_registered_download_completion_criterion,only_degraded_blocking,@@ -237,17 +238,26 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriterpolicy = _completion_request_policy(copilot_ctx)# A method-mandated criterion asserts HOW the goal was reached; the outcome# judge sees only end-state evidence.- criteria = policy.graded_completion_criteria() if policy is not None else []+ formed_criteria = policy.graded_completion_criteria() if policy is not None else []+ criteria = gradeable_completion_criteria(formed_criteria)authored_output_criteria = _authored_output_contract_criteria(copilot_ctx)- if authored_output_criteria and (not criteria or all(is_fallback_floor_criterion(c) for c in criteria)):+ if authored_output_criteria and (+ not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)+ ):return authored_output_criteriaif _accepted_staged_output_contract_missing(copilot_ctx) and (- not criteria or all(is_fallback_floor_criterion(c) for c in criteria)+ not formed_criteria or all(is_fallback_floor_criterion(c) for c in formed_criteria)):return [_authored_output_contract_missing_criterion()]return criteria+def _carry_degraded_ids(copilot_ctx: Any, verification: CompletionVerificationResult) -> CompletionVerificationResult:+ policy = _completion_request_policy(copilot_ctx)+ formed_criteria = policy.graded_completion_criteria() if policy is not None else []+ return carry_degraded_criterion_ids(verification, formed_criteria)++def _authored_output_contract_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:paths = _authored_output_contract_paths(copilot_ctx)return [@@ -459,7 +469,7 @@ async def _maybe_run_completion_verification_from_page_observation()if _classifier_status(copilot_ctx) == "fallback" and not run_criteria:verification = _no_gradeable_run_plane_result(criterion_ids)- verification = carry_degraded_criterion_ids(verification, criteria)+ verification = _carry_degraded_ids(copilot_ctx, verification)verification = carry_floor_rekeyed_criterion_ids(verification, criteria)copilot_ctx.completion_verification_result = verificationrecord_completion_verification(copilot_ctx, verification)@@ -616,7 +626,7 @@ async def _maybe_run_completion_verification_from_page_observation(requested_output_criteria_count=len(requested_output_criteria),)- verification = carry_degraded_criterion_ids(verification, criteria)+ verification = _carry_degraded_ids(copilot_ctx, verification)verification = carry_floor_rekeyed_criterion_ids(verification, criteria)if (isinstance(existing, CompletionVerificationResult)@@ -1355,7 +1365,7 @@ async def _maybe_run_completion_verification(verification = await _completion_verification_from_run_result(copilot_ctx, result, handler_start, criteria)if verification is None:return None- verification = carry_degraded_criterion_ids(verification, criteria)+ verification = _carry_degraded_ids(copilot_ctx, verification)verification = carry_floor_rekeyed_criterion_ids(verification, criteria)run_data = result.get("data")return _carry_floor_rekeyed_backing(diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.pyindex 0ee848b39..7c2cd9d6f 100644--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py@@ -718,7 +718,7 @@ class WorkflowParametersRepository(BaseRepository):organization_id: str,workflow_copilot_chat_id: str,goal_epoch: int,- criteria: list[dict],+ criteria: list[dict[str, Any]],source_turn_id: str | None = None,source_goal_text: str | None = None,consecutive_all_no_evidence: int = 0,diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.pyindex 89cc8ac9b..ff8059cb6 100644--- a/skyvern/forge/sdk/schemas/workflow_copilot.py+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py@@ -29,7 +29,7 @@ class WorkflowCopilotCompletionCriteriaSet(BaseModel):workflow_copilot_chat_id: strgoal_epoch: intstatus: str- criteria: list[dict]+ criteria: list[dict[str, Any]]source_turn_id: str | None = Nonesource_goal_text: str | None = Noneconsecutive_all_no_evidence: int = 0diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex ed3962076..87710324b 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -17,6 +17,7 @@ from structlog.testing import capture_logsfrom skyvern.config import settingsfrom skyvern.forge.sdk.api.llm.exceptions import LLMProviderErrorfrom skyvern.forge.sdk.copilot import agent as agent_module+from skyvern.forge.sdk.copilot import request_policy as request_policy_modulefrom skyvern.forge.sdk.copilot import tools as tools_modulefrom skyvern.forge.sdk.copilot.agent import (_VERIFIED_WORKFLOW_SUCCESS_REPLY,@@ -72,6 +73,7 @@ from skyvern.forge.sdk.copilot.request_policy import (_REDACTED_REFUSED_SECRET_TURN,TRANSCRIPT_ANCHOR_CHAR_CAP,CompletionCriterion,+ JudgmentTruthCondition,RequestPolicy,_classifier_fallback_policy,_classify_request,@@ -84,6 +86,7 @@ from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REAfrom skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_modulefrom skyvern.forge.sdk.copilot.tools.completion import (_authored_output_contract_criteria,+ _carry_degraded_ids,_completion_verification_criteria,_maybe_run_completion_verification,)@@ -3910,6 +3913,28 @@ class TestRequestPolicyCredentialResolution:assert policy.classifier_retry_count == 0assert policy.completion_contract_status == "present"+ @pytest.mark.asyncio+ async def test_request_policy_classifier_malformed_payload_preserves_timeout_failure(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ async def malformed_after_timeout(*_args: object, **_kwargs: object) -> tuple[str, str, int]:+ return "not-json", "timeout", 1++ monkeypatch.setattr(request_policy_module, "_run_request_policy_classifier", malformed_after_timeout)++ policy = await _classify_request(+ user_message="Build a workflow for https://example.com.",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ handler=AsyncMock(),+ )++ assert policy.classifier_status == "fallback"+ assert policy.classifier_failure_kind == "timeout"+ assert policy.classifier_retry_count == 1+ assert policy.completion_contract_status == "present"+@pytest.mark.asyncioasync def test_request_policy_classifier_retries_transient_error_then_succeeds(self) -> None:class RateLimitError(Exception):@@ -6512,6 +6537,89 @@ class TestClassifierFallbackCompletionFloor:assert policy.completion_criteriaassert _completion_verification_criteria(ctx) == [policy.completion_criteria[0]]+ def test_degraded_judgment_does_not_hide_contradictory_required_outputs(self) -> None:+ reached = CompletionCriterion(id="c0", outcome="the requested page is reached")+ degraded_judgment = CompletionCriterion(+ id="public_form_exists",+ outcome="the public-form judgment is returned",+ mint_degrade="undecidable_judgment",+ )+ required_output = CompletionCriterion(id="visible_page_path_label", outcome="the visible path is returned")+ contradiction = "evidence_contradicts"+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[reached.id, required_output.id],+ verdicts=[+ CriterionVerdict(criterion_id=reached.id, state="satisfied", reason_code="evidence_confirms"),+ CriterionVerdict(criterion_id=required_output.id, state="unsatisfied", reason_code=contradiction),+ ],+ )++ carried = _carry_degraded_ids(+ SimpleNamespace(+ request_policy=RequestPolicy(completion_criteria=[reached, degraded_judgment, required_output])+ ),+ verification,+ )+ assert carried.degraded_criterion_ids == [degraded_judgment.id]+ assert carried.is_fully_satisfied() is False++ judgment_only = _carry_degraded_ids(+ SimpleNamespace(request_policy=RequestPolicy(completion_criteria=[reached, degraded_judgment])),+ CompletionVerificationResult(+ status="evaluated", criterion_ids=[reached.id], verdicts=verification.verdicts[:1]+ ),+ )+ assert judgment_only.degraded_criterion_ids == [degraded_judgment.id]+ assert judgment_only.is_fully_satisfied() is True++ def test_p9_degraded_judgments_suppress_their_generic_corroborator(self) -> None:+ reached = CompletionCriterion(id="c2", outcome="the visible page path label is returned")+ degraded = [+ CompletionCriterion(+ id=criterion_id,+ outcome=outcome,+ kind="validation_classification",+ classification_output_key=output_key,+ judgment_truth_condition=truth_condition,+ mint_degrade="undecidable_judgment",+ mint_disposition="degraded",+ )+ for criterion_id, outcome, output_key, truth_condition in (+ ("c0", "whether a public form exists", "public_form_exists", None),+ (+ "c1",+ "whether the path is login-only",+ "login_only",+ JudgmentTruthCondition(predicate="login_gate_blocks_target", polarity_when_holds=True),+ ),+ )+ ]+ judgment_corroborator = CompletionCriterion(+ id="c3",+ outcome="the recommended next action is returned",+ requested_output_evidence_source="independent_run_evidence",+ requested_output_corroborator=True,+ )+ policy = RequestPolicy(completion_criteria=[*degraded, reached, judgment_corroborator])+ ctx = SimpleNamespace(request_policy=policy)++ assert _completion_verification_criteria(ctx) == [reached]++ carried = _carry_degraded_ids(+ ctx,+ CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[reached.id],+ verdicts=[+ CriterionVerdict(criterion_id=reached.id, state="satisfied", reason_code="evidence_confirms"),+ ],+ ),+ )++ assert carried.degraded_criterion_ids == ["c0", "c1"]+ assert carried.is_fully_satisfied() is True+def test_credential_aware_floor_adds_one_run_plane_criterion(self) -> None:base = build_classifier_fallback_floor([])credentialed = build_classifier_fallback_floor(["cred_1"])diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 3c609de18..751d025ae 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -81,6 +81,8 @@ from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llmfrom skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTargetfrom skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,+ CriterionKind,+ CriterionLevel,JudgmentTruthCondition,RequestedOutputEvidenceSource,RequestPolicy,@@ -112,6 +114,24 @@ from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetectedfrom skyvern.forge.sdk.workflow.models.block import CodeBlock+def _typed_completion_criterion(+ *,+ id: str,+ output_path: str,+ level: CriterionLevel,+ method_mandated: bool,+ kind: CriterionKind,+) -> CompletionCriterion:+ return CompletionCriterion(+ id=id,+ outcome=f"The run returns {output_path}.",+ output_path=output_path,+ level=level,+ method_mandated=method_mandated,+ kind=kind,+ )++def _yaml(body: str) -> str:return textwrap.dedent(body).strip() + "\n"@@ -3846,21 +3866,21 @@ class TestCodeRepairProgressClassification:label = "lookup_entry"ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.npi",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_locations",output_path="output.locations",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_statuses",output_path="output.statuses",level="run",@@ -3931,7 +3951,7 @@ class TestCodeRepairProgressClassification:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -4110,14 +4130,14 @@ class TestCodeRepairProgressClassification:required_paths = {"output.account_number", "output.confirmation_number"}ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_account",output_path="output.account_number",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_confirmation",output_path="output.confirmation_number",level="run",@@ -4451,7 +4471,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -4641,7 +4661,7 @@ class TestCodeRepairProgressClassification:ctx.turn_id = "stable-contract-key"ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -4701,7 +4721,7 @@ class TestCodeRepairProgressClassification:ctx.turn_id = turn_idctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path=output_path,level="run",@@ -4732,14 +4752,14 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_flags",output_path="output.flags",level="run",@@ -4785,7 +4805,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.npi",level="run",@@ -4849,7 +4869,7 @@ class TestCodeRepairProgressClassification:child_paths = ["output.npi"]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.npi",level="run",@@ -4978,7 +4998,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_account",output_path="output.account_number",level="run",@@ -5178,7 +5198,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.requested_value",level="run",@@ -6175,7 +6195,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.requested_value",level="run",@@ -6603,14 +6623,14 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_flags",output_path="output.flags",level="run",@@ -6658,7 +6678,7 @@ class TestCodeRepairProgressClassification:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -6717,7 +6737,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_npi",output_path="output.npi",level="run",@@ -6792,7 +6812,7 @@ class TestCodeRepairProgressClassification:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -6862,7 +6882,7 @@ class TestCodeRepairProgressClassification:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -6932,7 +6952,7 @@ class TestCodeRepairProgressClassification:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -7001,7 +7021,7 @@ class TestCodeRepairProgressClassification:ctx.scout_trajectory = []ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -7058,7 +7078,7 @@ class TestCodeRepairProgressClassification:)ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id=f"requested_{index}",output_path=path,level="run",@@ -7108,14 +7128,14 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.npi",level="run",method_mandated=False,kind="outcome",),- SimpleNamespace(+ _typed_completion_criterion(id="requested_statuses",output_path="output.statuses",level="run",@@ -7263,7 +7283,7 @@ class TestCodeRepairProgressClassification:ctx = _code_only_ctx()ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.requested_only",level="run",@@ -8018,7 +8038,7 @@ def _spine_actuation_ctx() -> CopilotContext:]ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_value",output_path="output.record_id",level="run",@@ -13293,7 +13313,7 @@ class TestWholeTrajectoryImposition:)ctx.request_policy = RequestPolicy(completion_criteria=[- SimpleNamespace(+ _typed_completion_criterion(id="requested_npi",output_path="output.npi",level="run",diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.pyindex acc0dacab..628db0951 100644--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py@@ -55,6 +55,7 @@ from skyvern.forge.sdk.copilot.enforcement import ()from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,+ JudgmentTruthCondition,RequestPolicy,_parse_completion_criteria,normalized_criterion_outcome_key,@@ -315,6 +316,118 @@ def test_criteria_json_round_trip_preserves_fields(criteria: tuple[CompletionCriassert criteria_from_json(criteria_to_json(criteria)) == criteria+def test_typed_criteria_list_round_trip_preserves_grading_metadata() -> None:+ criterion = CompletionCriterion(+ id="c0",+ outcome="The visible page path label is returned.",+ output_path="output.visible_page_path_label",+ expected_output_value="Public start-service path",+ pinability="unpinnable",+ mint_degrade="undecidable_judgment",+ mint_disposition="degraded",+ )++ raw = criteria_to_json([criterion])++ assert isinstance(raw, list)+ assert raw[0]["pinability"] == "unpinnable"+ assert criteria_from_json(raw) == (criterion,)+++def test_typed_boolean_validation_classification_round_trip_preserves_shape_and_evidence() -> None:+ """Typed boolean classifications persist goal_judgment_boolean / independent_run_evidence."""+ criterion = CompletionCriterion(+ id="c0",+ outcome="The run classifies whether a public form exists.",+ kind="validation_classification",+ classification_output_key="public_form_exists",+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="independent_run_evidence",+ judgment_truth_condition=JudgmentTruthCondition(+ predicate="login_gate_blocks_target", polarity_when_holds=False+ ),+ pinability="pinned",+ mint_disposition="pending",+ )++ raw = criteria_to_json([criterion])++ assert raw[0]["expected_output_shape"] == "goal_judgment_boolean"+ assert raw[0]["requested_output_evidence_source"] == "independent_run_evidence"+ reloaded = criteria_from_json(raw)+ assert reloaded == (criterion,)+ assert reloaded[0].expected_output_shape == "goal_judgment_boolean"+ assert reloaded[0].requested_output_evidence_source == "independent_run_evidence"+++def test_untyped_boolean_validation_classification_reload_normalizes_shape_and_evidence() -> None:+ """Rows without typed mint metadata carry no reliable shape/evidence markers."""+ legacy = {+ "id": "c0",+ "outcome": "The run classifies whether a public form exists.",+ "kind": "validation_classification",+ "classification_output_key": "public_form_exists",+ "expected_output_shape": "goal_judgment_boolean",+ "requested_output_evidence_source": "independent_run_evidence",+ }++ reloaded = criteria_from_json([legacy])++ assert reloaded[0].expected_output_shape is None+ assert reloaded[0].requested_output_evidence_source == "runtime_output"+++def test_typed_metadata_is_self_describing_without_a_storage_version() -> None:+ legacy = {"id": "c0", "outcome": "done", "output_path": "output.done"}+ decorated = {+ **legacy,+ "pinability": "unpinnable",+ "mint_disposition": "degraded",+ }++ assert criteria_from_json([legacy]) == (CompletionCriterion(id="c0", outcome="done", output_path="output.done"),)+ assert criteria_from_json([decorated]) == (+ CompletionCriterion(+ id="c0",+ outcome="done",+ output_path="output.done",+ pinability="unpinnable",+ mint_disposition="degraded",+ ),+ )+++def test_reconciliation_identity_ignores_pinability_and_derived_mint_state() -> None:+ stored = StoredCriteriaSet(+ set_id="wccs_1",+ goal_epoch=1,+ criteria=(+ CompletionCriterion(+ id="s0",+ outcome="done",+ output_path="output.done",+ expected_output_value=True,+ pinability="pinned",+ ),+ ),+ )+ fresh = CompletionCriterion(+ id="f0",+ outcome="done",+ output_path="output.done",+ expected_output_value=True,+ pinability="unpinnable",+ mint_degrade="undecidable_judgment",+ mint_disposition="degraded",+ )++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2), [fresh], actionable=True+ )++ assert decision.action == "adopt_stored"++def test_criteria_from_json_degrades_stored_pathless_contingent() -> None:(criterion,) = criteria_from_json([@@ -1159,6 +1272,17 @@ def test_reconcile_on_context_adopts_stored_criteria_onto_policy() -> None:assert ctx.completion_criteria_turn_state.decision.reason == "kept"+def test_reconcile_on_context_creates_typed_criteria_without_a_contract_version() -> None:+ policy = RequestPolicy(completion_criteria=[_criterion("c0", "the item is in the cart")])+ ctx = _ctx()++ _reconcile_completion_criteria_on_context(ctx, policy, _policy_inputs(StoredCriteriaSnapshot()))++ plan = plan_persistence(ctx.completion_criteria_turn_state)+ assert plan is not None+ assert plan.create_criteria == tuple(policy.completion_criteria)++def test_reconcile_on_context_skips_without_snapshot() -> None:policy = RequestPolicy(completion_criteria=[_criterion("c0", "the item is in the cart")])ctx = _ctx()diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 794df8dac..352c288c8 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -60,6 +60,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (grade_structured_record_criteria,grade_terminal_goal_record_criteria,grade_validation_classification_criteria,+ gradeable_completion_criteria,is_registered_download_completion_criterion,only_degraded_blocking,registered_download_completion_criterion,@@ -3390,6 +3391,38 @@ def test_zero_requested_output_criteria_credit_fires_only_with_payload() -> Noneassert zero_requested_output_criteria_credit(satisfied, has_meaningful_registered_output=False) is False+def test_gradeable_completion_criteria_excludes_pending_and_degraded_judgments() -> None:+ reached = CompletionCriterion(id="reached", outcome="The requested page is reached.")+ pending_judgment = CompletionCriterion(+ id="pending",+ outcome="The pending requested judgment is returned.",+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="independent_run_evidence",+ mint_disposition="pending",+ )+ degraded_judgment = CompletionCriterion(+ id="judgment",+ outcome="The requested judgment is returned.",+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="independent_run_evidence",+ mint_degrade="undecidable_judgment",+ mint_disposition="degraded",+ )+ pending_classification = CompletionCriterion(+ id="classification",+ outcome="The requested classification is returned.",+ kind="validation_classification",+ classification_output_key="login_only",+ mint_disposition="pending",+ )++ assert gradeable_completion_criteria([reached, pending_judgment, degraded_judgment, pending_classification]) == [+ reached+ ]+ assert gradeable_completion_criteria([pending_judgment]) == []+ assert gradeable_completion_criteria([degraded_judgment]) == []++def test_zero_requested_output_criteria_credit_ignored_when_criteria_formed() -> None:with_criteria = replace(_evaluated(("bill", True)), requested_output_criteria_count=1)diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.pyindex 64afac334..e9f854daa 100644--- a/tests/unit/test_copilot_enforcement_pruning.py+++ b/tests/unit/test_copilot_enforcement_pruning.py@@ -46,6 +46,7 @@ from skyvern.forge.sdk.copilot.enforcement import (_maybe_synthesized_block_offer_msg,_needs_suspicious_success_nudge,_prune_input_list,+ _requested_output_paths_for_ctx,_should_block_mutating_tool_after_synthesized_offer,_should_force_advisory_run_dispatch,_should_force_synthesized_block_persistence,@@ -2476,17 +2477,18 @@ class TestScoutOutputCoverageGate:def test_independent_run_evidence_is_exempt_from_repair_context(self) -> None:independent = CompletionCriterion(- id="output.login_gate_present",- outcome="whether a login gate blocked the target is recorded",- output_path="output.login_gate_present",+ id="output.login_gate_blocks_target",+ outcome="the login-gate judgment is independently observed after the run",+ output_path="output.login_gate_blocks_target",+ expected_output_shape="goal_judgment_boolean",requested_output_evidence_source="independent_run_evidence",)- runtime = _criterion("output.document_name", "the order status document name is captured")+ runtime = _criterion("output.document_name", "the document name is captured")ctx = self._authoring_ctx(independent, runtime)ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(block_label="extract_order",reason_code="metadata_reject",- required_goal_value_paths=["login_gate_present", "document_name"],+ required_goal_value_paths=["login_gate_blocks_target", "document_name"],)assert uncovered_requested_output_paths(ctx) == {"output.document_name"}@@ -2556,6 +2558,22 @@ class TestScoutOutputCoverageGate:assert uncovered_requested_output_paths(ctx) == {"output.document_name"}assert synthesized_trajectory_is_goal_complete(ctx) is False+ def test_pathless_post_run_criterion_does_not_erase_repair_output_field(self) -> None:+ independent = CompletionCriterion(+ id="c_independent",+ outcome="the judgment is independently observed after the run",+ output_path=None,+ requested_output_evidence_source="independent_run_evidence",+ )+ ctx = self._authoring_ctx(independent)+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(+ block_label="extract_order",+ reason_code="metadata_reject",+ required_goal_value_paths=["field"],+ )++ assert _requested_output_paths_for_ctx(ctx) == {"output.field"}+def test_runtime_output_stays_gated_alongside_exempt_source(self) -> None:registered = CompletionCriterion(id="output.confirmation_number",diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex c08d310fe..e32ffdf48 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -443,6 +443,7 @@ class TestMCPFailedStepLoopDetection:last_code_authoring_repair_context=None,scouted_output_covered_paths=set(),reached_download_target=None,+ request_policy=None,)server = SkyvernOverlayMCPServer(transport=MagicMock(),diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.pyindex 52e077e24..ea3e72844 100644--- a/tests/unit/test_copilot_output_contract_actuation.py+++ b/tests/unit/test_copilot_output_contract_actuation.py@@ -1954,6 +1954,24 @@ def test_non_minted_registered_download_path_still_requires_author_coverage() ->assert wu._requested_output_child_paths(ctx) == {"output.downloaded_files"}+def test_independent_validation_classification_is_post_run_only() -> None:+ ctx = _antecedent_ctx(+ CompletionCriterion(+ id="c1",+ outcome="The run classifies whether the path is login-only.",+ kind="validation_classification",+ classification_output_key="login_only",+ expected_classification=True,+ expected_output_shape="goal_judgment_boolean",+ requested_output_evidence_source="independent_run_evidence",+ mint_disposition="pending",+ )+ )++ assert wu._requested_output_child_paths(ctx) == set()+ assert wu._output_contract_required_paths_source(ctx).observation_paths == set()++def test_runtime_repair_contract_carries_declaration_lane_with_stable_signature() -> None:ctx = _antecedent_ctx(CompletionCriterion(diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex 243521a41..fbd1c4648 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -20,10 +20,10 @@ from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,JudgmentTruthCondition,RequestPolicy,+ _apply_classifier_typed_requested_output_corroborators,_apply_requested_output_completion_criteria,_apply_validation_classification_completion_criteria,_classifier_fallback_policy,- _classify_request,_criterion_grounding_mode,_degrade_pathless_contingent_criteria,_parse_completion_criteria,@@ -41,22 +41,31 @@ async def _policy_for_message(*,config: CopilotConfig | None = None,) -> RequestPolicy:- async def _handler(**_: Any) -> dict[str, Any]:- return {- "testing_intent": "require_test",- "credential_input_kind": "none",- "requires_user_clarification": False,- "completion_criteria": criteria,- }+ """Exercise the retained deterministic fallback canonicalizers directly.- return await _classify_request(- user_message,- workflow_yaml="",- chat_history=[],- global_llm_context="",- handler=_handler,- config=config,+ Fresh classifier responses use typed request-slot binding instead; those integration+ semantics are covered by test_copilot_request_policy_pinability_consumer.py.+ """+ policy = RequestPolicy(+ testing_intent="require_test",+ classifier_status="success",+ classifier_failure_kind="none",+ completion_criteria=_parse_completion_criteria(criteria),)+ aliases = config.requested_output_path_aliases if config is not None else {}+ _apply_requested_output_completion_criteria(policy, user_message, aliases)+ _apply_validation_classification_completion_criteria(policy)+ _apply_classifier_typed_requested_output_corroborators(policy)+ _degrade_pathless_contingent_criteria(policy)+ policy.classifier_non_runtime_requested_output_evidence_sources = sorted(+ {+ criterion.requested_output_evidence_source+ for criterion in policy.completion_criteria+ if criterion.output_path is not None and criterion.requested_output_evidence_source != "runtime_output"+ }+ )+ policy.completion_contract_status = "present" if policy.graded_completion_criteria() else "absent"+ return policydef _stored(*criteria: CompletionCriterion) -> StoredCriteriaSet:@@ -76,6 +85,63 @@ def _criteria_fingerprint(criteria: list[CompletionCriterion]) -> str:return hashlib.sha256(payload.encode("utf-8")).hexdigest()+def test_p9_exact_value_unpinnable_output_mint_degrades() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c0",+ outcome="The visible page path label is returned.",+ output_path="output.visible_page_path_label",+ expected_output_value="Public start-service path",+ expected_output_shape="status_label",+ pinability="unpinnable",+ )+ ]+ )++ _apply_requested_output_completion_criteria(policy, "Return visible_page_path_label.")++ criterion = _criteria_for_path(policy, "output.visible_page_path_label")[0]+ assert criterion.mint_disposition == "degraded"+ assert criterion.mint_degrade == "undecidable_judgment"+++def test_p9_classifier_drift_normalizes_to_one_contract_shape() -> None:+ prompt = "Return public_form_exists, visible_page_path_label, and recommended_next_action."+ classifier_shapes = [+ [+ CompletionCriterion(id="c0", outcome="a public form exists for starting service"),+ CompletionCriterion(id="c1", outcome="the visible page path label is returned"),+ ],+ [+ CompletionCriterion(+ id="c0",+ outcome="a public form exists for starting service",+ output_path="output.public_form_exists",+ ),+ CompletionCriterion(id="c1", outcome="the recommended next action is returned"),+ ],+ [+ CompletionCriterion(id="c0", outcome="a public form exists for starting service"),+ CompletionCriterion(id="c1", outcome="the visible page path label is returned"),+ CompletionCriterion(+ id="c2",+ outcome="the public form result is returned",+ output_path="output.public_form_exists",+ ),+ CompletionCriterion(id="c3", outcome="the recommended next action is returned"),+ ],+ ]+ policies = [RequestPolicy(completion_criteria=criteria) for criteria in classifier_shapes]++ for policy in policies:+ _apply_requested_output_completion_criteria(policy, prompt)++ serialized = [criteria_to_json(policy.completion_criteria) for policy in policies]+ assert [len(policy.completion_criteria) for policy in policies] == [3, 3, 3]+ assert serialized == [serialized[0], serialized[0], serialized[0]]++def _requested_output_subset(policy: RequestPolicy, requested_output_paths: set[str]) -> list[CompletionCriterion]:return [criterion@@ -654,7 +720,7 @@ def test_requested_output_corroborator_respects_completion_criteria_cap() -> Non}-def test_requested_output_corroborator_survives_when_requested_outputs_exceed_cap() -> None:+def test_requested_output_canonicalization_drops_redundant_corroborator_when_outputs_exceed_cap() -> None:policy = RequestPolicy(completion_criteria=[CompletionCriterion(@@ -686,9 +752,7 @@ def test_requested_output_corroborator_survives_when_requested_outputs_exceed_caassert {criterion.output_path for criterion in _requested_output_subset(policy, requested_output_paths)} == requested_output_paths- assert len(corroborators) == 1- assert corroborators[0].id == "name_source"- assert corroborators[0].requested_output_corroborator is True+ assert corroborators == []@pytest.mark.asyncio@@ -868,7 +932,8 @@ def test_parse_completion_criteria_preserves_validation_classification_contract(assert len(parsed) == 2assert parsed[0].kind == "validation_classification"assert parsed[0].classification_output_key == "login_gated"- assert parsed[0].expected_classification is True+ assert parsed[0].expected_classification is None+ assert parsed[0].mint_disposition == "pending"assert parsed[1].kind == "outcome"@@ -2160,7 +2225,8 @@ def test_scope_boundary_boolean_on_login_only_rekinds_and_drops_judgment_invariaassert promoted.kind == "validation_classification"assert promoted.classification_output_key == "login_only"assert promoted.expected_output_value is None- assert promoted.requested_output_evidence_source == "runtime_output"+ assert promoted.requested_output_evidence_source == "independent_run_evidence"+ assert promoted.mint_disposition == "pending"def test_scope_boundary_boolean_on_non_enumerated_path_keeps_judgment_invariant() -> None:diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.pyindex 21a368aca..b7b5123f8 100644--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py@@ -68,7 +68,6 @@ def _envelope() -> RequestSlotEnvelopeV1:def _fresh_payload() -> dict[str, Any]:return {- "contract_version": 1,"testing_intent": "require_test","credential_input_kind": "none","requires_user_clarification": False,@@ -109,7 +108,6 @@ def test_fresh_contract_consumes_declared_pinability_without_guessed_polarity()request_slot_contract=contract,)- assert policy.classifier_contract_version == 1assert [criterion.request_slot_id for criterion in policy.completion_criteria] == [slot.slot_id for slot in contract.slots]@@ -276,14 +274,12 @@ def test_followup_source_digest_change_adopts_stored_semantic_criteria() -> Noneassert decision.superseded_set_id is None-def test_invalid_fresh_version_fails_safe_without_entering_legacy_minting() -> None:+def test_unbound_typed_criteria_fail_safe_without_entering_legacy_minting() -> None:payload = _fresh_payload()- payload["contract_version"] = "1"policy = _classification_from_raw(payload)- assert policy.classifier_contract_version == -1- assert policy.request_slot_failure_kind == "unsupported_contract_version"+ assert policy.request_slot_failure_kind == "missing_request_slot_contract"assert all(criterion.output_path is None for criterion in policy.completion_criteria)assert all(criterion.expected_output_value is None for criterion in policy.completion_criteria)assert all(criterion.expected_classification is None for criterion in policy.completion_criteria)@@ -293,7 +289,6 @@ def test_invalid_fresh_version_fails_safe_without_entering_legacy_minting() -> Ndef test_fresh_slot_failure_preserves_non_output_outcome() -> None:policy = _classification_from_raw({- "contract_version": 1,"completion_criteria": [{"outcome": "The application is submitted."}],},request_slot_failure_kind="invalid_output",@@ -348,7 +343,6 @@ async def test_request_slot_producer_failure_degrades_without_legacy_guessing()policy = await _classify_request(P9_REQUEST, "", [], "", handler)assert calls == ["workflow-copilot-request-policy", *([REQUEST_SLOT_PROMPT_NAME] * 4)]- assert policy.classifier_contract_version == 1assert policy.request_slot_failure_kind == "invalid_output"assert all(criterion.output_path is None for criterion in policy.completion_criteria)assert all(criterion.expected_output_value is None for criterion in policy.completion_criteria)@@ -357,7 +351,7 @@ async def test_request_slot_producer_failure_degrades_without_legacy_guessing()@pytest.mark.asyncio-async def test_versionless_classifier_payload_never_invokes_request_slot_producer() -> None:+async def test_classifier_payload_without_request_slots_does_not_invoke_request_slot_producer() -> None:calls: list[str] = []async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:@@ -369,9 +363,8 @@ async def test_versionless_classifier_payload_never_invokes_request_slot_produce"completion_criteria": [{"outcome": "The requested workflow is complete."}],}- policy = await _classify_request("Build the workflow.", "", [], "", handler)+ await _classify_request("Build the workflow.", "", [], "", handler)- assert policy.classifier_contract_version is Noneassert calls == ["workflow-copilot-request-policy"]@@ -382,7 +375,6 @@ async def test_fresh_non_output_request_does_not_invoke_request_slot_producer()async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:calls.append(prompt_name)return {- "contract_version": 1,"testing_intent": "require_test","credential_input_kind": "none","requires_user_clarification": False,@@ -391,7 +383,7 @@ async def test_fresh_non_output_request_does_not_invoke_request_slot_producer()policy = await _classify_request("Submit the application.", "", [], "", handler)- assert policy.request_slot_failure_kind == "no_declared_slots"+ assert policy.request_slot_failure_kind is Noneassert policy.completion_criteria[0].outcome == "The application is submitted."assert policy.completion_criteria[0].mint_degrade is Noneassert calls == ["workflow-copilot-request-policy"]diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex 894034aee..cf4203084 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -94,6 +94,8 @@ class TestRequestPolicyPromptStructure:"expected_output_shape, requested_output_evidence_source, kind, terminal_action_family, ""classification_output_key, expected_classification, judgment_predicate, judgment_polarity_when_holds}") in rendered+ assert "typed request-slot producer owns its final identity, path, plane, and pinability" in rendered+ assert "Do not infer or emit pinability here" in renderedassert "never hide it in outcome prose" in renderedassert ("reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label, "diff --git a/tests/unit/test_copilot_sky11865_mint_satisfiability.py b/tests/unit/test_copilot_sky11865_mint_satisfiability.pyindex 52e5eef72..3ebfd7d6b 100644--- a/tests/unit/test_copilot_sky11865_mint_satisfiability.py+++ b/tests/unit/test_copilot_sky11865_mint_satisfiability.py@@ -327,10 +327,10 @@ def test_mint_degrade_survives_json_round_trip() -> None:assert restored[0].mint_degrade == _DEGRADE-def test_mint_degrade_is_identity_bearing_in_reconcile_key() -> None:+def test_mint_degrade_is_not_identity_bearing_in_reconcile_key() -> None:base = CompletionCriterion(id="req", outcome="return the status", level="run", output_path="output.status")degraded = replace(base, mint_degrade=_DEGRADE)- assert _criterion_reconcile_key(base) != _criterion_reconcile_key(degraded)+ assert _criterion_reconcile_key(base) == _criterion_reconcile_key(degraded)def test_producer_floor_excludes_degraded_from_rekey() -> None:diff --git a/tests/unit/test_workflow_copilot_completion_criteria_schema.py b/tests/unit/test_workflow_copilot_completion_criteria_schema.pynew file mode 100644index 000000000..c678f47f6--- /dev/null+++ b/tests/unit/test_workflow_copilot_completion_criteria_schema.py@@ -0,0 +1,23 @@+from datetime import UTC, datetime++from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotCompletionCriteriaSet+++def test_completion_criteria_set_keeps_list_storage_shape() -> None:+ now = datetime.now(UTC)+ criteria = [{"id": "c0", "outcome": "done", "pinability": "pinned"}]++ stored = WorkflowCopilotCompletionCriteriaSet.model_validate(+ {+ "completion_criteria_set_id": "wccs_1",+ "organization_id": "o_1",+ "workflow_copilot_chat_id": "wcc_1",+ "goal_epoch": 1,+ "status": "active",+ "criteria": criteria,+ "created_at": now,+ "modified_at": now,+ }+ )++ assert stored.criteria == criteria