Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11544: Fix value-grounded Copilot completion criteria (#6899)
Andrew Neilsone75f13c8b815
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2index 67d1fa256..dd75a36d3 100644--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2@@ -25,11 +25,13 @@ completion_contract:- Set to null when the user does not provide an explicit completion condition.completion_criteria:-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}.+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, output_path, expected_output_value, expected_output_shape, kind, terminal_action_family}.- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.- deliverable_kind: null unless this criterion's requested output is satisfied by execution-owned browser-download registration rather than a user-authored extraction field. The only supported non-null value is "registered_download".+- expected_output_value: null unless this criterion requires an exact structured output value the user explicitly provided. Put that value here as data; never hide it in outcome prose.+- expected_output_shape: null unless this criterion requires a generated/provider structured output value whose exact value is not known before the run. Use only one of: reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label. Do not use this as a non-empty-value proxy; leave null when the field's shape is not one of those declared types.- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.pyindex 7fce09698..2389c7ab3 100644--- a/skyvern/forge/sdk/api/llm/config_registry.py+++ b/skyvern/forge/sdk/api/llm/config_registry.py@@ -1978,7 +1978,10 @@ if settings.ENABLE_OPENROUTER:["OPENROUTER_API_KEY"],supports_vision=True,add_assistant_prefix=False,- max_completion_tokens=131072,+ # MiMo v2.5 is a reasoning model; uncapped it emits 100k+ reasoning/output+ # tokens (multi-minute calls) on short prompts. Cap output + reasoning effort.+ max_completion_tokens=16384,+ reasoning_effort="low",litellm_params=LiteLLMParams(api_key=settings.OPENROUTER_API_KEY,api_base=settings.OPENROUTER_API_BASE,diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.pyindex ca24fb2c2..c4ac27145 100644--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py@@ -18,7 +18,9 @@ from skyvern.forge.sdk.copilot.completion_verification import (from skyvern.forge.sdk.copilot.request_policy import (CompletionCriterion,CriterionKind,+ ExpectedOutputShape,TerminalActionFamily,+ _coerce_expected_output_shape,_normalize_contingent_antecedent_output_path,_normalize_deliverable_kind,is_fallback_floor_criterion,@@ -123,6 +125,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion"method_mandated": criterion.method_mandated,"level": criterion.level,"output_path": criterion.output_path,+ "expected_output_value": criterion.expected_output_value,+ "expected_output_shape": criterion.expected_output_shape,"kind": criterion.kind,"terminal_action_family": criterion.terminal_action_family,}@@ -143,6 +147,8 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:continuelevel = item.get("level")output_path = item.get("output_path")+ expected_output_value = item.get("expected_output_value")+ expected_output_shape = _coerce_expected_output_shape(item.get("expected_output_shape"))contingent_on = item.get("contingent_on")contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(item.get("contingent_antecedent_output_path")@@ -166,6 +172,10 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:method_mandated=bool(item.get("method_mandated")),level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,+ expected_output_value=expected_output_value.strip()+ if isinstance(expected_output_value, str) and expected_output_value.strip()+ else None,+ expected_output_shape=cast(ExpectedOutputShape | None, expected_output_shape),kind=cast(CriterionKind, kind),terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),)@@ -177,11 +187,15 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:contingent_key = criterion.contingent_on or ""contingent_path_key = criterion.contingent_antecedent_output_path or ""deliverable_kind_key = criterion.deliverable_kind or ""+ expected_output_value_key = criterion.expected_output_value or ""+ expected_output_shape_key = criterion.expected_output_shape or ""if criterion.output_path:return (f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"f"\x1fdeliverable_kind:{deliverable_kind_key}"f"\x1foutput_path:{criterion.output_path}"+ f"\x1fexpected_output_value:{expected_output_value_key}"+ f"\x1fexpected_output_shape:{expected_output_shape_key}")return (f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"@@ -203,7 +217,7 @@ _REQUESTED_OUTPUT_WORDS = frozenset()_REQUESTED_OUTPUT_FIELD_TOKENS = frozenset("address addresses date dates email emails id identifier identifiers license licenses location locations name names "- "npi number numbers phone phones result specialties specialty status statuses taxonomy".split()+ "npi number numbers owner owners phone phones result specialties specialty status statuses taxonomy".split())_GENERIC_PROFILE_MARKERS = ("profile details",diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.pyindex 395ee180c..8f6461e25 100644--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py@@ -5,7 +5,7 @@ import jsonimport reimport textwrapfrom collections.abc import Mapping-from typing import Any, Protocol+from typing import Any, Literal, Protocolimport yaml@@ -19,6 +19,7 @@ from skyvern.utils.yaml_loader import safe_load_no_dates_GOAL_PATH_INDEX_PATTERN = re.compile(r"\[\d+\]")_REQUESTED_OUTPUT_PREFIX = "output."+_STRUCTURAL_ABSTENTION_REASON_CODE = "structurally_abstained"# Ignored at any requested-output traversal depth, including schema-declared nested fields._IGNORED_RUNTIME_FIELD_NAMES = frozenset({"evidence_text"})@@ -67,14 +68,83 @@ def grade_requested_output_criteria())continue- evidence_ref = _runtime_output_path_evidence_ref(snapshot.block_outputs, path)- if evidence_ref is not None:+ expected_value = criterion.expected_output_value+ expected_shape = criterion.expected_output_shape+ grounding_mode = _criterion_grounding_mode(criterion)+ trace_fields = _criterion_trace_fields(criterion, grounding_mode)+ if expected_value is not None:+ match_state, evidence_ref = _runtime_output_path_match(snapshot.block_outputs, path, expected_value)+ elif expected_shape is not None:+ match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)+ else:+ match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)+ if match_state == "present" and evidence_ref is not None:+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="unsatisfied",+ reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,+ evidence_ref=evidence_ref,+ missing_evidence=(+ "requested-output field is present, but the criterion has no typed expected_output_value "+ "that can prove or refute the value"+ ),+ **trace_fields,+ )+ )+ continue+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="unsatisfied",+ reason_code="no_evidence",+ missing_evidence=(+ "requested-output criterion lacks typed expected_output_value or expected_output_shape; "+ "presence-only output cannot confirm value-grounded criterion"+ ),+ **trace_fields,+ )+ )+ continue+ if match_state == "present" and evidence_ref is not None:+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="unsatisfied",+ reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,+ evidence_ref=evidence_ref,+ missing_evidence=(+ "requested-output field is present with a typed expected_output_shape, but no exact "+ "expected_output_value can prove or refute the value"+ ),+ **trace_fields,+ )+ )+ continue+ if match_state == "satisfied" and evidence_ref is not None:verdicts.append(CriterionVerdict(criterion_id=criterion.id,state="satisfied",reason_code="evidence_confirms",evidence_ref=evidence_ref,+ **trace_fields,+ )+ )+ continue+ if match_state == "contradicted":+ verdicts.append(+ CriterionVerdict(+ criterion_id=criterion.id,+ state="unsatisfied",+ reason_code="evidence_contradicts",+ evidence_ref=evidence_ref,+ missing_evidence=(+ f"run output included output.{path} but it did not match the expected value"+ if expected_value is not None+ else f"run output included output.{path} but it could not be value-grounded"+ ),+ **trace_fields,))continue@@ -84,11 +154,29 @@ def grade_requested_output_criteria(state="unsatisfied",reason_code="missing_exact_field",missing_evidence=f"run output did not include exact structured field output.{path}",+ **trace_fields,))return verdicts+def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:+ if criterion.expected_output_value is not None:+ return "exact_value"+ if criterion.expected_output_shape is not None:+ return "shape"+ return "missing"+++def _criterion_trace_fields(criterion: CompletionCriterion, grounding_mode: str) -> dict[str, Any]:+ return {+ "output_path": criterion.output_path,+ "grounding_mode": grounding_mode,+ "expected_output_shape": criterion.expected_output_shape,+ "has_exact_value": criterion.expected_output_value is not None,+ }++def _authored_output_contract_paths(copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None) -> set[str]:metadata = copilot_ctx.code_artifact_metadataif not isinstance(metadata, Mapping):@@ -244,40 +332,85 @@ def _dict_keys(node: ast.Dict) -> set[str]:return {key.value for key in node.keys if isinstance(key, ast.Constant) and isinstance(key.value, str)}-def _runtime_output_path_evidence_ref(block_outputs: Mapping[str, Any], path: str) -> str | None:+def _runtime_output_path_match(+ block_outputs: Mapping[str, Any], path: str, expected_value: str | None+) -> tuple[str, str | None]:parts = _path_parts(path)- if not parts:- return None+ if not parts or expected_value is None:+ return "missing", None+ contradicted_ref: str | None = Nonefor label, payload in block_outputs.items():- if _path_has_meaningful_value(payload, parts):- return f"block_outputs:{label}.{path}"- return None+ values, source_path = _runtime_path_values(payload, parts, path)+ if not values:+ continue+ evidence_ref = f"block_outputs:{label}.{source_path}"+ if any(_value_matches_expected(value, expected_value) for value in values):+ return "satisfied", evidence_ref+ contradicted_ref = contradicted_ref or evidence_ref+ if contradicted_ref is not None:+ return "contradicted", contradicted_ref+ return "missing", None-def _path_has_meaningful_value(value: Any, parts: list[str]) -> bool:+def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -> tuple[str, str | None]:+ parts = _path_parts(path)if not parts:- return _is_meaningful_requested_value(value)+ return "missing", None+ for label, payload in block_outputs.items():+ values, source_path = _runtime_present_path_values(payload, parts, path)+ if not values:+ continue+ evidence_ref = f"block_outputs:{label}.{source_path}"+ return "present", evidence_ref+ return "missing", None+++def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:+ canonical_values = _path_values(payload, parts)+ if canonical_values:+ filtered_canonical_values = [value for value in canonical_values if _is_meaningful_requested_value(value)]+ return filtered_canonical_values or canonical_values, path+ wrapped_parts = ["output", *parts]+ wrapped_path = ".".join(wrapped_parts)+ wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]+ return wrapped_values, wrapped_path+++def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:+ raw_canonical_values = _path_values(payload, parts)+ if raw_canonical_values:+ canonical_values = [value for value in raw_canonical_values if _is_meaningful_requested_value(value)]+ return canonical_values, path+ wrapped_parts = ["output", *parts]+ wrapped_path = ".".join(wrapped_parts)+ wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]+ return wrapped_values, wrapped_path+++def _path_values(value: Any, parts: list[str]) -> list[Any]:+ if not parts:+ return [value]part = parts[0]if part == "[]":if not isinstance(value, list):- return False- return any(_path_has_meaningful_value(item, parts[1:]) for item in value)+ return []+ return [matched for item in value for matched in _path_values(item, parts[1:])]expects_array = part.endswith("[]")key = part[:-2] if expects_array else partif key in _IGNORED_RUNTIME_FIELD_NAMES:- return False+ return []if isinstance(value, Mapping):if key not in value:- return False+ return []child = value[key]if expects_array:if not isinstance(child, list):- return False- return any(_path_has_meaningful_value(item, parts[1:]) for item in child)- return _path_has_meaningful_value(child, parts[1:])+ return []+ return [matched for item in child for matched in _path_values(item, parts[1:])]+ return _path_values(child, parts[1:])if isinstance(value, list):- return any(_path_has_meaningful_value(item, parts) for item in value)- return False+ return [matched for item in value for matched in _path_values(item, parts)]+ return []def _is_meaningful_requested_value(value: Any) -> bool:@@ -292,6 +425,26 @@ def _is_meaningful_requested_value(value: Any) -> bool:return True+def _value_matches_expected(value: Any, expected: str) -> bool:+ if isinstance(value, Mapping):+ return any(_value_matches_expected(item, expected) for item in value.values())+ if isinstance(value, list):+ return any(_value_matches_expected(item, expected) for item in value)+ observed = _normalized_expected_text(value)+ target = _normalized_expected_text(expected)+ if not observed or not target:+ return False+ return observed == target or _compact_expected_text(observed) == _compact_expected_text(target)+++def _normalized_expected_text(value: Any) -> str:+ return " ".join(str(value).casefold().split())+++def _compact_expected_text(value: str) -> str:+ return "".join(char for char in value if char.isalnum())++def _normalize_output_path(path: str | None) -> str:if not isinstance(path, str):return ""diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 2582f1f0f..b35862840 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -13,7 +13,7 @@ from __future__ import annotationsimport asyncioimport jsonimport re-from collections.abc import Iterable+from collections.abc import Collection, Iterablefrom dataclasses import dataclass, fieldfrom typing import Any, Literal@@ -40,6 +40,7 @@ _MISSING_EVIDENCE_MAX_CHARS = 500_MAX_BLOCK_OUTPUTS = 20_MAX_TRACE_VERDICTS = 8_REASON_CODES = frozenset({"evidence_confirms", "no_evidence", "evidence_contradicts", "unknown"})+_STRUCTURAL_ABSTENTION_REASON_CODE = "structurally_abstained"_CONTINGENT_ABSTENTION_REASON_CODES = frozenset({"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"})@@ -78,6 +79,10 @@ class CriterionVerdict:reason_code: strevidence_ref: str | None = Nonemissing_evidence: str | None = None+ output_path: str | None = None+ grounding_mode: Literal["exact_value", "shape", "missing"] | None = None+ expected_output_shape: str | None = None+ has_exact_value: bool = False@propertydef satisfied(self) -> bool:@@ -119,6 +124,8 @@ class CompletionVerificationResult:self.structural_unfired_criterion_ids,):continue+ if verdict is not None and _is_structural_requested_output_abstention(verdict):+ continuereturn Falsereturn satisfied_run_plane_count > 0@@ -133,7 +140,9 @@ class CompletionVerificationResult:unmet = [verdictfor verdict in self.verdicts- if not verdict.satisfied and not self.is_structural_contingent_abstention(verdict)+ if not verdict.satisfied+ and not self.is_structural_contingent_abstention(verdict)+ and not _is_structural_requested_output_abstention(verdict)]missing_evidence: list[str] = []for verdict in unmet:@@ -162,6 +171,19 @@ class CompletionVerificationResult:data[f"{prefix}_state"] = verdict.statedata[f"{prefix}_satisfied"] = verdict.satisfieddata[f"{prefix}_reason_code"] = verdict.reason_code+ if verdict.output_path:+ data[f"{prefix}_output_path"] = verdict.output_path+ if verdict.grounding_mode:+ data[f"{prefix}_grounding_mode"] = verdict.grounding_mode+ if verdict.expected_output_shape:+ data[f"{prefix}_expected_output_shape"] = verdict.expected_output_shape+ if (+ verdict.output_path+ or verdict.grounding_mode+ or verdict.expected_output_shape+ or verdict.has_exact_value+ ):+ data[f"{prefix}_has_exact_value"] = verdict.has_exact_valueif contingent_on := self.contingent_on_by_criterion_id.get(verdict.criterion_id):data[f"{prefix}_contingent_on"] = contingent_onif contingent_path := self.contingent_antecedent_output_path_by_criterion_id.get(verdict.criterion_id):@@ -172,7 +194,12 @@ class CompletionVerificationResult:evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)if evidence_ref:data[f"{prefix}_evidence_ref"] = evidence_ref- detail = None if self.is_structural_contingent_abstention(verdict) else verdict_missing_evidence(verdict)+ detail = (+ None+ if self.is_structural_contingent_abstention(verdict)+ or _is_structural_requested_output_abstention(verdict)+ else verdict_missing_evidence(verdict)+ )if detail:data[f"{prefix}_missing_evidence"] = detailreturn data@@ -398,7 +425,7 @@ def _field_aliases_for_output_path(path: str) -> tuple[str, str]:return field, f"{field}_output"-def _find_structured_field_values(value: Any, field_names: set[str]) -> Iterable[Any]:+def _find_structured_field_values(value: Any, field_names: Collection[str]) -> Iterable[Any]:if isinstance(value, dict):for key, item in value.items():if isinstance(key, str) and key in field_names:@@ -424,6 +451,12 @@ def structural_unfired_contingent_criterion_ids(if key in field_names:values.append(value)values.extend(_find_structured_field_values(value, field_names))+ if _is_blocker_contingent_criterion(criterion):+ if _has_real_blocker_evidence(snapshot):+ continue+ if _has_structural_no_blocker_evidence(snapshot):+ unfired_ids.append(criterion.id)+ continueif not values:continueif any(_is_meaningful_contingent_antecedent_value(value) for value in values):@@ -432,6 +465,63 @@ def structural_unfired_contingent_criterion_ids(return unfired_ids+def _is_blocker_contingent_criterion(criterion: CompletionCriterion) -> bool:+ path = criterion.contingent_antecedent_output_path+ if path is None:+ return False+ return _output_path_field(path) in {"blocker", "manual_service_blocker"}+++_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})+++def _blocker_family_values(snapshot: RunEvidenceSnapshot) -> list[Any]:+ values: list[Any] = []+ for key, value in snapshot.block_outputs.items():+ if key in _BLOCKER_FIELDS:+ values.append(value)+ values.extend(_find_structured_field_values(value, _BLOCKER_FIELDS))+ return values+++def _has_real_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:+ return any(_is_real_blocker_evidence(value) for value in _blocker_family_values(snapshot))+++def _is_real_blocker_evidence(value: Any) -> bool:+ return _is_meaningful_contingent_antecedent_value(value) and not _is_structural_no_blocker_marker(value)+++def _has_structural_no_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:+ for value in _blocker_family_values(snapshot):+ if _is_structural_no_blocker_marker(value):+ return True+ return False+++def _is_structural_no_blocker_marker(value: Any) -> bool:+ if isinstance(value, bool):+ return not value+ if value is None:+ return True+ if not isinstance(value, str):+ return False+ normalized = " ".join(value.casefold().split())+ return normalized in {+ "",+ "none",+ "null",+ "false",+ "no",+ "n/a",+ "na",+ "no blocker",+ "no blockers",+ "none found",+ "not blocked",+ }++def _coerce_result(raw: Any,criterion_ids: list[str],@@ -1577,6 +1667,10 @@ def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:return verdict.state == "unknown" and verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX)+def _is_structural_requested_output_abstention(verdict: CriterionVerdict) -> bool:+ return verdict.state == "unsatisfied" and verdict.reason_code == _STRUCTURAL_ABSTENTION_REASON_CODE++def _is_contingent_abstention(verdict: CriterionVerdict,contingent_criterion_ids: Iterable[str],diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 8837fa2d5..03a4b30fc 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -151,8 +151,10 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "_MAX_COMPLETION_CRITERIA = 8+_MAX_TRACE_COMPLETION_CRITERIA = 8_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200_COMPLETION_CRITERION_CONTINGENT_ON_MAX_CHARS = 200+_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS = 500_CONTINGENT_ANTECEDENT_OUTPUT_PATH_RE = re.compile(r"^output\.[A-Za-z_][A-Za-z0-9_]*$")_REQUESTED_OUTPUT_CRITERION_ID_PREFIX = "__copilot_requested_output__"@@ -166,15 +168,25 @@ CriterionLevel = Literal["definition", "run"]_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})CriterionKind = Literal["outcome", "terminal_action"]TerminalActionFamily = Literal["request", "application", "form", "order"]+ExpectedOutputShape = Literal[+ "reference_code",+ "numeric_identifier",+ "date",+ "address",+ "status_label",+ "money_amount",+ "owner_label",+]_CRITERION_KINDS: frozenset[str] = frozenset({"outcome", "terminal_action"})_TERMINAL_ACTION_FAMILIES: frozenset[str] = frozenset({"request", "application", "form", "order"})+_EXPECTED_OUTPUT_SHAPES: frozenset[str] = frozenset(get_args(ExpectedOutputShape))_OUTPUT_INTENT_RE = re.compile(r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"r"final\s+(?:extracted\s+)?fields?|result\s+records?|returned\s+records?)\b",re.I,)-_OUTPUT_SPAN_END_RE = re.compile(r"[\n.!?]")+_OUTPUT_SPAN_END_RE = re.compile(r"[\n!?]")_OUTPUT_METHOD_TAIL_RE = re.compile(r"\b(?:by|via|using|after|before|then|click(?:ing)?|open(?:ing)?|select(?:ing)?|"r"choose|choosing|search(?:ing)?|navigate|go\s+to)\b.*",@@ -188,7 +200,8 @@ _OUTPUT_FIELD_CONNECTOR_RE = re.compile(_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")_OUTPUT_FIELD_WORDS = frozenset("address addresses date dates email emails id identifier identifiers license licenses location locations "- "name names number numbers phone phones specialties specialty status statuses taxonomy".split()+ "amount amounts domain domains name names number numbers owner owners phone phones rate rates specialties specialty "+ "status statuses taxonomy total totals url urls website websites".split())_OUTPUT_GENERIC_WORDS = frozenset("a all an data detail details each entity final for information its of output outputs profile record records "@@ -217,6 +230,8 @@ class CompletionCriterion:# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".level: CriterionLevel = "run"output_path: str | None = None+ expected_output_value: str | None = None+ expected_output_shape: ExpectedOutputShape | None = Nonekind: CriterionKind = "outcome"terminal_action_family: TerminalActionFamily | None = None@@ -256,7 +271,7 @@ class RequestPolicy:return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]def to_trace_data(self) -> dict[str, Any]:- return {+ data: dict[str, Any] = {"testing_intent": self.testing_intent,"credential_input_kind": self.credential_input_kind,"clarification_reason": self.clarification_reason,@@ -284,6 +299,19 @@ class RequestPolicy:len(origins) for origins in self.existing_workflow_credential_origins.values()),}+ requested_output_criteria = [+ criterion for criterion in self.graded_completion_criteria() if criterion.output_path is not None+ ]+ data["requested_output_criteria_count"] = len(requested_output_criteria)+ for index, criterion in enumerate(requested_output_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):+ prefix = f"requested_output_criterion_{index}"+ data[f"{prefix}_id"] = criterion.id+ data[f"{prefix}_output_path"] = criterion.output_path+ data[f"{prefix}_grounding_mode"] = _criterion_grounding_mode(criterion)+ data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None+ if criterion.expected_output_shape:+ data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape+ return datadef prompt_summary(self) -> str:lines = [@@ -317,6 +345,14 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicyreturn request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)+def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:+ if criterion.expected_output_value is not None:+ return "exact_value"+ if criterion.expected_output_shape is not None:+ return "shape"+ return "missing"++_TRANSCRIPT_TOTAL_CHAR_BUDGET = 2048TRANSCRIPT_ANCHOR_CHAR_CAP = 512_TRANSCRIPT_RETAINED_MIN_CHARS = 512@@ -518,6 +554,12 @@ def _coerce_terminal_action_family(value: Any, kind: CriterionKind) -> TerminalAreturn None+def _coerce_expected_output_shape(value: Any) -> ExpectedOutputShape | None:+ if isinstance(value, str) and value in _EXPECTED_OUTPUT_SHAPES:+ return cast(ExpectedOutputShape, value)+ return None++def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:if isinstance(raw, str):raw = parse_final_response(raw)@@ -603,6 +645,14 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:continueoutput_path_raw = item.get("output_path")output_path = output_path_raw.strip() if isinstance(output_path_raw, str) and output_path_raw.strip() else None+ expected_output_value_raw = item.get("expected_output_value")+ expected_output_value = (+ " ".join(expected_output_value_raw.split())[:_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS].strip()+ if isinstance(expected_output_value_raw, str)+ else None+ )+ expected_output_value = expected_output_value or None+ expected_output_shape = _coerce_expected_output_shape(item.get("expected_output_shape"))contingent_on_raw = item.get("contingent_on")contingent_on = (" ".join(contingent_on_raw.split())[:_COMPLETION_CRITERION_CONTINGENT_ON_MAX_CHARS].strip()@@ -638,6 +688,8 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELSelse "run",output_path=output_path,+ expected_output_value=expected_output_value,+ expected_output_shape=expected_output_shape,kind=kind,terminal_action_family=_coerce_terminal_action_family(item.get("terminal_action_family"), kind),)@@ -668,14 +720,26 @@ def _output_intent_spans(user_message: str) -> list[str]:spans: list[str] = []for match in _OUTPUT_INTENT_RE.finditer(message):tail = message[match.start() :]- end_match = _OUTPUT_SPAN_END_RE.search(tail)- span = tail[: end_match.start()] if end_match is not None else tail+ span = tail[: _output_span_end_index(tail)]span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")if span:spans.append(span)return spans+def _output_span_end_index(text: str) -> int:+ for index, char in enumerate(text):+ if _OUTPUT_SPAN_END_RE.fullmatch(char):+ return index+ if char != ".":+ continue+ next_char = text[index + 1] if index + 1 < len(text) else ""+ if next_char and not next_char.isspace():+ continue+ return index + 1+ return len(text)++def _normalize_requested_output_aliases(aliases: dict[str, str] | None) -> dict[str, str]:normalized: dict[str, str] = {}for field_name, output_path in (aliases or {}).items():@@ -741,6 +805,10 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | Nonormalized_aliases = _normalize_requested_output_aliases(aliases)if " ".join(_word_tokens(candidate)) in normalized_aliases:return candidate+ candidate_tokens = _word_tokens(candidate)+ for alias_key in sorted(normalized_aliases, key=lambda key: len(key.split()), reverse=True):+ if _tokens_contain_sequence(candidate_tokens, alias_key.split()):+ return _matched_alias_phrase(candidate, alias_key) or alias_keycandidate_words = candidate.split()for word in candidate_words:if " ".join(_word_tokens(word)) in normalized_aliases:@@ -793,6 +861,15 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | Noreturn " ".join(phrase_words)+def _matched_alias_phrase(candidate: str, alias_key: str) -> str | None:+ alias_words = alias_key.split()+ if not alias_words:+ return None+ pattern = r"\b" + r"[\W_]+".join(re.escape(word) for word in alias_words) + r"\b"+ match = re.search(pattern, candidate, re.I)+ return match.group(0).strip() if match is not None else None++def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:fields: list[str] = []seen: set[str] = set()@@ -861,6 +938,42 @@ def _criterion_text_covers_any_requested_output(criterion: CompletionCriterion,return any(_criterion_text_covers_requested_output(criterion, field_name) for field_name in requested_fields)+def _requested_output_expected_values_from_criteria(+ criteria: list[CompletionCriterion],+ requested_specs: list[tuple[str, str, str]],+) -> dict[str, str]:+ values: dict[str, str] = {}+ for criterion in criteria:+ if criterion.level == "definition" or criterion.method_mandated:+ continue+ for field_name, output_path, _field_label in requested_specs:+ if criterion.output_path != output_path and not _criterion_text_covers_requested_output(+ criterion, field_name+ ):+ continue+ value = criterion.expected_output_value+ if value:+ values.setdefault(output_path, value)+ return values+++def _requested_output_shapes_from_criteria(+ criteria: list[CompletionCriterion],+ requested_specs: list[tuple[str, str, str]],+) -> dict[str, ExpectedOutputShape]:+ shapes: dict[str, ExpectedOutputShape] = {}+ for criterion in criteria:+ if criterion.level == "definition" or criterion.method_mandated or criterion.expected_output_shape is None:+ continue+ for field_name, output_path, _field_label in requested_specs:+ if criterion.output_path != output_path and not _criterion_text_covers_requested_output(+ criterion, field_name+ ):+ continue+ shapes.setdefault(output_path, criterion.expected_output_shape)+ return shapes++def _requested_output_criterion_id(output_path: str) -> str:slug = "_".join(_word_tokens(output_path)) or "field"return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"@@ -911,6 +1024,27 @@ def _cap_completion_criteria(return capped+def _preserve_after_requested_output_canonicalization(+ criterion: CompletionCriterion,+ requested_fields: list[str],+ requested_output_paths: set[str],+) -> bool:+ if criterion.level == "definition" or criterion.method_mandated:+ return True+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(+ criterion, requested_fields+ ):+ return False+ if (+ requested_output_paths+ and not criterion.output_path+ and criterion.kind == "outcome"+ and _generic_completion_criterion(criterion)+ ):+ return False+ return True++def _apply_requested_output_completion_criteria(policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None) -> None:@@ -939,6 +1073,9 @@ def _apply_requested_output_completion_criteria(if not requested_output_paths:return+ value_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)+ shape_by_output_path = _requested_output_shapes_from_criteria(policy.completion_criteria, requested_specs)+metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}for criterion in policy.completion_criteria:if criterion.level == "definition" or criterion.method_mandated:@@ -966,12 +1103,7 @@ def _apply_requested_output_completion_criteria(preserved_criteria = [criterionfor criterion in policy.completion_criteria- if criterion.level == "definition"- or criterion.method_mandated- or (- criterion.output_path not in requested_output_paths- and not _criterion_text_covers_any_requested_output(criterion, requested_fields)- )+ if _preserve_after_requested_output_canonicalization(criterion, requested_fields, requested_output_paths)]canonical_requested_criteria = [CompletionCriterion(@@ -979,6 +1111,8 @@ def _apply_requested_output_completion_criteria(outcome=f"The returned record includes {field_label}.",level="run",output_path=output_path,+ expected_output_value=value_by_output_path.get(output_path),+ expected_output_shape=shape_by_output_path.get(output_path),contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],@@ -1012,6 +1146,10 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Nonitem["deliverable_kind"] = criterion.deliverable_kindif criterion.output_path:item["output_path"] = criterion.output_path+ if criterion.expected_output_value:+ item["expected_output_value"] = criterion.expected_output_value+ if criterion.expected_output_shape:+ item["expected_output_shape"] = criterion.expected_output_shaperendered.append(item)return json.dumps(rendered)diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex 06eaf9fed..79188a459 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -15,6 +15,8 @@ from skyvern.forge.sdk.copilot.agent import (_rewrite_failed_test_response,_verified_workflow_or_none,)+from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criteria_to_json+from skyvern.forge.sdk.copilot.completion_output_grounding import grade_requested_output_criteriafrom skyvern.forge.sdk.copilot.completion_verification import (REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,CompletionVerificationResult,@@ -97,6 +99,8 @@ def _criterion(kind: str = "outcome",terminal_action_family: str | None = None,deliverable_kind: str | None = None,+ expected_output_value: str | None = None,+ expected_output_shape: str | None = None,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -108,6 +112,8 @@ def _criterion(kind=kind,terminal_action_family=terminal_action_family,deliverable_kind=deliverable_kind, # type: ignore[arg-type]+ expected_output_value=expected_output_value,+ expected_output_shape=expected_output_shape, # type: ignore[arg-type])@@ -547,6 +553,20 @@ def test_structural_unfired_ids_derive_from_empty_output_path() -> None:assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == ["c0"]+def test_p7_manual_service_no_blocker_abstains() -> None:+ criteria = [+ _criterion(+ "c0",+ "Any manual service blocker is reported to the user.",+ contingent_on="a manual service blocker exists",+ contingent_antecedent_output_path="output.blocker",+ )+ ]+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": {"manual_service_blocker": None}})++ assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == ["c0"]++@pytest.mark.parametrize("reason_code,state", [("evidence_contradicts", "unsatisfied"), ("unknown", "unknown")])def test_false_contingent_antecedent_output_abstains(reason_code: str, state: str) -> None:criteria = [@@ -651,6 +671,35 @@ def test_structural_fired_evidence_overrides_empty_output_path() -> None:assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == []+def test_real_blocker_family_evidence_overrides_primary_no_blocker_marker() -> None:+ criteria = [+ _criterion(+ "c0",+ "A provider blocker is reported to the user.",+ contingent_on="the provider site blocks online submission",+ contingent_antecedent_output_path="output.blocker",+ )+ ]+ snapshot = RunEvidenceSnapshot(+ block_outputs={"terminal_result": {"blocker": None, "manual_service_blocker": "Provider requires phone call"}}+ )+ structural_unfired_ids = structural_unfired_contingent_criterion_ids(criteria, snapshot)+ result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["c0", "c1"],+ contingent_criterion_ids=["c0"],+ contingent_antecedent_output_path_by_criterion_id={"c0": "output.blocker"},+ structural_unfired_criterion_ids=structural_unfired_ids,+ verdicts=[+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts"),+ CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms"),+ ],+ )++ assert structural_unfired_ids == []+ assert result.is_fully_satisfied() is False++def test_empty_verdicts_with_criteria_is_not_vacuously_satisfied() -> None:result = CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[])assert result.is_fully_satisfied() is False@@ -3336,7 +3385,12 @@ async def test_requested_output_path_ignores_evidence_text(monkeypatch: pytest.Massert verification is not Noneassert verification.is_fully_satisfied() is False- assert verification.verdicts[0].reason_code == "missing_exact_field"+ assert verification.verdicts[0].reason_code == "no_evidence"+ assert (+ verification.verdicts[0].missing_evidence+ == "requested-output criterion lacks typed expected_output_value or expected_output_shape; "+ "presence-only output cannot confirm value-grounded criterion"+ )@pytest.mark.asyncio@@ -3360,11 +3414,13 @@ async def test_requested_output_path_ignores_block_level_prose(monkeypatch: pyteassert verification is not Noneassert verification.is_fully_satisfied() is False- assert verification.verdicts[0].reason_code == "missing_exact_field"+ assert verification.verdicts[0].reason_code == "no_evidence"@pytest.mark.asyncio-async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch: pytest.MonkeyPatch) -> None:+async def test_requested_output_path_exact_runtime_field_with_expected_value_satisfies(+ monkeypatch: pytest.MonkeyPatch,+) -> None:async def fail_handler(**_: object) -> object:raise AssertionError("exact requested-output evidence must bypass the judge")@@ -3373,7 +3429,14 @@ async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch:_set_workflow_labels(ctx, "extract_profile")ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")ctx.request_policy = RequestPolicy(- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]+ completion_criteria=[+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ )+ ])verification = await _maybe_run_completion_verification(@@ -3386,185 +3449,1100 @@ async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch:assert verification.is_fully_satisfied() is Trueassert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.npi"+ trace = verification.to_trace_data()+ assert trace["verdict_0_criterion_id"] == "c_npi"+ assert trace["verdict_0_output_path"] == "output.npi"+ assert trace["verdict_0_grounding_mode"] == "exact_value"+ assert "verdict_0_expected_output_shape" not in trace+ assert trace["verdict_0_has_exact_value"] is True+ assert "1234567890" not in repr(trace)-async def test_requested_output_path_does_not_match_block_label_only(monkeypatch: pytest.MonkeyPatch) -> None:- async def fail_handler(**_: object) -> object:- raise AssertionError("requested-output criteria must not reach the judge")- _patch_completion_handler(monkeypatch, fail_handler)+def test_requested_output_path_without_expected_value_structurally_abstains_for_present_values() -> None:ctx = _run_ctx()- _set_workflow_labels(ctx, "npi")- ctx.code_artifact_metadata = {"npi": _metadata_for_requested_paths("npi")["extract_profile"]}+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "request_id",+ "provider_captured_address",+ "requested_date",+ "status",+ )ctx.request_policy = RequestPolicy(- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]+ completion_criteria=[+ CompletionCriterion(+ id="c_request_id",+ outcome="The returned record includes request id.",+ output_path="output.request_id",+ ),+ CompletionCriterion(+ id="c_provider_captured_address",+ outcome="The returned record includes provider captured address.",+ output_path="output.provider_captured_address",+ ),+ CompletionCriterion(+ id="c_requested_date",+ outcome="The returned record includes requested date.",+ output_path="output.requested_date",+ ),+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ ),+ ])- verification = await _maybe_run_completion_verification(+ verdicts = grade_requested_output_criteria(ctx,- {- "ok": True,- "data": {- "workflow_run_id": "wr_requested_output",- "overall_status": "completed",- "executed_block_labels": ["npi"],- "current_url": "https://example.test/profile",- "blocks": [- {- "label": "npi",- "block_type": "CODE",- "status": "completed",- "extracted_data": {"value": "1234567890"},- }- ],- },- },- time.monotonic(),+ ctx.request_policy.completion_criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "request_id": "100245",+ "provider_captured_address": "100245",+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",+ "status": "2026-06-24",+ "evidence_text": "The request completed successfully.",+ }+ }+ ),)- assert verification is not None- assert verification.is_fully_satisfied() is False- assert verification.verdicts[0].reason_code == "missing_exact_field"+ assert {verdict.criterion_id for verdict in verdicts} == {+ "c_request_id",+ "c_provider_captured_address",+ "c_requested_date",+ "c_status",+ }+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}+ assert {verdict.grounding_mode for verdict in verdicts} == {"missing"}+ assert all(verdict.has_exact_value is False for verdict in verdicts)- ("goal_value_path", "requested_path", "runtime_output", "evidence_ref"),- [- ("$.npi", "output.npi", {"npi": "1234567890"}, "block_outputs:extract_profile.npi"),- ("$[*].npi", "output.npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.npi"),- ("$[0].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),- ("$[].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),- (- "records[0].npi",- "output.records[].npi",- {"records": [{"npi": "1234567890"}]},- "block_outputs:extract_profile.records[].npi",+def test_requested_output_shape_only_generated_fields_structurally_abstain_without_exact_values() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "account_number",+ "selected_start_date",+ )+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",),- (- "output.records[0].npi",- "output.records[].npi",- {"records": [{"npi": "1234567890"}]},- "block_outputs:extract_profile.records[].npi",+ CompletionCriterion(+ id="c_account_number",+ outcome="The returned record includes account number.",+ output_path="output.account_number",+ expected_output_shape="numeric_identifier",),- ],-)-async def test_requested_output_path_normalizes_jsonpath_and_indexes(- monkeypatch: pytest.MonkeyPatch,- goal_value_path: str,- requested_path: str,- runtime_output: Any,- evidence_ref: str,-) -> None:- async def fail_handler(**_: object) -> object:- raise AssertionError("normalized requested-output evidence must bypass the judge")+ CompletionCriterion(+ id="c_selected_start_date",+ outcome="The returned record includes selected start date.",+ output_path="output.selected_start_date",+ expected_output_shape="date",+ ),+ ]- _patch_completion_handler(monkeypatch, fail_handler)+ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_citrus_turn_on": {+ "confirmation_number": "WTR-1842-DEMO",+ "account_number": "100245",+ "selected_start_date": "2026-06-22",+ }+ }+ ),+ )++ assert {verdict.criterion_id for verdict in verdicts} == {+ "c_confirmation_number",+ "c_account_number",+ "c_selected_start_date",+ }+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}+ trace = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=[criterion.id for criterion in criteria],+ verdicts=verdicts,+ ).to_trace_data()+ assert trace["verdict_0_output_path"] == "output.confirmation_number"+ assert trace["verdict_0_grounding_mode"] == "shape"+ assert trace["verdict_0_expected_output_shape"] == "reference_code"+ assert trace["verdict_0_has_exact_value"] is False+ assert "WTR-1842-DEMO" not in repr(trace)+ assert trace["fully_satisfied"] is False+ assert trace["unmet_criterion_ids"] == []+++def test_requested_output_shape_abstains_on_p8_scrambled_values_and_ignores_evidence_text_status() -> None:ctx = _run_ctx()- _set_workflow_labels(ctx, "extract_profile")- ctx.code_artifact_metadata = _metadata_for_requested_paths(goal_value_path)- ctx.request_policy = RequestPolicy(- completion_criteria=[- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path=requested_path)- ]+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "provider_captured_address",+ "requested_date",+ "status",)+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ CompletionCriterion(+ id="c_provider_captured_address",+ outcome="The returned record includes provider captured address.",+ output_path="output.provider_captured_address",+ expected_output_shape="address",+ ),+ CompletionCriterion(+ id="c_requested_date",+ outcome="The returned record includes requested date.",+ output_path="output.requested_date",+ expected_output_shape="date",+ ),+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_shape="status_label",+ ),+ ]- verification = await _maybe_run_completion_verification(+ verdicts = grade_requested_output_criteria(ctx,- _requested_output_result(runtime_output),- time.monotonic(),+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "confirmation_number": "100245",+ "provider_captured_address": "100245",+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",+ "status": "2026-06-24",+ "evidence_text": "Submitted / Processing",+ }+ }+ ),)- assert verification is not None- assert verification.is_fully_satisfied() is True- assert verification.verdicts[0].evidence_ref == evidence_ref+ assert {verdict.criterion_id for verdict in verdicts} == {+ "c_confirmation_number",+ "c_provider_captured_address",+ "c_requested_date",+ "c_status",+ }+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}@pytest.mark.asyncio-async def test_requested_output_path_aliases_authored_contract_to_requested_path(+async def test_requested_output_verifier_rejects_reconstructed_p8_scrambled_values(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:- raise AssertionError("aliased requested-output evidence must bypass the judge")+ raise AssertionError("typed requested-output verifier should not delegate reconstructed P8 proof")_patch_completion_handler(monkeypatch, fail_handler)ctx = _run_ctx()_set_workflow_labels(ctx, "extract_profile")- ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "account_number",+ "provider_captured_address",+ "requested_date",+ "status",+ )ctx.request_policy = RequestPolicy(completion_criteria=[- CompletionCriterion(- id="c_npi",- outcome="The nested record NPI is returned.",- output_path="output.records[].npi",- )+ _criterion(+ "c_confirmation_number",+ "The output includes the confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ _criterion(+ "c_account_number",+ "The output includes the account number.",+ output_path="output.account_number",+ expected_output_shape="numeric_identifier",+ ),+ _criterion(+ "c_provider_captured_address",+ "The output includes the provider captured address.",+ output_path="output.provider_captured_address",+ expected_output_shape="address",+ ),+ _criterion(+ "c_requested_date",+ "The output includes the requested date.",+ output_path="output.requested_date",+ expected_output_shape="date",+ ),+ _criterion(+ "c_status",+ "The output includes the request status.",+ output_path="output.status",+ expected_output_shape="status_label",+ ),])verification = await _maybe_run_completion_verification(ctx,- _requested_output_result({"records": [{"npi": "1234567890"}]}),+ _requested_output_result(+ {+ "confirmation_number": "QC-2002-DEMO",+ "account_number": "100245",+ "provider_captured_address": "100245",+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",+ "status": "2026-06-24",+ "evidence_text": "Submitted / Processing",+ }+ ),time.monotonic(),)assert verification is not None- assert verification.is_fully_satisfied() is True- assert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.records[].npi"+ assert verification.status == "evaluated"+ assert verification.is_fully_satisfied() is False+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_confirmation_number"].reason_code == "structurally_abstained"+ assert verdicts["c_account_number"].reason_code == "structurally_abstained"+ for criterion_id in ("c_provider_captured_address", "c_requested_date", "c_status"):+ assert verdicts[criterion_id].state == "unsatisfied"+ assert verdicts[criterion_id].reason_code == "structurally_abstained"@pytest.mark.asyncio-async def test_requested_output_path_requires_exact_nested_runtime_field(+async def test_requested_output_verifier_accepts_value_correct_reconstructed_p8(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:- raise AssertionError("requested-output criteria must not reach the judge")+ raise AssertionError("typed requested-output verifier should not delegate value-correct P8 proof")_patch_completion_handler(monkeypatch, fail_handler)ctx = _run_ctx()_set_workflow_labels(ctx, "extract_profile")- ctx.code_artifact_metadata = _metadata_for_requested_paths("locations[].address")+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "account_number",+ "provider_captured_address",+ "requested_date",+ "status",+ )ctx.request_policy = RequestPolicy(completion_criteria=[- CompletionCriterion(- id="c_address",- outcome="Each listed location includes address.",- output_path="output.locations[].address",- )+ _criterion(+ "c_confirmation_number",+ "The output includes the confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ _criterion(+ "c_account_number",+ "The output includes the account number.",+ output_path="output.account_number",+ expected_output_shape="numeric_identifier",+ ),+ _criterion(+ "c_provider_captured_address",+ "The output includes the provider captured address.",+ output_path="output.provider_captured_address",+ expected_output_shape="address",+ ),+ _criterion(+ "c_requested_date",+ "The output includes the requested date.",+ output_path="output.requested_date",+ expected_output_shape="date",+ ),+ _criterion(+ "c_status",+ "The output includes the request status.",+ output_path="output.status",+ expected_output_shape="status_label",+ ),])- missing = await _maybe_run_completion_verification(+ verification = await _maybe_run_completion_verification(ctx,- _requested_output_result({"locations": [{"name": "North Clinic"}]}),+ _requested_output_result(+ {+ "confirmation_number": "QC-2002-DEMO",+ "account_number": "100245",+ "provider_captured_address": "77 Gaslight Way, Decatur, GA 30030",+ "requested_date": "2026-06-24",+ "status": "Submitted / Processing",+ }+ ),time.monotonic(),)- assert missing is not None- assert missing.is_fully_satisfied() is False- assert missing.verdicts[0].reason_code == "missing_exact_field"- satisfied = await _maybe_run_completion_verification(- ctx,- _requested_output_result({"locations": [{"address": "100 Main St"}]}),- time.monotonic(),- )- assert satisfied is not None- assert satisfied.is_fully_satisfied() is True- assert satisfied.verdicts[0].evidence_ref == "block_outputs:extract_profile.locations[].address"+ assert verification is not None+ assert verification.status == "evaluated"+ assert verification.is_fully_satisfied() is False+ assert {verdict.reason_code for verdict in verification.verdicts} == {"structurally_abstained"}@pytest.mark.asyncio-async def test_requested_output_path_uses_only_accepted_metadata(monkeypatch: pytest.MonkeyPatch) -> None:- async def fail_handler(**_: object) -> object:- raise AssertionError("unproducible requested-output criteria must not reach the judge")+async def test_requested_output_verifier_accepts_p7_with_unfired_blocker_and_fee_suffix(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}- _patch_completion_handler(monkeypatch, fail_handler)+ _patch_completion_handler(monkeypatch, handler)ctx = _run_ctx()_set_workflow_labels(ctx, "extract_profile")- ctx.raw_code_artifact_metadata = _metadata_for_requested_paths("npi")+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "account_number",+ "selected_start_date",+ "deposit_amount",+ "next_owner",+ "blocker",+ )+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c_submit",+ "The service request is submitted.",+ kind="terminal_action",+ terminal_action_family="request",+ ),+ _criterion(+ "c_confirmation_number",+ "The output includes the confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ _criterion(+ "c_account_number",+ "The output includes the account number.",+ output_path="output.account_number",+ expected_output_shape="numeric_identifier",+ ),+ _criterion(+ "c_selected_start_date",+ "The output includes the selected start date.",+ output_path="output.selected_start_date",+ expected_output_shape="date",+ ),+ _criterion(+ "c_deposit_amount",+ "The output includes the deposit amount.",+ output_path="output.deposit_amount",+ expected_output_shape="money_amount",+ ),+ _criterion(+ "c_next_owner",+ "The output includes the next owner.",+ output_path="output.next_owner",+ expected_output_shape="owner_label",+ ),+ _criterion(+ "c_blocker",+ "Any manual service blocker is reported to the user.",+ output_path="output.blocker",+ contingent_on="a manual service blocker exists",+ contingent_antecedent_output_path="output.blocker",+ ),+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(+ {+ "manual_service_blocker": None,+ "output": {+ "confirmation_number": "WTR-1842-DEMO",+ "account_number": "100245",+ "selected_start_date": "2026-06-22",+ "deposit_amount": "$41.00 plus initiation fee",+ "next_owner": "Provider",+ },+ "evidence_text": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.status == "evaluated"+ assert verification.is_fully_satisfied() is True+ assert verification.structural_unfired_criterion_ids == ["c_blocker"]+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_submit"].reason_code == "evidence_confirms"+ assert verdicts["c_blocker"].reason_code == "no_evidence"+ assert verdicts["c_deposit_amount"].reason_code == "structurally_abstained"+ assert all(+ verdict.reason_code == "structurally_abstained"+ for cid, verdict in verdicts.items()+ if cid not in {"c_submit", "c_blocker"}+ )+++def test_requested_output_canonical_path_precedence_rejects_wrong_top_level_value() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_value="WTR-1842-DEMO",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_citrus_turn_on": {+ "confirmation_number": "QC-2002-DEMO",+ "output": {"confirmation_number": "WTR-1842-DEMO"},+ }+ }+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "evidence_contradicts"+ assert verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.confirmation_number"++[email protected]("canonical_value", [None, ""])+def test_requested_output_exact_value_canonical_placeholder_not_rescued_by_wrapper(+ canonical_value: str | None,+) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_value="WTR-1842-DEMO",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_citrus_turn_on": {+ "confirmation_number": canonical_value,+ "output": {"confirmation_number": "WTR-1842-DEMO"},+ }+ }+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "evidence_contradicts"+ assert verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.confirmation_number"+++def test_requested_output_one_wrapper_runtime_field_abstains_without_broad_search() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ )+ ]++ wrapped_verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={"utility_citrus_turn_on": {"output": {"confirmation_number": "WTR-1842-DEMO"}}}+ ),+ )+ unrelated_nested_verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={"utility_citrus_turn_on": {"details": {"output": {"confirmation_number": "WTR-1842-DEMO"}}}}+ ),+ )++ assert wrapped_verdicts[0].state == "unsatisfied"+ assert wrapped_verdicts[0].reason_code == "structurally_abstained"+ assert wrapped_verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.output.confirmation_number"+ assert unrelated_nested_verdicts[0].state == "unsatisfied"+ assert unrelated_nested_verdicts[0].reason_code == "missing_exact_field"+++def test_requested_output_wrapped_wrong_neighbors_structurally_abstain() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("requested_date", "status")+ criteria = [+ CompletionCriterion(+ id="c_requested_date",+ outcome="The returned record includes requested date.",+ output_path="output.requested_date",+ expected_output_shape="date",+ ),+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_shape="status_label",+ ),+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "output": {+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",+ "status": "2026-06-24",+ }+ }+ }+ ),+ )++ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}++[email protected]("canonical_status", [None, ""])+def test_requested_output_shape_canonical_placeholder_not_rescued_by_wrapper(+ canonical_status: str | None,+) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")+ criteria = [+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_shape="status_label",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "status": canonical_status,+ "output": {"status": "Submitted / Processing"},+ }+ }+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "missing_exact_field"+ assert verdicts[0].evidence_ref is None+++def test_requested_output_exact_values_reject_p8_scrambled_neighbors() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths(+ "confirmation_number",+ "provider_captured_address",+ "requested_date",+ "status",+ )+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_value="QC-2002-DEMO",+ ),+ CompletionCriterion(+ id="c_provider_captured_address",+ outcome="The returned record includes provider captured address.",+ output_path="output.provider_captured_address",+ expected_output_value="77 Gaslight Way, Decatur, GA 30030",+ ),+ CompletionCriterion(+ id="c_requested_date",+ outcome="The returned record includes requested date.",+ output_path="output.requested_date",+ expected_output_value="2026-06-24",+ ),+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_value="Submitted / Processing",+ ),+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "confirmation_number": "QC-2002-DEMO",+ "provider_captured_address": "100245",+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",+ "status": "2026-06-24",+ "evidence_text": "Submitted / Processing",+ }+ }+ ),+ )++ verdicts_by_id = {verdict.criterion_id: verdict for verdict in verdicts}+ assert verdicts_by_id["c_confirmation_number"].reason_code == "evidence_confirms"+ for criterion_id in ("c_provider_captured_address", "c_requested_date", "c_status"):+ assert verdicts_by_id[criterion_id].state == "unsatisfied"+ assert verdicts_by_id[criterion_id].reason_code == "evidence_contradicts"++[email protected]("status", [None, "None", "null", "n/a", "na", "", "-", "--"])+def test_requested_output_status_shape_treats_null_placeholders_as_missing(status: str | None) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")+ criteria = [+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_shape="status_label",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "utility_peach_gas_quickconnect": {+ "status": status,+ "evidence_text": "Submitted / Processing",+ }+ }+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code in {"missing_exact_field", "structurally_abstained"}++[email protected]("status", ["Submitted / Processing", "Approved", "Pending Review", "Not Credentialed"])+def test_requested_output_status_shape_structurally_abstains_for_real_labels(status: str) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")+ criteria = [+ CompletionCriterion(+ id="c_status",+ outcome="The returned record includes status.",+ output_path="output.status",+ expected_output_shape="status_label",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"utility_peach_gas_quickconnect": {"status": status}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "structurally_abstained"+++ "deposit_amount",+ [+ "2026-06-22",+ "100245",+ "77 Gaslight Way, Decatur, GA 30030",+ "Submitted / Processing",+ "https://utility.example.test/pay",+ "initiation fee applies",+ ],+)+def test_requested_output_money_amount_shape_structurally_abstains_for_present_neighbors(deposit_amount: str) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("deposit_amount")+ criteria = [+ _criterion(+ "c_deposit_amount",+ "The output includes the deposit amount.",+ output_path="output.deposit_amount",+ expected_output_shape="money_amount",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"deposit_amount": deposit_amount}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "structurally_abstained"+++def test_requested_output_exact_wrong_value_beats_matching_shape() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_value="WTR-1842-DEMO",+ expected_output_shape="reference_code",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"confirmation_number": "QC-2002-DEMO"}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "evidence_contradicts"+++def test_requested_output_shape_missing_field_is_missing_exact_field() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")+ criteria = [+ CompletionCriterion(+ id="c_confirmation_number",+ outcome="The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"evidence_text": "WTR-1842-DEMO"}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "missing_exact_field"+++async def test_requested_output_path_requires_expected_value_match(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("value-grounded requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("service_address", "requested_start_date")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_address",+ outcome="The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ ),+ CompletionCriterion(+ id="c_date",+ outcome="The returned record includes requested start date.",+ output_path="output.requested_start_date",+ expected_output_value="2026-06-22",+ ),+ ]+ )++ swapped = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(+ {+ "service_address": "2026-06-22",+ "requested_start_date": "1234 Sample Utility Way",+ }+ ),+ time.monotonic(),+ )++ assert swapped is not None+ assert swapped.is_fully_satisfied() is False+ assert {verdict.reason_code for verdict in swapped.verdicts} == {"evidence_contradicts"}++ matched = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(+ {+ "service_address": "1234 Sample Utility Way",+ "requested_start_date": "2026-06-22",+ }+ ),+ time.monotonic(),+ )++ assert matched is not None+ assert matched.is_fully_satisfied() is True+ assert {verdict.reason_code for verdict in matched.verdicts} == {"evidence_confirms"}+++async def test_rehydrated_requested_output_expected_value_blocks_scrambled_output(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("value-grounded requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("service_address", "requested_start_date")+ criteria = (+ CompletionCriterion(+ id="c_address",+ outcome="The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ ),+ CompletionCriterion(+ id="c_date",+ outcome="The returned record includes requested start date.",+ output_path="output.requested_start_date",+ expected_output_value="2026-06-22",+ ),+ )+ ctx.request_policy = RequestPolicy(completion_criteria=list(criteria_from_json(criteria_to_json(criteria))))++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(+ {+ "service_address": "2026-06-22",+ "requested_start_date": "1234 Sample Utility Way",+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert {verdict.reason_code for verdict in verification.verdicts} == {"evidence_contradicts"}+++async def test_requested_output_path_does_not_match_block_label_only(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("requested-output criteria must not reach the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "npi")+ ctx.code_artifact_metadata = {"npi": _metadata_for_requested_paths("npi")["extract_profile"]}ctx.request_policy = RequestPolicy(completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")])+ verification = await _maybe_run_completion_verification(+ ctx,+ {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_requested_output",+ "overall_status": "completed",+ "executed_block_labels": ["npi"],+ "current_url": "https://example.test/profile",+ "blocks": [+ {+ "label": "npi",+ "block_type": "CODE",+ "status": "completed",+ "extracted_data": {"value": "1234567890"},+ }+ ],+ },+ },+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert verification.verdicts[0].reason_code == "no_evidence"+++ ("goal_value_path", "requested_path", "runtime_output", "evidence_ref"),+ [+ ("$.npi", "output.npi", {"npi": "1234567890"}, "block_outputs:extract_profile.npi"),+ ("$[*].npi", "output.npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.npi"),+ ("$[0].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),+ ("$[].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),+ (+ "records[0].npi",+ "output.records[].npi",+ {"records": [{"npi": "1234567890"}]},+ "block_outputs:extract_profile.records[].npi",+ ),+ (+ "output.records[0].npi",+ "output.records[].npi",+ {"records": [{"npi": "1234567890"}]},+ "block_outputs:extract_profile.records[].npi",+ ),+ ],+)+async def test_requested_output_path_normalizes_jsonpath_and_indexes(+ monkeypatch: pytest.MonkeyPatch,+ goal_value_path: str,+ requested_path: str,+ runtime_output: Any,+ evidence_ref: str,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("normalized requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths(goal_value_path)+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path=requested_path,+ expected_output_value="1234567890",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(runtime_output),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].evidence_ref == evidence_ref+++async def test_requested_output_path_aliases_authored_contract_to_requested_path(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("aliased requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_npi",+ outcome="The nested record NPI is returned.",+ output_path="output.records[].npi",+ expected_output_value="1234567890",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"records": [{"npi": "1234567890"}]}),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.records[].npi"+++async def test_requested_output_path_requires_exact_nested_runtime_field(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("requested-output criteria must not reach the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("locations[].address")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_address",+ outcome="Each listed location includes address.",+ output_path="output.locations[].address",+ expected_output_value="100 Main St",+ )+ ]+ )++ missing = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"locations": [{"name": "North Clinic"}]}),+ time.monotonic(),+ )+ assert missing is not None+ assert missing.is_fully_satisfied() is False+ assert missing.verdicts[0].reason_code == "missing_exact_field"+ assert missing.verdicts[0].missing_evidence == (+ "run output did not include exact structured field output.locations[].address"+ )++ satisfied = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"locations": [{"address": "100 Main St"}]}),+ time.monotonic(),+ )+ assert satisfied is not None+ assert satisfied.is_fully_satisfied() is True+ assert satisfied.verdicts[0].evidence_ref == "block_outputs:extract_profile.locations[].address"+++async def test_requested_output_path_uses_only_accepted_metadata(monkeypatch: pytest.MonkeyPatch) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("unproducible requested-output criteria must not reach the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.raw_code_artifact_metadata = _metadata_for_requested_paths("npi")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ )+ ]+ )+rejected = await _maybe_run_completion_verification(ctx,_requested_output_result({"npi": "1234567890"}),@@ -3607,7 +4585,14 @@ async def test_requested_output_path_can_use_static_return_key_contract(""")ctx.request_policy = RequestPolicy(- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]+ completion_criteria=[+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ )+ ])verification = await _maybe_run_completion_verification(@@ -3643,7 +4628,12 @@ async def test_requested_output_path_can_use_static_list_return_key_contract()ctx.request_policy = RequestPolicy(completion_criteria=[- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.[].npi")+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.[].npi",+ expected_output_value="1234567890",+ )])@@ -3673,7 +4663,12 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pytectx.code_artifact_metadata = _metadata_for_requested_paths("npi")ctx.request_policy = RequestPolicy(completion_criteria=[- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi"),+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ ),CompletionCriterion(id="c_cart", outcome="The cart contains the selected item."),])@@ -3691,6 +4686,78 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyteassert {verdict.criterion_id for verdict in verification.verdicts} == {"c_npi", "c_cart"}+async def test_present_generic_requested_output_without_expected_value_abstains_without_vetoing_run_success(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def handler(**_: object) -> dict:+ return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}++ _patch_completion_handler(monkeypatch, handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_customer_name",+ outcome="The returned record includes customer name.",+ output_path="output.customer_name",+ ),+ _criterion(+ "c_submit",+ "The record extraction completes.",+ kind="terminal_action",+ terminal_action_family="request",+ ),+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"customer_name": "Sample Customer"}),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}+ assert verdicts["c_customer_name"].reason_code == "structurally_abstained"+ assert verdicts["c_submit"].reason_code == "evidence_confirms"+++async def test_all_abstained_requested_outputs_do_not_satisfy_completion(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("requested-output only abstention should not reach the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_customer_name",+ outcome="The returned record includes customer name.",+ output_path="output.customer_name",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result({"customer_name": "Sample Customer"}),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert verification.verdicts[0].reason_code == "structurally_abstained"++@pytest.mark.asyncioasync def test_requested_output_bypasses_judge_satisfaction_without_exact_field(monkeypatch: pytest.MonkeyPatch,@@ -3714,7 +4781,7 @@ async def test_requested_output_bypasses_judge_satisfaction_without_exact_field(assert verification is not Noneassert verification.is_fully_satisfied() is False- assert verification.verdicts[0].reason_code == "missing_exact_field"+ assert verification.verdicts[0].reason_code == "no_evidence"@pytest.mark.asyncio@@ -3745,13 +4812,13 @@ async def test_requested_output_verdict_survives_unavailable_judge_result(assert verification.status == "evaluated"assert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_npi"].reason_code == "missing_exact_field"+ assert verdicts["c_npi"].reason_code == "no_evidence"assert verdicts["c_npi"].satisfied is Falseassert verdicts["c_cart"].state == "unknown"@pytest.mark.asyncio[email protected]("reason_code", ["missing_exact_field", "unproducible"])[email protected]("reason_code", ["no_evidence", "unproducible"])async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_run_criterion(monkeypatch: pytest.MonkeyPatch,reason_code: str,@@ -3762,7 +4829,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied__patch_completion_handler(monkeypatch, fail_handler)ctx = _run_ctx()_set_workflow_labels(ctx, "extract_profile")- if reason_code == "missing_exact_field":+ if reason_code == "no_evidence":ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")output = {"status": "DONE", "blocker": False}else:@@ -3791,7 +4858,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_@pytest.mark.asyncio[email protected]("reason_code", ["missing_exact_field", "unproducible"])[email protected]("reason_code", ["no_evidence", "unproducible"])async def test_fired_contingent_requested_output_miss_still_vetoes(monkeypatch: pytest.MonkeyPatch,reason_code: str,@@ -3802,7 +4869,7 @@ async def test_fired_contingent_requested_output_miss_still_vetoes(_patch_completion_handler(monkeypatch, fail_handler)ctx = _run_ctx()_set_workflow_labels(ctx, "extract_profile")- if reason_code == "missing_exact_field":+ if reason_code == "no_evidence":ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")output = {"status": "DONE", "blocker": True}else:@@ -3876,7 +4943,12 @@ async def test_requested_output_satisfaction_is_not_vetoed_by_fallback_record_abctx.code_artifact_metadata = _metadata_for_requested_paths("npi")ctx.request_policy = RequestPolicy(completion_criteria=[- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi"),+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ ),*_structured_record_criteria(),])@@ -4518,7 +5590,7 @@ async def test_download_registered_output_parameter_injects_and_verifies_without@pytest.mark.asyncio-async def test_marked_requested_output_id_still_requires_exact_field(+async def test_marked_requested_output_id_without_expected_value_is_no_evidence(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:@@ -4551,13 +5623,13 @@ async def test_marked_requested_output_id_still_requires_exact_field(assert verification is not Noneassert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert verdicts["c_output_id"].reason_code == "no_evidence"assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdictsassert ctx.request_policy.completion_criteria == [requested]@pytest.mark.asyncio-async def test_marked_requested_output_npi_still_requires_exact_field(+async def test_marked_requested_output_npi_without_expected_value_is_no_evidence(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:@@ -4590,13 +5662,13 @@ async def test_marked_requested_output_npi_still_requires_exact_field(assert verification is not Noneassert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_npi"].reason_code == "missing_exact_field"+ assert verdicts["c_npi"].reason_code == "no_evidence"assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdictsassert ctx.request_policy.completion_criteria == [requested]@pytest.mark.asyncio-async def test_unmarked_requested_output_id_still_requires_exact_field(+async def test_unmarked_requested_output_id_without_expected_value_is_no_evidence(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:@@ -4631,7 +5703,7 @@ async def test_unmarked_requested_output_id_still_requires_exact_field(assert verification is not Noneassert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert verdicts["c_output_id"].reason_code == "no_evidence"@pytest.mark.asyncio@@ -4672,8 +5744,8 @@ async def test_marked_download_deliverable_does_not_remove_mixed_extraction_fielassert verification is not Noneassert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_output_id"].reason_code == "missing_exact_field"- assert verdicts["c_npi"].reason_code == "missing_exact_field"+ assert verdicts["c_output_id"].reason_code == "no_evidence"+ assert verdicts["c_npi"].reason_code == "no_evidence"assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts@@ -4758,7 +5830,7 @@ async def test_marked_download_deliverable_without_registered_evidence_fails_cloassert verification is not Noneassert verification.is_fully_satisfied() is Falseverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}- assert verdicts["c_output_id"].reason_code == "missing_exact_field"+ assert verdicts["c_output_id"].reason_code == "no_evidence"assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "no_evidence"diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex 4a86d060d..2ac2c71f3 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -63,6 +63,8 @@ def _criterion(contingent_on: str | None = None,contingent_antecedent_output_path: str | None = None,deliverable_kind: str | None = None,+ expected_output_value: str | None = None,+ expected_output_shape: str | None = None,) -> CompletionCriterion:return CompletionCriterion(id=cid,@@ -73,6 +75,8 @@ def _criterion(contingent_on=contingent_on,contingent_antecedent_output_path=contingent_antecedent_output_path,deliverable_kind=deliverable_kind, # type: ignore[arg-type]+ expected_output_value=expected_output_value,+ expected_output_shape=expected_output_shape, # type: ignore[arg-type])@@ -260,6 +264,247 @@ async def test_classifier_drift_canonicalizes_to_stable_generic_requested_output)+async def test_requested_output_criteria_preserve_stable_paths_without_derived_carriers() -> None:+ prompt = "Return a final record with service address 1234 Sample Utility Way and requested start date 2026-06-22."+ classifier_shapes = [+ [+ {+ "outcome": "The returned record includes requested start date 2026-06-22.",+ "output_path": "output.requested_start_date",+ },+ {+ "outcome": "The returned record includes service address 1234 Sample Utility Way.",+ "output_path": "output.service_address",+ },+ ],+ [+ {"outcome": "The returned record includes service address.", "output_path": "output.service_address"},+ {+ "outcome": "The returned record includes requested start date.",+ "output_path": "output.requested_start_date",+ },+ {"outcome": "The workflow runs to an expected result.", "method_mandated": True},+ ],+ ]++ policies = [await _policy_for_message(prompt, criteria) for criteria in classifier_shapes]+ requested_output_paths = {"output.service_address", "output.requested_start_date"}+ canonical_subsets = [_requested_output_subset(policy, requested_output_paths) for policy in policies]+ canonical_json = [criteria_to_json(subset) for subset in canonical_subsets]++ assert canonical_json == [canonical_json[0], canonical_json[0]]+ assert [criterion.id for criterion in canonical_subsets[0]] == [+ "__copilot_requested_output__output_service_address",+ "__copilot_requested_output__output_requested_start_date",+ ]+ assert [criterion.output_path for criterion in canonical_subsets[0]] == [+ "output.service_address",+ "output.requested_start_date",+ ]+ assert [criterion.outcome for criterion in canonical_subsets[0]] == [+ "The returned record includes service address.",+ "The returned record includes requested start date.",+ ]+ assert [criterion.expected_output_value for criterion in canonical_subsets[0]] == [+ None,+ None,+ ]+ assert [criterion.expected_output_shape for criterion in canonical_subsets[0]] == [None, None]+++async def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:+ policy = await _policy_for_message(+ "Return a final record with provider captured address, account number, requested date, status, "+ "confirmation number, deposit amount, and next owner.",+ [],+ )++ requested_output_paths = {+ "output.provider_captured_address",+ "output.account_number",+ "output.requested_date",+ "output.status",+ "output.confirmation_number",+ "output.deposit_amount",+ "output.next_owner",+ }++ assert {+ criterion.output_path: criterion.expected_output_shape+ for criterion in _requested_output_subset(policy, requested_output_paths)+ } == {+ "output.provider_captured_address": None,+ "output.account_number": None,+ "output.requested_date": None,+ "output.status": None,+ "output.confirmation_number": None,+ "output.deposit_amount": None,+ "output.next_owner": None,+ }+++ ("prompt", "output_path", "_expected_value"),+ [+ (+ "Return a final record with service URL https://example.com/a:b.",+ "output.service_url",+ "https://example.com/a:b",+ ),+ ("Return a final record with tax rate 1.25.", "output.tax_rate", "1.25"),+ ("Return a final record with company domain example.com.", "output.company_domain", "example.com"),+ (+ "Return a final record with department name Research and Development.",+ "output.department_name",+ "Research and Development",+ ),+ ],+)+async def test_requested_output_expected_values_are_not_synthesized_from_user_message_regex(+ prompt: str, output_path: str, _expected_value: str+) -> None:+ policy = await _policy_for_message(prompt, [])++ criteria = _criteria_for_path(policy, output_path)+ assert len(criteria) == 1+ assert (+ criteria[0].outcome == f"The returned record includes {output_path.removeprefix('output.').replace('_', ' ')}."+ )+ assert criteria[0].expected_output_value is None+++async def test_message_text_cannot_override_classifier_typed_expected_output_value() -> None:+ policy = await _policy_for_message(+ "Return a final record with service address 999 Wrong Way.",+ [+ {+ "outcome": "The returned record includes service address.",+ "output_path": "output.service_address",+ "expected_output_value": "1234 Sample Utility Way",+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.service_address")+ assert len(criteria) == 1+ assert criteria[0].expected_output_value == "1234 Sample Utility Way"+++async def test_classifier_typed_expected_output_value_is_preserved_without_outcome_carrier() -> None:+ policy = await _policy_for_message(+ "Return a final record with service address.",+ [+ {+ "outcome": "The returned record includes service address.",+ "output_path": "output.service_address",+ "expected_output_value": "1234 Sample Utility Way",+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.service_address")+ assert len(criteria) == 1+ assert criteria[0].outcome == "The returned record includes service address."+ assert criteria[0].expected_output_value == "1234 Sample Utility Way"+++async def test_classifier_typed_expected_output_shape_is_preserved_without_outcome_carrier() -> None:+ policy = await _policy_for_message(+ "Return a final record with confirmation number.",+ [+ {+ "outcome": "The returned record includes confirmation number.",+ "output_path": "output.confirmation_number",+ "expected_output_shape": "reference_code",+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.confirmation_number")+ assert len(criteria) == 1+ assert criteria[0].expected_output_shape == "reference_code"+++async def test_classifier_invalid_expected_output_shape_is_ignored() -> None:+ policy = await _policy_for_message(+ "Return a final record with custom value.",+ [+ {+ "outcome": "The returned record includes custom value.",+ "output_path": "output.custom_value",+ "expected_output_shape": "anything_non_empty",+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.custom_value")+ assert len(criteria) == 1+ assert criteria[0].expected_output_shape is None+++def test_parse_completion_criteria_accepts_only_declared_expected_output_shapes() -> None:+ valid, invalid = _parse_completion_criteria(+ [+ {+ "outcome": "The returned record includes confirmation number.",+ "output_path": "output.confirmation_number",+ "expected_output_shape": "reference_code",+ },+ {+ "outcome": "The returned record includes custom value.",+ "output_path": "output.custom_value",+ "expected_output_shape": "anything_non_empty",+ },+ ]+ )++ assert valid.expected_output_shape == "reference_code"+ assert invalid.expected_output_shape is None+++async def test_requested_output_canonicalization_preserves_classifier_shape_first() -> None:+ policy = await _policy_for_message(+ "Return a final record with confirmation number.",+ [+ {+ "outcome": "The returned record includes confirmation number.",+ "output_path": "output.confirmation_number",+ "expected_output_shape": "numeric_identifier",+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.confirmation_number")+ assert len(criteria) == 1+ assert criteria[0].expected_output_shape == "numeric_identifier"+++async def test_classifier_non_string_expected_output_value_is_ignored() -> None:+ policy = await _policy_for_message(+ "Return a final record with service address.",+ [+ {+ "outcome": "The returned record includes service address.",+ "output_path": "output.service_address",+ "expected_output_value": {"value": "1234 Sample Utility Way"},+ }+ ],+ )++ criteria = _criteria_for_path(policy, "output.service_address")+ assert len(criteria) == 1+ assert criteria[0].expected_output_value is None++@pytest.mark.asyncioasync def test_method_and_setup_text_do_not_become_completion_criteria() -> None:policy = await _policy_for_message(@@ -288,7 +533,7 @@ async def test_reusable_input_id_does_not_cover_requested_output_id() -> None:@pytest.mark.asyncio-async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_gate_remains() -> None:+async def test_unbound_requested_output_narrative_is_replaced_but_specific_run_gate_remains() -> None:policy = await _policy_for_message("Return a final record with record id, status, and phone.",[@@ -308,6 +553,34 @@ async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_} == {"output.record_id", "output.status", "output.phone"}+async def test_generic_ungrounded_run_gate_drops_after_requested_output_canonicalization() -> None:+ policy = await _policy_for_message(+ "Return a final record with record id.",+ [{"outcome": "The workflow runs to its intended end state with the expected output."}],+ )++ assert "The workflow runs to its intended end state with the expected output." not in _outcomes(policy)+ assert _criteria_for_path(policy, "output.record_id")+++async def test_terminal_action_run_gate_survives_requested_output_canonicalization() -> None:+ policy = await _policy_for_message(+ "Submit the request and return a final record with confirmation number.",+ [+ {+ "outcome": "The request submission action is completed.",+ "kind": "terminal_action",+ "terminal_action_family": "request",+ },+ ],+ )++ assert "The request submission action is completed." in _outcomes(policy)+ assert _criteria_for_path(policy, "output.confirmation_number")++@pytest.mark.asyncioasync def test_requested_output_coverage_uses_whole_token_sequence_matching() -> None:policy = await _policy_for_message(@@ -592,6 +865,58 @@ def test_active_criteria_rendering_includes_deliverable_kind() -> None:]+def test_active_criteria_rendering_includes_expected_output_value_as_structured_data() -> None:+ rendered = _render_active_criteria_for_prompt(+ [+ _criterion(+ "c0",+ "The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ )+ ]+ )++ assert json.loads(rendered) == [+ {+ "outcome": "The returned record includes service address.",+ "implicit": False,+ "method_mandated": False,+ "level": "run",+ "kind": "outcome",+ "terminal_action_family": None,+ "output_path": "output.service_address",+ "expected_output_value": "1234 Sample Utility Way",+ }+ ]+++def test_active_criteria_rendering_includes_expected_output_shape_as_structured_data() -> None:+ rendered = _render_active_criteria_for_prompt(+ [+ _criterion(+ "c0",+ "The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ )+ ]+ )++ assert json.loads(rendered) == [+ {+ "outcome": "The returned record includes confirmation number.",+ "implicit": False,+ "method_mandated": False,+ "level": "run",+ "kind": "outcome",+ "terminal_action_family": None,+ "output_path": "output.confirmation_number",+ "expected_output_shape": "reference_code",+ }+ ]++def test_criteria_json_round_trips_contingent_on() -> None:criteria = (_criterion(@@ -606,6 +931,131 @@ def test_criteria_json_round_trips_contingent_on() -> None:assert restored == criteria+def test_criteria_json_round_trips_expected_output_value() -> None:+ criteria = (+ _criterion(+ "c0",+ "The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ ),+ )++ restored = criteria_from_json(criteria_to_json(criteria))++ assert restored == criteria+++def test_criteria_json_round_trips_expected_output_shape() -> None:+ criteria = (+ _criterion(+ "c0",+ "The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ )++ restored = criteria_from_json(criteria_to_json(criteria))++ assert restored == criteria+++def test_request_policy_trace_exposes_requested_output_grounding_contract_without_values() -> None:+ policy = RequestPolicy(+ completion_criteria=[+ _criterion(+ "c_address",+ "The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ ),+ _criterion(+ "c_confirmation",+ "The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ ),+ _criterion("c_status", "The returned record includes status.", output_path="output.status"),+ ]+ )++ trace = policy.to_trace_data()++ assert trace["requested_output_criteria_count"] == 3+ assert trace["requested_output_criterion_0_id"] == "c_address"+ assert trace["requested_output_criterion_0_output_path"] == "output.service_address"+ assert trace["requested_output_criterion_0_grounding_mode"] == "exact_value"+ assert trace["requested_output_criterion_0_has_exact_value"] is True+ assert "requested_output_criterion_0_expected_output_shape" not in trace+ assert trace["requested_output_criterion_1_id"] == "c_confirmation"+ assert trace["requested_output_criterion_1_output_path"] == "output.confirmation_number"+ assert trace["requested_output_criterion_1_grounding_mode"] == "shape"+ assert trace["requested_output_criterion_1_expected_output_shape"] == "reference_code"+ assert trace["requested_output_criterion_1_has_exact_value"] is False+ assert trace["requested_output_criterion_2_id"] == "c_status"+ assert trace["requested_output_criterion_2_output_path"] == "output.status"+ assert trace["requested_output_criterion_2_grounding_mode"] == "missing"+ assert trace["requested_output_criterion_2_has_exact_value"] is False+ assert "1234 Sample Utility Way" not in repr(trace)+++def test_reconcile_supersedes_same_output_path_with_different_expected_output_value() -> None:+ stored = _stored(+ _criterion(+ "s0",+ "The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="1234 Sample Utility Way",+ )+ )+ fresh = [+ _criterion(+ "c0",+ "The returned record includes service address.",+ output_path="output.service_address",+ expected_output_value="7890 Changed Ave",+ )+ ]++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ fresh,+ actionable=True,+ )++ assert decision.action == "create"+ assert decision.criteria == tuple(fresh)+++def test_reconcile_supersedes_same_output_path_with_different_expected_output_shape() -> None:+ stored = _stored(+ _criterion(+ "s0",+ "The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="reference_code",+ )+ )+ fresh = [+ _criterion(+ "c0",+ "The returned record includes confirmation number.",+ output_path="output.confirmation_number",+ expected_output_shape="numeric_identifier",+ )+ ]++ decision = reconcile_completion_criteria(+ StoredCriteriaSnapshot(active=stored, next_epoch=2),+ fresh,+ actionable=True,+ )++ assert decision.action == "create"+ assert decision.criteria == tuple(fresh)++def test_criteria_json_round_trips_contingent_antecedent_output_path() -> None:criteria = (_criterion(diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.pyindex b755739bc..21068227a 100644--- a/tests/unit/test_copilot_request_policy_prompt.py+++ b/tests/unit/test_copilot_request_policy_prompt.py@@ -89,8 +89,11 @@ class TestRequestPolicyPromptStructure:rendered = _render()assert ("{outcome, contingent_on, contingent_antecedent_output_path, "- "deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}"+ "deliverable_kind, implicit, method_mandated, level, output_path, expected_output_value, "+ "expected_output_shape, kind, terminal_action_family}") in rendered+ assert "never hide it in outcome prose" in rendered+ assert "reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label" in renderedassert "kind=outcome|terminal_action" in renderedassert "terminal_action_family=request|application|form|order|null" in renderedassert 'The only supported non-null value is "registered_download"' in rendered@@ -101,6 +104,8 @@ class TestRequestPolicyPromptStructure:CompletionCriterion(id="c0",outcome="a commercial water service request is started",+ expected_output_value="WTR-1842-DEMO",+ expected_output_shape="reference_code",kind="terminal_action",terminal_action_family="request",)@@ -111,6 +116,8 @@ class TestRequestPolicyPromptStructure:assert '"kind": "terminal_action"' in renderedassert '"terminal_action_family": "request"' in rendered+ assert '"expected_output_value": "WTR-1842-DEMO"' in rendered+ assert '"expected_output_shape": "reference_code"' in renderedclass TestRawSecretBackstop: