Skip to content

Automate browser based workflows with AI

SKY-11544: Fix value-grounded Copilot completion criteria (#6899)

Andrew Neilsone75f13c8b815
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 67d1fa256..dd75a36d3 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -25,11 +25,13 @@ completion_contract:
- Set to null when the user does not provide an explicit completion condition.
completion_criteria:
-- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}.
+- An array of the END-STATE OUTCOMES that must be true for the user's goal to be done. Each item is {outcome, contingent_on, contingent_antecedent_output_path, deliverable_kind, implicit, method_mandated, level, output_path, expected_output_value, expected_output_shape, kind, terminal_action_family}.
- outcome: a short statement of an end state the workflow must achieve (an END, not a step). Extract the goal as outcomes the user wants to be true, not the recipe they describe.
- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.
- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.
- deliverable_kind: null unless this criterion's requested output is satisfied by execution-owned browser-download registration rather than a user-authored extraction field. The only supported non-null value is "registered_download".
+- expected_output_value: null unless this criterion requires an exact structured output value the user explicitly provided. Put that value here as data; never hide it in outcome prose.
+- expected_output_shape: null unless this criterion requires a generated/provider structured output value whose exact value is not known before the run. Use only one of: reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label. Do not use this as a non-empty-value proxy; leave null when the field's shape is not one of those declared types.
- kind=outcome|terminal_action. Use terminal_action only when the criterion's end state is a terminal action in the external site for one of the supported families below. Use outcome for all other outcomes.
- terminal_action_family=request|application|form|order|null. Set the matching family when kind=terminal_action; set null when kind=outcome.
- Capture ALL distinct outcomes the goal requires, including implicit constraints the user did not state as a step (for example "added exactly once / not duplicated", "the item is present in the cart"). Mark such items implicit=true.
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 7fce09698..2389c7ab3 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -1978,7 +1978,10 @@ if settings.ENABLE_OPENROUTER:
["OPENROUTER_API_KEY"],
supports_vision=True,
add_assistant_prefix=False,
- max_completion_tokens=131072,
+ # MiMo v2.5 is a reasoning model; uncapped it emits 100k+ reasoning/output
+ # tokens (multi-minute calls) on short prompts. Cap output + reasoning effort.
+ max_completion_tokens=16384,
+ reasoning_effort="low",
litellm_params=LiteLLMParams(
api_key=settings.OPENROUTER_API_KEY,
api_base=settings.OPENROUTER_API_BASE,
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index ca24fb2c2..c4ac27145 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -18,7 +18,9 @@ from skyvern.forge.sdk.copilot.completion_verification import (
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
CriterionKind,
+ ExpectedOutputShape,
TerminalActionFamily,
+ _coerce_expected_output_shape,
_normalize_contingent_antecedent_output_path,
_normalize_deliverable_kind,
is_fallback_floor_criterion,
@@ -123,6 +125,8 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
"method_mandated": criterion.method_mandated,
"level": criterion.level,
"output_path": criterion.output_path,
+ "expected_output_value": criterion.expected_output_value,
+ "expected_output_shape": criterion.expected_output_shape,
"kind": criterion.kind,
"terminal_action_family": criterion.terminal_action_family,
}
@@ -143,6 +147,8 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
continue
level = item.get("level")
output_path = item.get("output_path")
+ expected_output_value = item.get("expected_output_value")
+ expected_output_shape = _coerce_expected_output_shape(item.get("expected_output_shape"))
contingent_on = item.get("contingent_on")
contingent_antecedent_output_path = _normalize_contingent_antecedent_output_path(
item.get("contingent_antecedent_output_path")
@@ -166,6 +172,10 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
method_mandated=bool(item.get("method_mandated")),
level=level if isinstance(level, str) and level in _CRITERION_LEVELS else "run", # type: ignore[arg-type]
output_path=output_path.strip() if isinstance(output_path, str) and output_path.strip() else None,
+ expected_output_value=expected_output_value.strip()
+ if isinstance(expected_output_value, str) and expected_output_value.strip()
+ else None,
+ expected_output_shape=cast(ExpectedOutputShape | None, expected_output_shape),
kind=cast(CriterionKind, kind),
terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),
)
@@ -177,11 +187,15 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
deliverable_kind_key = criterion.deliverable_kind or ""
+ expected_output_value_key = criterion.expected_output_value or ""
+ expected_output_shape_key = criterion.expected_output_shape or ""
if criterion.output_path:
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
f"\x1fdeliverable_kind:{deliverable_kind_key}"
f"\x1foutput_path:{criterion.output_path}"
+ f"\x1fexpected_output_value:{expected_output_value_key}"
+ f"\x1fexpected_output_shape:{expected_output_shape_key}"
)
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
@@ -203,7 +217,7 @@ _REQUESTED_OUTPUT_WORDS = frozenset(
)
_REQUESTED_OUTPUT_FIELD_TOKENS = frozenset(
"address addresses date dates email emails id identifier identifiers license licenses location locations name names "
- "npi number numbers phone phones result specialties specialty status statuses taxonomy".split()
+ "npi number numbers owner owners phone phones result specialties specialty status statuses taxonomy".split()
)
_GENERIC_PROFILE_MARKERS = (
"profile details",
diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.py
index 395ee180c..8f6461e25 100644
--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py
+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py
@@ -5,7 +5,7 @@ import json
import re
import textwrap
from collections.abc import Mapping
-from typing import Any, Protocol
+from typing import Any, Literal, Protocol
import yaml
@@ -19,6 +19,7 @@ from skyvern.utils.yaml_loader import safe_load_no_dates
_GOAL_PATH_INDEX_PATTERN = re.compile(r"\[\d+\]")
_REQUESTED_OUTPUT_PREFIX = "output."
+_STRUCTURAL_ABSTENTION_REASON_CODE = "structurally_abstained"
# Ignored at any requested-output traversal depth, including schema-declared nested fields.
_IGNORED_RUNTIME_FIELD_NAMES = frozenset({"evidence_text"})
@@ -67,14 +68,83 @@ def grade_requested_output_criteria(
)
)
continue
- evidence_ref = _runtime_output_path_evidence_ref(snapshot.block_outputs, path)
- if evidence_ref is not None:
+ expected_value = criterion.expected_output_value
+ expected_shape = criterion.expected_output_shape
+ grounding_mode = _criterion_grounding_mode(criterion)
+ trace_fields = _criterion_trace_fields(criterion, grounding_mode)
+ if expected_value is not None:
+ match_state, evidence_ref = _runtime_output_path_match(snapshot.block_outputs, path, expected_value)
+ elif expected_shape is not None:
+ match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)
+ else:
+ match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)
+ if match_state == "present" and evidence_ref is not None:
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unsatisfied",
+ reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,
+ evidence_ref=evidence_ref,
+ missing_evidence=(
+ "requested-output field is present, but the criterion has no typed expected_output_value "
+ "that can prove or refute the value"
+ ),
+ **trace_fields,
+ )
+ )
+ continue
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unsatisfied",
+ reason_code="no_evidence",
+ missing_evidence=(
+ "requested-output criterion lacks typed expected_output_value or expected_output_shape; "
+ "presence-only output cannot confirm value-grounded criterion"
+ ),
+ **trace_fields,
+ )
+ )
+ continue
+ if match_state == "present" and evidence_ref is not None:
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unsatisfied",
+ reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,
+ evidence_ref=evidence_ref,
+ missing_evidence=(
+ "requested-output field is present with a typed expected_output_shape, but no exact "
+ "expected_output_value can prove or refute the value"
+ ),
+ **trace_fields,
+ )
+ )
+ continue
+ if match_state == "satisfied" and evidence_ref is not None:
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
state="satisfied",
reason_code="evidence_confirms",
evidence_ref=evidence_ref,
+ **trace_fields,
+ )
+ )
+ continue
+ if match_state == "contradicted":
+ verdicts.append(
+ CriterionVerdict(
+ criterion_id=criterion.id,
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ evidence_ref=evidence_ref,
+ missing_evidence=(
+ f"run output included output.{path} but it did not match the expected value"
+ if expected_value is not None
+ else f"run output included output.{path} but it could not be value-grounded"
+ ),
+ **trace_fields,
)
)
continue
@@ -84,11 +154,29 @@ def grade_requested_output_criteria(
state="unsatisfied",
reason_code="missing_exact_field",
missing_evidence=f"run output did not include exact structured field output.{path}",
+ **trace_fields,
)
)
return verdicts
+def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:
+ if criterion.expected_output_value is not None:
+ return "exact_value"
+ if criterion.expected_output_shape is not None:
+ return "shape"
+ return "missing"
+
+
+def _criterion_trace_fields(criterion: CompletionCriterion, grounding_mode: str) -> dict[str, Any]:
+ return {
+ "output_path": criterion.output_path,
+ "grounding_mode": grounding_mode,
+ "expected_output_shape": criterion.expected_output_shape,
+ "has_exact_value": criterion.expected_output_value is not None,
+ }
+
+
def _authored_output_contract_paths(copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None) -> set[str]:
metadata = copilot_ctx.code_artifact_metadata
if not isinstance(metadata, Mapping):
@@ -244,40 +332,85 @@ def _dict_keys(node: ast.Dict) -> set[str]:
return {key.value for key in node.keys if isinstance(key, ast.Constant) and isinstance(key.value, str)}
-def _runtime_output_path_evidence_ref(block_outputs: Mapping[str, Any], path: str) -> str | None:
+def _runtime_output_path_match(
+ block_outputs: Mapping[str, Any], path: str, expected_value: str | None
+) -> tuple[str, str | None]:
parts = _path_parts(path)
- if not parts:
- return None
+ if not parts or expected_value is None:
+ return "missing", None
+ contradicted_ref: str | None = None
for label, payload in block_outputs.items():
- if _path_has_meaningful_value(payload, parts):
- return f"block_outputs:{label}.{path}"
- return None
+ values, source_path = _runtime_path_values(payload, parts, path)
+ if not values:
+ continue
+ evidence_ref = f"block_outputs:{label}.{source_path}"
+ if any(_value_matches_expected(value, expected_value) for value in values):
+ return "satisfied", evidence_ref
+ contradicted_ref = contradicted_ref or evidence_ref
+ if contradicted_ref is not None:
+ return "contradicted", contradicted_ref
+ return "missing", None
-def _path_has_meaningful_value(value: Any, parts: list[str]) -> bool:
+def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -> tuple[str, str | None]:
+ parts = _path_parts(path)
if not parts:
- return _is_meaningful_requested_value(value)
+ return "missing", None
+ for label, payload in block_outputs.items():
+ values, source_path = _runtime_present_path_values(payload, parts, path)
+ if not values:
+ continue
+ evidence_ref = f"block_outputs:{label}.{source_path}"
+ return "present", evidence_ref
+ return "missing", None
+
+
+def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:
+ canonical_values = _path_values(payload, parts)
+ if canonical_values:
+ filtered_canonical_values = [value for value in canonical_values if _is_meaningful_requested_value(value)]
+ return filtered_canonical_values or canonical_values, path
+ wrapped_parts = ["output", *parts]
+ wrapped_path = ".".join(wrapped_parts)
+ wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]
+ return wrapped_values, wrapped_path
+
+
+def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:
+ raw_canonical_values = _path_values(payload, parts)
+ if raw_canonical_values:
+ canonical_values = [value for value in raw_canonical_values if _is_meaningful_requested_value(value)]
+ return canonical_values, path
+ wrapped_parts = ["output", *parts]
+ wrapped_path = ".".join(wrapped_parts)
+ wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]
+ return wrapped_values, wrapped_path
+
+
+def _path_values(value: Any, parts: list[str]) -> list[Any]:
+ if not parts:
+ return [value]
part = parts[0]
if part == "[]":
if not isinstance(value, list):
- return False
- return any(_path_has_meaningful_value(item, parts[1:]) for item in value)
+ return []
+ return [matched for item in value for matched in _path_values(item, parts[1:])]
expects_array = part.endswith("[]")
key = part[:-2] if expects_array else part
if key in _IGNORED_RUNTIME_FIELD_NAMES:
- return False
+ return []
if isinstance(value, Mapping):
if key not in value:
- return False
+ return []
child = value[key]
if expects_array:
if not isinstance(child, list):
- return False
- return any(_path_has_meaningful_value(item, parts[1:]) for item in child)
- return _path_has_meaningful_value(child, parts[1:])
+ return []
+ return [matched for item in child for matched in _path_values(item, parts[1:])]
+ return _path_values(child, parts[1:])
if isinstance(value, list):
- return any(_path_has_meaningful_value(item, parts) for item in value)
- return False
+ return [matched for item in value for matched in _path_values(item, parts)]
+ return []
def _is_meaningful_requested_value(value: Any) -> bool:
@@ -292,6 +425,26 @@ def _is_meaningful_requested_value(value: Any) -> bool:
return True
+def _value_matches_expected(value: Any, expected: str) -> bool:
+ if isinstance(value, Mapping):
+ return any(_value_matches_expected(item, expected) for item in value.values())
+ if isinstance(value, list):
+ return any(_value_matches_expected(item, expected) for item in value)
+ observed = _normalized_expected_text(value)
+ target = _normalized_expected_text(expected)
+ if not observed or not target:
+ return False
+ return observed == target or _compact_expected_text(observed) == _compact_expected_text(target)
+
+
+def _normalized_expected_text(value: Any) -> str:
+ return " ".join(str(value).casefold().split())
+
+
+def _compact_expected_text(value: str) -> str:
+ return "".join(char for char in value if char.isalnum())
+
+
def _normalize_output_path(path: str | None) -> str:
if not isinstance(path, str):
return ""
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 2582f1f0f..b35862840 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -13,7 +13,7 @@ from __future__ import annotations
import asyncio
import json
import re
-from collections.abc import Iterable
+from collections.abc import Collection, Iterable
from dataclasses import dataclass, field
from typing import Any, Literal
@@ -40,6 +40,7 @@ _MISSING_EVIDENCE_MAX_CHARS = 500
_MAX_BLOCK_OUTPUTS = 20
_MAX_TRACE_VERDICTS = 8
_REASON_CODES = frozenset({"evidence_confirms", "no_evidence", "evidence_contradicts", "unknown"})
+_STRUCTURAL_ABSTENTION_REASON_CODE = "structurally_abstained"
_CONTINGENT_ABSTENTION_REASON_CODES = frozenset(
{"unknown", "no_evidence", "evidence_contradicts", "missing_exact_field", "unproducible"}
)
@@ -78,6 +79,10 @@ class CriterionVerdict:
reason_code: str
evidence_ref: str | None = None
missing_evidence: str | None = None
+ output_path: str | None = None
+ grounding_mode: Literal["exact_value", "shape", "missing"] | None = None
+ expected_output_shape: str | None = None
+ has_exact_value: bool = False
@property
def satisfied(self) -> bool:
@@ -119,6 +124,8 @@ class CompletionVerificationResult:
self.structural_unfired_criterion_ids,
):
continue
+ if verdict is not None and _is_structural_requested_output_abstention(verdict):
+ continue
return False
return satisfied_run_plane_count > 0
@@ -133,7 +140,9 @@ class CompletionVerificationResult:
unmet = [
verdict
for verdict in self.verdicts
- if not verdict.satisfied and not self.is_structural_contingent_abstention(verdict)
+ if not verdict.satisfied
+ and not self.is_structural_contingent_abstention(verdict)
+ and not _is_structural_requested_output_abstention(verdict)
]
missing_evidence: list[str] = []
for verdict in unmet:
@@ -162,6 +171,19 @@ class CompletionVerificationResult:
data[f"{prefix}_state"] = verdict.state
data[f"{prefix}_satisfied"] = verdict.satisfied
data[f"{prefix}_reason_code"] = verdict.reason_code
+ if verdict.output_path:
+ data[f"{prefix}_output_path"] = verdict.output_path
+ if verdict.grounding_mode:
+ data[f"{prefix}_grounding_mode"] = verdict.grounding_mode
+ if verdict.expected_output_shape:
+ data[f"{prefix}_expected_output_shape"] = verdict.expected_output_shape
+ if (
+ verdict.output_path
+ or verdict.grounding_mode
+ or verdict.expected_output_shape
+ or verdict.has_exact_value
+ ):
+ data[f"{prefix}_has_exact_value"] = verdict.has_exact_value
if contingent_on := self.contingent_on_by_criterion_id.get(verdict.criterion_id):
data[f"{prefix}_contingent_on"] = contingent_on
if contingent_path := self.contingent_antecedent_output_path_by_criterion_id.get(verdict.criterion_id):
@@ -172,7 +194,12 @@ class CompletionVerificationResult:
evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)
if evidence_ref:
data[f"{prefix}_evidence_ref"] = evidence_ref
- detail = None if self.is_structural_contingent_abstention(verdict) else verdict_missing_evidence(verdict)
+ detail = (
+ None
+ if self.is_structural_contingent_abstention(verdict)
+ or _is_structural_requested_output_abstention(verdict)
+ else verdict_missing_evidence(verdict)
+ )
if detail:
data[f"{prefix}_missing_evidence"] = detail
return data
@@ -398,7 +425,7 @@ def _field_aliases_for_output_path(path: str) -> tuple[str, str]:
return field, f"{field}_output"
-def _find_structured_field_values(value: Any, field_names: set[str]) -> Iterable[Any]:
+def _find_structured_field_values(value: Any, field_names: Collection[str]) -> Iterable[Any]:
if isinstance(value, dict):
for key, item in value.items():
if isinstance(key, str) and key in field_names:
@@ -424,6 +451,12 @@ def structural_unfired_contingent_criterion_ids(
if key in field_names:
values.append(value)
values.extend(_find_structured_field_values(value, field_names))
+ if _is_blocker_contingent_criterion(criterion):
+ if _has_real_blocker_evidence(snapshot):
+ continue
+ if _has_structural_no_blocker_evidence(snapshot):
+ unfired_ids.append(criterion.id)
+ continue
if not values:
continue
if any(_is_meaningful_contingent_antecedent_value(value) for value in values):
@@ -432,6 +465,63 @@ def structural_unfired_contingent_criterion_ids(
return unfired_ids
+def _is_blocker_contingent_criterion(criterion: CompletionCriterion) -> bool:
+ path = criterion.contingent_antecedent_output_path
+ if path is None:
+ return False
+ return _output_path_field(path) in {"blocker", "manual_service_blocker"}
+
+
+_BLOCKER_FIELDS = frozenset({"blocker", "blocker_output", "manual_service_blocker", "manual_service_blocker_output"})
+
+
+def _blocker_family_values(snapshot: RunEvidenceSnapshot) -> list[Any]:
+ values: list[Any] = []
+ for key, value in snapshot.block_outputs.items():
+ if key in _BLOCKER_FIELDS:
+ values.append(value)
+ values.extend(_find_structured_field_values(value, _BLOCKER_FIELDS))
+ return values
+
+
+def _has_real_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:
+ return any(_is_real_blocker_evidence(value) for value in _blocker_family_values(snapshot))
+
+
+def _is_real_blocker_evidence(value: Any) -> bool:
+ return _is_meaningful_contingent_antecedent_value(value) and not _is_structural_no_blocker_marker(value)
+
+
+def _has_structural_no_blocker_evidence(snapshot: RunEvidenceSnapshot) -> bool:
+ for value in _blocker_family_values(snapshot):
+ if _is_structural_no_blocker_marker(value):
+ return True
+ return False
+
+
+def _is_structural_no_blocker_marker(value: Any) -> bool:
+ if isinstance(value, bool):
+ return not value
+ if value is None:
+ return True
+ if not isinstance(value, str):
+ return False
+ normalized = " ".join(value.casefold().split())
+ return normalized in {
+ "",
+ "none",
+ "null",
+ "false",
+ "no",
+ "n/a",
+ "na",
+ "no blocker",
+ "no blockers",
+ "none found",
+ "not blocked",
+ }
+
+
def _coerce_result(
raw: Any,
criterion_ids: list[str],
@@ -1577,6 +1667,10 @@ def _is_definition_plane_abstention(verdict: CriterionVerdict) -> bool:
return verdict.state == "unknown" and verdict.reason_code.startswith(_DEFINITION_REASON_PREFIX)
+def _is_structural_requested_output_abstention(verdict: CriterionVerdict) -> bool:
+ return verdict.state == "unsatisfied" and verdict.reason_code == _STRUCTURAL_ABSTENTION_REASON_CODE
+
+
def _is_contingent_abstention(
verdict: CriterionVerdict,
contingent_criterion_ids: Iterable[str],
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 8837fa2d5..03a4b30fc 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -151,8 +151,10 @@ _CREDENTIAL_CODE_MARKERS = ("saved credential", "login_credentials", ".otp()", "
_MAX_COMPLETION_CRITERIA = 8
+_MAX_TRACE_COMPLETION_CRITERIA = 8
_COMPLETION_CRITERION_OUTCOME_MAX_CHARS = 200
_COMPLETION_CRITERION_CONTINGENT_ON_MAX_CHARS = 200
+_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS = 500
_CONTINGENT_ANTECEDENT_OUTPUT_PATH_RE = re.compile(r"^output\.[A-Za-z_][A-Za-z0-9_]*$")
_REQUESTED_OUTPUT_CRITERION_ID_PREFIX = "__copilot_requested_output__"
@@ -166,15 +168,25 @@ CriterionLevel = Literal["definition", "run"]
_CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})
CriterionKind = Literal["outcome", "terminal_action"]
TerminalActionFamily = Literal["request", "application", "form", "order"]
+ExpectedOutputShape = Literal[
+ "reference_code",
+ "numeric_identifier",
+ "date",
+ "address",
+ "status_label",
+ "money_amount",
+ "owner_label",
+]
_CRITERION_KINDS: frozenset[str] = frozenset({"outcome", "terminal_action"})
_TERMINAL_ACTION_FAMILIES: frozenset[str] = frozenset({"request", "application", "form", "order"})
+_EXPECTED_OUTPUT_SHAPES: frozenset[str] = frozenset(get_args(ExpectedOutputShape))
_OUTPUT_INTENT_RE = re.compile(
r"\b(?:read|capture|extract|output|return|returns|returned|include|includes|including|"
r"final\s+(?:extracted\s+)?fields?|result\s+records?|returned\s+records?)\b",
re.I,
)
-_OUTPUT_SPAN_END_RE = re.compile(r"[\n.!?]")
+_OUTPUT_SPAN_END_RE = re.compile(r"[\n!?]")
_OUTPUT_METHOD_TAIL_RE = re.compile(
r"\b(?:by|via|using|after|before|then|click(?:ing)?|open(?:ing)?|select(?:ing)?|"
r"choose|choosing|search(?:ing)?|navigate|go\s+to)\b.*",
@@ -188,7 +200,8 @@ _OUTPUT_FIELD_CONNECTOR_RE = re.compile(
_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")
_OUTPUT_FIELD_WORDS = frozenset(
"address addresses date dates email emails id identifier identifiers license licenses location locations "
- "name names number numbers phone phones specialties specialty status statuses taxonomy".split()
+ "amount amounts domain domains name names number numbers owner owners phone phones rate rates specialties specialty "
+ "status statuses taxonomy total totals url urls website websites".split()
)
_OUTPUT_GENERIC_WORDS = frozenset(
"a all an data detail details each entity final for information its of output outputs profile record records "
@@ -217,6 +230,8 @@ class CompletionCriterion:
# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".
level: CriterionLevel = "run"
output_path: str | None = None
+ expected_output_value: str | None = None
+ expected_output_shape: ExpectedOutputShape | None = None
kind: CriterionKind = "outcome"
terminal_action_family: TerminalActionFamily | None = None
@@ -256,7 +271,7 @@ class RequestPolicy:
return [criterion for criterion in self.completion_criteria if not criterion.method_mandated]
def to_trace_data(self) -> dict[str, Any]:
- return {
+ data: dict[str, Any] = {
"testing_intent": self.testing_intent,
"credential_input_kind": self.credential_input_kind,
"clarification_reason": self.clarification_reason,
@@ -284,6 +299,19 @@ class RequestPolicy:
len(origins) for origins in self.existing_workflow_credential_origins.values()
),
}
+ requested_output_criteria = [
+ criterion for criterion in self.graded_completion_criteria() if criterion.output_path is not None
+ ]
+ data["requested_output_criteria_count"] = len(requested_output_criteria)
+ for index, criterion in enumerate(requested_output_criteria[:_MAX_TRACE_COMPLETION_CRITERIA]):
+ prefix = f"requested_output_criterion_{index}"
+ data[f"{prefix}_id"] = criterion.id
+ data[f"{prefix}_output_path"] = criterion.output_path
+ data[f"{prefix}_grounding_mode"] = _criterion_grounding_mode(criterion)
+ data[f"{prefix}_has_exact_value"] = criterion.expected_output_value is not None
+ if criterion.expected_output_shape:
+ data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape
+ return data
def prompt_summary(self) -> str:
lines = [
@@ -317,6 +345,14 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy
return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
+def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:
+ if criterion.expected_output_value is not None:
+ return "exact_value"
+ if criterion.expected_output_shape is not None:
+ return "shape"
+ return "missing"
+
+
_TRANSCRIPT_TOTAL_CHAR_BUDGET = 2048
TRANSCRIPT_ANCHOR_CHAR_CAP = 512
_TRANSCRIPT_RETAINED_MIN_CHARS = 512
@@ -518,6 +554,12 @@ def _coerce_terminal_action_family(value: Any, kind: CriterionKind) -> TerminalA
return None
+def _coerce_expected_output_shape(value: Any) -> ExpectedOutputShape | None:
+ if isinstance(value, str) and value in _EXPECTED_OUTPUT_SHAPES:
+ return cast(ExpectedOutputShape, value)
+ return None
+
+
def _coerce_classifier_payload(raw: Any) -> dict[str, Any] | None:
if isinstance(raw, str):
raw = parse_final_response(raw)
@@ -603,6 +645,14 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
continue
output_path_raw = item.get("output_path")
output_path = output_path_raw.strip() if isinstance(output_path_raw, str) and output_path_raw.strip() else None
+ expected_output_value_raw = item.get("expected_output_value")
+ expected_output_value = (
+ " ".join(expected_output_value_raw.split())[:_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS].strip()
+ if isinstance(expected_output_value_raw, str)
+ else None
+ )
+ expected_output_value = expected_output_value or None
+ expected_output_shape = _coerce_expected_output_shape(item.get("expected_output_shape"))
contingent_on_raw = item.get("contingent_on")
contingent_on = (
" ".join(contingent_on_raw.split())[:_COMPLETION_CRITERION_CONTINGENT_ON_MAX_CHARS].strip()
@@ -638,6 +688,8 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
if isinstance(level_raw, str) and level_raw in _CRITERION_LEVELS
else "run",
output_path=output_path,
+ expected_output_value=expected_output_value,
+ expected_output_shape=expected_output_shape,
kind=kind,
terminal_action_family=_coerce_terminal_action_family(item.get("terminal_action_family"), kind),
)
@@ -668,14 +720,26 @@ def _output_intent_spans(user_message: str) -> list[str]:
spans: list[str] = []
for match in _OUTPUT_INTENT_RE.finditer(message):
tail = message[match.start() :]
- end_match = _OUTPUT_SPAN_END_RE.search(tail)
- span = tail[: end_match.start()] if end_match is not None else tail
+ span = tail[: _output_span_end_index(tail)]
span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")
if span:
spans.append(span)
return spans
+def _output_span_end_index(text: str) -> int:
+ for index, char in enumerate(text):
+ if _OUTPUT_SPAN_END_RE.fullmatch(char):
+ return index
+ if char != ".":
+ continue
+ next_char = text[index + 1] if index + 1 < len(text) else ""
+ if next_char and not next_char.isspace():
+ continue
+ return index + 1
+ return len(text)
+
+
def _normalize_requested_output_aliases(aliases: dict[str, str] | None) -> dict[str, str]:
normalized: dict[str, str] = {}
for field_name, output_path in (aliases or {}).items():
@@ -741,6 +805,10 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | No
normalized_aliases = _normalize_requested_output_aliases(aliases)
if " ".join(_word_tokens(candidate)) in normalized_aliases:
return candidate
+ candidate_tokens = _word_tokens(candidate)
+ for alias_key in sorted(normalized_aliases, key=lambda key: len(key.split()), reverse=True):
+ if _tokens_contain_sequence(candidate_tokens, alias_key.split()):
+ return _matched_alias_phrase(candidate, alias_key) or alias_key
candidate_words = candidate.split()
for word in candidate_words:
if " ".join(_word_tokens(word)) in normalized_aliases:
@@ -793,6 +861,15 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | No
return " ".join(phrase_words)
+def _matched_alias_phrase(candidate: str, alias_key: str) -> str | None:
+ alias_words = alias_key.split()
+ if not alias_words:
+ return None
+ pattern = r"\b" + r"[\W_]+".join(re.escape(word) for word in alias_words) + r"\b"
+ match = re.search(pattern, candidate, re.I)
+ return match.group(0).strip() if match is not None else None
+
+
def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:
fields: list[str] = []
seen: set[str] = set()
@@ -861,6 +938,42 @@ def _criterion_text_covers_any_requested_output(criterion: CompletionCriterion,
return any(_criterion_text_covers_requested_output(criterion, field_name) for field_name in requested_fields)
+def _requested_output_expected_values_from_criteria(
+ criteria: list[CompletionCriterion],
+ requested_specs: list[tuple[str, str, str]],
+) -> dict[str, str]:
+ values: dict[str, str] = {}
+ for criterion in criteria:
+ if criterion.level == "definition" or criterion.method_mandated:
+ continue
+ for field_name, output_path, _field_label in requested_specs:
+ if criterion.output_path != output_path and not _criterion_text_covers_requested_output(
+ criterion, field_name
+ ):
+ continue
+ value = criterion.expected_output_value
+ if value:
+ values.setdefault(output_path, value)
+ return values
+
+
+def _requested_output_shapes_from_criteria(
+ criteria: list[CompletionCriterion],
+ requested_specs: list[tuple[str, str, str]],
+) -> dict[str, ExpectedOutputShape]:
+ shapes: dict[str, ExpectedOutputShape] = {}
+ for criterion in criteria:
+ if criterion.level == "definition" or criterion.method_mandated or criterion.expected_output_shape is None:
+ continue
+ for field_name, output_path, _field_label in requested_specs:
+ if criterion.output_path != output_path and not _criterion_text_covers_requested_output(
+ criterion, field_name
+ ):
+ continue
+ shapes.setdefault(output_path, criterion.expected_output_shape)
+ return shapes
+
+
def _requested_output_criterion_id(output_path: str) -> str:
slug = "_".join(_word_tokens(output_path)) or "field"
return f"{_REQUESTED_OUTPUT_CRITERION_ID_PREFIX}{slug}"
@@ -911,6 +1024,27 @@ def _cap_completion_criteria(
return capped
+def _preserve_after_requested_output_canonicalization(
+ criterion: CompletionCriterion,
+ requested_fields: list[str],
+ requested_output_paths: set[str],
+) -> bool:
+ if criterion.level == "definition" or criterion.method_mandated:
+ return True
+ if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(
+ criterion, requested_fields
+ ):
+ return False
+ if (
+ requested_output_paths
+ and not criterion.output_path
+ and criterion.kind == "outcome"
+ and _generic_completion_criterion(criterion)
+ ):
+ return False
+ return True
+
+
def _apply_requested_output_completion_criteria(
policy: RequestPolicy, user_message: str, aliases: dict[str, str] | None = None
) -> None:
@@ -939,6 +1073,9 @@ def _apply_requested_output_completion_criteria(
if not requested_output_paths:
return
+ value_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)
+ shape_by_output_path = _requested_output_shapes_from_criteria(policy.completion_criteria, requested_specs)
+
metadata_by_output_path: dict[str, tuple[str | None, str | None, Literal["registered_download"] | None]] = {}
for criterion in policy.completion_criteria:
if criterion.level == "definition" or criterion.method_mandated:
@@ -966,12 +1103,7 @@ def _apply_requested_output_completion_criteria(
preserved_criteria = [
criterion
for criterion in policy.completion_criteria
- if criterion.level == "definition"
- or criterion.method_mandated
- or (
- criterion.output_path not in requested_output_paths
- and not _criterion_text_covers_any_requested_output(criterion, requested_fields)
- )
+ if _preserve_after_requested_output_canonicalization(criterion, requested_fields, requested_output_paths)
]
canonical_requested_criteria = [
CompletionCriterion(
@@ -979,6 +1111,8 @@ def _apply_requested_output_completion_criteria(
outcome=f"The returned record includes {field_label}.",
level="run",
output_path=output_path,
+ expected_output_value=value_by_output_path.get(output_path),
+ expected_output_shape=shape_by_output_path.get(output_path),
contingent_on=metadata_by_output_path.get(output_path, (None, None, None))[0],
contingent_antecedent_output_path=metadata_by_output_path.get(output_path, (None, None, None))[1],
deliverable_kind=metadata_by_output_path.get(output_path, (None, None, None))[2],
@@ -1012,6 +1146,10 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
item["deliverable_kind"] = criterion.deliverable_kind
if criterion.output_path:
item["output_path"] = criterion.output_path
+ if criterion.expected_output_value:
+ item["expected_output_value"] = criterion.expected_output_value
+ if criterion.expected_output_shape:
+ item["expected_output_shape"] = criterion.expected_output_shape
rendered.append(item)
return json.dumps(rendered)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 06eaf9fed..79188a459 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -15,6 +15,8 @@ from skyvern.forge.sdk.copilot.agent import (
_rewrite_failed_test_response,
_verified_workflow_or_none,
)
+from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criteria_to_json
+from skyvern.forge.sdk.copilot.completion_output_grounding import grade_requested_output_criteria
from skyvern.forge.sdk.copilot.completion_verification import (
REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID,
CompletionVerificationResult,
@@ -97,6 +99,8 @@ def _criterion(
kind: str = "outcome",
terminal_action_family: str | None = None,
deliverable_kind: str | None = None,
+ expected_output_value: str | None = None,
+ expected_output_shape: str | None = None,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -108,6 +112,8 @@ def _criterion(
kind=kind,
terminal_action_family=terminal_action_family,
deliverable_kind=deliverable_kind, # type: ignore[arg-type]
+ expected_output_value=expected_output_value,
+ expected_output_shape=expected_output_shape, # type: ignore[arg-type]
)
@@ -547,6 +553,20 @@ def test_structural_unfired_ids_derive_from_empty_output_path() -> None:
assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == ["c0"]
+def test_p7_manual_service_no_blocker_abstains() -> None:
+ criteria = [
+ _criterion(
+ "c0",
+ "Any manual service blocker is reported to the user.",
+ contingent_on="a manual service blocker exists",
+ contingent_antecedent_output_path="output.blocker",
+ )
+ ]
+ snapshot = RunEvidenceSnapshot(block_outputs={"terminal_result": {"manual_service_blocker": None}})
+
+ assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == ["c0"]
+
+
@pytest.mark.parametrize("reason_code,state", [("evidence_contradicts", "unsatisfied"), ("unknown", "unknown")])
def test_false_contingent_antecedent_output_abstains(reason_code: str, state: str) -> None:
criteria = [
@@ -651,6 +671,35 @@ def test_structural_fired_evidence_overrides_empty_output_path() -> None:
assert structural_unfired_contingent_criterion_ids(criteria, snapshot) == []
+def test_real_blocker_family_evidence_overrides_primary_no_blocker_marker() -> None:
+ criteria = [
+ _criterion(
+ "c0",
+ "A provider blocker is reported to the user.",
+ contingent_on="the provider site blocks online submission",
+ contingent_antecedent_output_path="output.blocker",
+ )
+ ]
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={"terminal_result": {"blocker": None, "manual_service_blocker": "Provider requires phone call"}}
+ )
+ structural_unfired_ids = structural_unfired_contingent_criterion_ids(criteria, snapshot)
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", "c1"],
+ contingent_criterion_ids=["c0"],
+ contingent_antecedent_output_path_by_criterion_id={"c0": "output.blocker"},
+ structural_unfired_criterion_ids=structural_unfired_ids,
+ verdicts=[
+ CriterionVerdict(criterion_id="c0", state="unsatisfied", reason_code="evidence_contradicts"),
+ CriterionVerdict(criterion_id="c1", state="satisfied", reason_code="evidence_confirms"),
+ ],
+ )
+
+ assert structural_unfired_ids == []
+ assert result.is_fully_satisfied() is False
+
+
def test_empty_verdicts_with_criteria_is_not_vacuously_satisfied() -> None:
result = CompletionVerificationResult(status="evaluated", criterion_ids=["c0"], verdicts=[])
assert result.is_fully_satisfied() is False
@@ -3336,7 +3385,12 @@ async def test_requested_output_path_ignores_evidence_text(monkeypatch: pytest.M
assert verification is not None
assert verification.is_fully_satisfied() is False
- assert verification.verdicts[0].reason_code == "missing_exact_field"
+ assert verification.verdicts[0].reason_code == "no_evidence"
+ assert (
+ verification.verdicts[0].missing_evidence
+ == "requested-output criterion lacks typed expected_output_value or expected_output_shape; "
+ "presence-only output cannot confirm value-grounded criterion"
+ )
@pytest.mark.asyncio
@@ -3360,11 +3414,13 @@ async def test_requested_output_path_ignores_block_level_prose(monkeypatch: pyte
assert verification is not None
assert verification.is_fully_satisfied() is False
- assert verification.verdicts[0].reason_code == "missing_exact_field"
+ assert verification.verdicts[0].reason_code == "no_evidence"
@pytest.mark.asyncio
-async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_requested_output_path_exact_runtime_field_with_expected_value_satisfies(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
async def fail_handler(**_: object) -> object:
raise AssertionError("exact requested-output evidence must bypass the judge")
@@ -3373,7 +3429,14 @@ async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch:
_set_workflow_labels(ctx, "extract_profile")
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ )
+ ]
)
verification = await _maybe_run_completion_verification(
@@ -3386,185 +3449,1100 @@ async def test_requested_output_path_exact_runtime_field_satisfies(monkeypatch:
assert verification.is_fully_satisfied() is True
assert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.npi"
+ trace = verification.to_trace_data()
+ assert trace["verdict_0_criterion_id"] == "c_npi"
+ assert trace["verdict_0_output_path"] == "output.npi"
+ assert trace["verdict_0_grounding_mode"] == "exact_value"
+ assert "verdict_0_expected_output_shape" not in trace
+ assert trace["verdict_0_has_exact_value"] is True
+ assert "1234567890" not in repr(trace)
-async def test_requested_output_path_does_not_match_block_label_only(monkeypatch: pytest.MonkeyPatch) -> None:
- async def fail_handler(**_: object) -> object:
- raise AssertionError("requested-output criteria must not reach the judge")
- _patch_completion_handler(monkeypatch, fail_handler)
+def test_requested_output_path_without_expected_value_structurally_abstains_for_present_values() -> None:
ctx = _run_ctx()
- _set_workflow_labels(ctx, "npi")
- ctx.code_artifact_metadata = {"npi": _metadata_for_requested_paths("npi")["extract_profile"]}
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "request_id",
+ "provider_captured_address",
+ "requested_date",
+ "status",
+ )
ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_request_id",
+ outcome="The returned record includes request id.",
+ output_path="output.request_id",
+ ),
+ CompletionCriterion(
+ id="c_provider_captured_address",
+ outcome="The returned record includes provider captured address.",
+ output_path="output.provider_captured_address",
+ ),
+ CompletionCriterion(
+ id="c_requested_date",
+ outcome="The returned record includes requested date.",
+ output_path="output.requested_date",
+ ),
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ ),
+ ]
)
- verification = await _maybe_run_completion_verification(
+ verdicts = grade_requested_output_criteria(
ctx,
- {
- "ok": True,
- "data": {
- "workflow_run_id": "wr_requested_output",
- "overall_status": "completed",
- "executed_block_labels": ["npi"],
- "current_url": "https://example.test/profile",
- "blocks": [
- {
- "label": "npi",
- "block_type": "CODE",
- "status": "completed",
- "extracted_data": {"value": "1234567890"},
- }
- ],
- },
- },
- time.monotonic(),
+ ctx.request_policy.completion_criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "request_id": "100245",
+ "provider_captured_address": "100245",
+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",
+ "status": "2026-06-24",
+ "evidence_text": "The request completed successfully.",
+ }
+ }
+ ),
)
- assert verification is not None
- assert verification.is_fully_satisfied() is False
- assert verification.verdicts[0].reason_code == "missing_exact_field"
+ assert {verdict.criterion_id for verdict in verdicts} == {
+ "c_request_id",
+ "c_provider_captured_address",
+ "c_requested_date",
+ "c_status",
+ }
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}
+ assert {verdict.grounding_mode for verdict in verdicts} == {"missing"}
+ assert all(verdict.has_exact_value is False for verdict in verdicts)
- ("goal_value_path", "requested_path", "runtime_output", "evidence_ref"),
- [
- ("$.npi", "output.npi", {"npi": "1234567890"}, "block_outputs:extract_profile.npi"),
- ("$[*].npi", "output.npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.npi"),
- ("$[0].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),
- ("$[].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),
- (
- "records[0].npi",
- "output.records[].npi",
- {"records": [{"npi": "1234567890"}]},
- "block_outputs:extract_profile.records[].npi",
+def test_requested_output_shape_only_generated_fields_structurally_abstain_without_exact_values() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "account_number",
+ "selected_start_date",
+ )
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
),
- (
- "output.records[0].npi",
- "output.records[].npi",
- {"records": [{"npi": "1234567890"}]},
- "block_outputs:extract_profile.records[].npi",
+ CompletionCriterion(
+ id="c_account_number",
+ outcome="The returned record includes account number.",
+ output_path="output.account_number",
+ expected_output_shape="numeric_identifier",
),
- ],
-)
-async def test_requested_output_path_normalizes_jsonpath_and_indexes(
- monkeypatch: pytest.MonkeyPatch,
- goal_value_path: str,
- requested_path: str,
- runtime_output: Any,
- evidence_ref: str,
-) -> None:
- async def fail_handler(**_: object) -> object:
- raise AssertionError("normalized requested-output evidence must bypass the judge")
+ CompletionCriterion(
+ id="c_selected_start_date",
+ outcome="The returned record includes selected start date.",
+ output_path="output.selected_start_date",
+ expected_output_shape="date",
+ ),
+ ]
- _patch_completion_handler(monkeypatch, fail_handler)
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_citrus_turn_on": {
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ }
+ }
+ ),
+ )
+
+ assert {verdict.criterion_id for verdict in verdicts} == {
+ "c_confirmation_number",
+ "c_account_number",
+ "c_selected_start_date",
+ }
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}
+ trace = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[criterion.id for criterion in criteria],
+ verdicts=verdicts,
+ ).to_trace_data()
+ assert trace["verdict_0_output_path"] == "output.confirmation_number"
+ assert trace["verdict_0_grounding_mode"] == "shape"
+ assert trace["verdict_0_expected_output_shape"] == "reference_code"
+ assert trace["verdict_0_has_exact_value"] is False
+ assert "WTR-1842-DEMO" not in repr(trace)
+ assert trace["fully_satisfied"] is False
+ assert trace["unmet_criterion_ids"] == []
+
+
+def test_requested_output_shape_abstains_on_p8_scrambled_values_and_ignores_evidence_text_status() -> None:
ctx = _run_ctx()
- _set_workflow_labels(ctx, "extract_profile")
- ctx.code_artifact_metadata = _metadata_for_requested_paths(goal_value_path)
- ctx.request_policy = RequestPolicy(
- completion_criteria=[
- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path=requested_path)
- ]
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "provider_captured_address",
+ "requested_date",
+ "status",
)
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ CompletionCriterion(
+ id="c_provider_captured_address",
+ outcome="The returned record includes provider captured address.",
+ output_path="output.provider_captured_address",
+ expected_output_shape="address",
+ ),
+ CompletionCriterion(
+ id="c_requested_date",
+ outcome="The returned record includes requested date.",
+ output_path="output.requested_date",
+ expected_output_shape="date",
+ ),
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ ),
+ ]
- verification = await _maybe_run_completion_verification(
+ verdicts = grade_requested_output_criteria(
ctx,
- _requested_output_result(runtime_output),
- time.monotonic(),
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "confirmation_number": "100245",
+ "provider_captured_address": "100245",
+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",
+ "status": "2026-06-24",
+ "evidence_text": "Submitted / Processing",
+ }
+ }
+ ),
)
- assert verification is not None
- assert verification.is_fully_satisfied() is True
- assert verification.verdicts[0].evidence_ref == evidence_ref
+ assert {verdict.criterion_id for verdict in verdicts} == {
+ "c_confirmation_number",
+ "c_provider_captured_address",
+ "c_requested_date",
+ "c_status",
+ }
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}
@pytest.mark.asyncio
-async def test_requested_output_path_aliases_authored_contract_to_requested_path(
+async def test_requested_output_verifier_rejects_reconstructed_p8_scrambled_values(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
- raise AssertionError("aliased requested-output evidence must bypass the judge")
+ raise AssertionError("typed requested-output verifier should not delegate reconstructed P8 proof")
_patch_completion_handler(monkeypatch, fail_handler)
ctx = _run_ctx()
_set_workflow_labels(ctx, "extract_profile")
- ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "account_number",
+ "provider_captured_address",
+ "requested_date",
+ "status",
+ )
ctx.request_policy = RequestPolicy(
completion_criteria=[
- CompletionCriterion(
- id="c_npi",
- outcome="The nested record NPI is returned.",
- output_path="output.records[].npi",
- )
+ _criterion(
+ "c_confirmation_number",
+ "The output includes the confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ _criterion(
+ "c_account_number",
+ "The output includes the account number.",
+ output_path="output.account_number",
+ expected_output_shape="numeric_identifier",
+ ),
+ _criterion(
+ "c_provider_captured_address",
+ "The output includes the provider captured address.",
+ output_path="output.provider_captured_address",
+ expected_output_shape="address",
+ ),
+ _criterion(
+ "c_requested_date",
+ "The output includes the requested date.",
+ output_path="output.requested_date",
+ expected_output_shape="date",
+ ),
+ _criterion(
+ "c_status",
+ "The output includes the request status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ ),
]
)
verification = await _maybe_run_completion_verification(
ctx,
- _requested_output_result({"records": [{"npi": "1234567890"}]}),
+ _requested_output_result(
+ {
+ "confirmation_number": "QC-2002-DEMO",
+ "account_number": "100245",
+ "provider_captured_address": "100245",
+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",
+ "status": "2026-06-24",
+ "evidence_text": "Submitted / Processing",
+ }
+ ),
time.monotonic(),
)
assert verification is not None
- assert verification.is_fully_satisfied() is True
- assert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.records[].npi"
+ assert verification.status == "evaluated"
+ assert verification.is_fully_satisfied() is False
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_confirmation_number"].reason_code == "structurally_abstained"
+ assert verdicts["c_account_number"].reason_code == "structurally_abstained"
+ for criterion_id in ("c_provider_captured_address", "c_requested_date", "c_status"):
+ assert verdicts[criterion_id].state == "unsatisfied"
+ assert verdicts[criterion_id].reason_code == "structurally_abstained"
@pytest.mark.asyncio
-async def test_requested_output_path_requires_exact_nested_runtime_field(
+async def test_requested_output_verifier_accepts_value_correct_reconstructed_p8(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
- raise AssertionError("requested-output criteria must not reach the judge")
+ raise AssertionError("typed requested-output verifier should not delegate value-correct P8 proof")
_patch_completion_handler(monkeypatch, fail_handler)
ctx = _run_ctx()
_set_workflow_labels(ctx, "extract_profile")
- ctx.code_artifact_metadata = _metadata_for_requested_paths("locations[].address")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "account_number",
+ "provider_captured_address",
+ "requested_date",
+ "status",
+ )
ctx.request_policy = RequestPolicy(
completion_criteria=[
- CompletionCriterion(
- id="c_address",
- outcome="Each listed location includes address.",
- output_path="output.locations[].address",
- )
+ _criterion(
+ "c_confirmation_number",
+ "The output includes the confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ _criterion(
+ "c_account_number",
+ "The output includes the account number.",
+ output_path="output.account_number",
+ expected_output_shape="numeric_identifier",
+ ),
+ _criterion(
+ "c_provider_captured_address",
+ "The output includes the provider captured address.",
+ output_path="output.provider_captured_address",
+ expected_output_shape="address",
+ ),
+ _criterion(
+ "c_requested_date",
+ "The output includes the requested date.",
+ output_path="output.requested_date",
+ expected_output_shape="date",
+ ),
+ _criterion(
+ "c_status",
+ "The output includes the request status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ ),
]
)
- missing = await _maybe_run_completion_verification(
+ verification = await _maybe_run_completion_verification(
ctx,
- _requested_output_result({"locations": [{"name": "North Clinic"}]}),
+ _requested_output_result(
+ {
+ "confirmation_number": "QC-2002-DEMO",
+ "account_number": "100245",
+ "provider_captured_address": "77 Gaslight Way, Decatur, GA 30030",
+ "requested_date": "2026-06-24",
+ "status": "Submitted / Processing",
+ }
+ ),
time.monotonic(),
)
- assert missing is not None
- assert missing.is_fully_satisfied() is False
- assert missing.verdicts[0].reason_code == "missing_exact_field"
- satisfied = await _maybe_run_completion_verification(
- ctx,
- _requested_output_result({"locations": [{"address": "100 Main St"}]}),
- time.monotonic(),
- )
- assert satisfied is not None
- assert satisfied.is_fully_satisfied() is True
- assert satisfied.verdicts[0].evidence_ref == "block_outputs:extract_profile.locations[].address"
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert verification.is_fully_satisfied() is False
+ assert {verdict.reason_code for verdict in verification.verdicts} == {"structurally_abstained"}
@pytest.mark.asyncio
-async def test_requested_output_path_uses_only_accepted_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
- async def fail_handler(**_: object) -> object:
- raise AssertionError("unproducible requested-output criteria must not reach the judge")
+async def test_requested_output_verifier_accepts_p7_with_unfired_blocker_and_fee_suffix(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}
- _patch_completion_handler(monkeypatch, fail_handler)
+ _patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
_set_workflow_labels(ctx, "extract_profile")
- ctx.raw_code_artifact_metadata = _metadata_for_requested_paths("npi")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "account_number",
+ "selected_start_date",
+ "deposit_amount",
+ "next_owner",
+ "blocker",
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_submit",
+ "The service request is submitted.",
+ kind="terminal_action",
+ terminal_action_family="request",
+ ),
+ _criterion(
+ "c_confirmation_number",
+ "The output includes the confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ _criterion(
+ "c_account_number",
+ "The output includes the account number.",
+ output_path="output.account_number",
+ expected_output_shape="numeric_identifier",
+ ),
+ _criterion(
+ "c_selected_start_date",
+ "The output includes the selected start date.",
+ output_path="output.selected_start_date",
+ expected_output_shape="date",
+ ),
+ _criterion(
+ "c_deposit_amount",
+ "The output includes the deposit amount.",
+ output_path="output.deposit_amount",
+ expected_output_shape="money_amount",
+ ),
+ _criterion(
+ "c_next_owner",
+ "The output includes the next owner.",
+ output_path="output.next_owner",
+ expected_output_shape="owner_label",
+ ),
+ _criterion(
+ "c_blocker",
+ "Any manual service blocker is reported to the user.",
+ output_path="output.blocker",
+ contingent_on="a manual service blocker exists",
+ contingent_antecedent_output_path="output.blocker",
+ ),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(
+ {
+ "manual_service_blocker": None,
+ "output": {
+ "confirmation_number": "WTR-1842-DEMO",
+ "account_number": "100245",
+ "selected_start_date": "2026-06-22",
+ "deposit_amount": "$41.00 plus initiation fee",
+ "next_owner": "Provider",
+ },
+ "evidence_text": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.status == "evaluated"
+ assert verification.is_fully_satisfied() is True
+ assert verification.structural_unfired_criterion_ids == ["c_blocker"]
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_submit"].reason_code == "evidence_confirms"
+ assert verdicts["c_blocker"].reason_code == "no_evidence"
+ assert verdicts["c_deposit_amount"].reason_code == "structurally_abstained"
+ assert all(
+ verdict.reason_code == "structurally_abstained"
+ for cid, verdict in verdicts.items()
+ if cid not in {"c_submit", "c_blocker"}
+ )
+
+
+def test_requested_output_canonical_path_precedence_rejects_wrong_top_level_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_citrus_turn_on": {
+ "confirmation_number": "QC-2002-DEMO",
+ "output": {"confirmation_number": "WTR-1842-DEMO"},
+ }
+ }
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.confirmation_number"
+
+
[email protected]("canonical_value", [None, ""])
+def test_requested_output_exact_value_canonical_placeholder_not_rescued_by_wrapper(
+ canonical_value: str | None,
+) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_citrus_turn_on": {
+ "confirmation_number": canonical_value,
+ "output": {"confirmation_number": "WTR-1842-DEMO"},
+ }
+ }
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.confirmation_number"
+
+
+def test_requested_output_one_wrapper_runtime_field_abstains_without_broad_search() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ )
+ ]
+
+ wrapped_verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"utility_citrus_turn_on": {"output": {"confirmation_number": "WTR-1842-DEMO"}}}
+ ),
+ )
+ unrelated_nested_verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"utility_citrus_turn_on": {"details": {"output": {"confirmation_number": "WTR-1842-DEMO"}}}}
+ ),
+ )
+
+ assert wrapped_verdicts[0].state == "unsatisfied"
+ assert wrapped_verdicts[0].reason_code == "structurally_abstained"
+ assert wrapped_verdicts[0].evidence_ref == "block_outputs:utility_citrus_turn_on.output.confirmation_number"
+ assert unrelated_nested_verdicts[0].state == "unsatisfied"
+ assert unrelated_nested_verdicts[0].reason_code == "missing_exact_field"
+
+
+def test_requested_output_wrapped_wrong_neighbors_structurally_abstain() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("requested_date", "status")
+ criteria = [
+ CompletionCriterion(
+ id="c_requested_date",
+ outcome="The returned record includes requested date.",
+ output_path="output.requested_date",
+ expected_output_shape="date",
+ ),
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ ),
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "output": {
+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",
+ "status": "2026-06-24",
+ }
+ }
+ }
+ ),
+ )
+
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}
+
+
[email protected]("canonical_status", [None, ""])
+def test_requested_output_shape_canonical_placeholder_not_rescued_by_wrapper(
+ canonical_status: str | None,
+) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
+ criteria = [
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "status": canonical_status,
+ "output": {"status": "Submitted / Processing"},
+ }
+ }
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_ref is None
+
+
+def test_requested_output_exact_values_reject_p8_scrambled_neighbors() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(
+ "confirmation_number",
+ "provider_captured_address",
+ "requested_date",
+ "status",
+ )
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="QC-2002-DEMO",
+ ),
+ CompletionCriterion(
+ id="c_provider_captured_address",
+ outcome="The returned record includes provider captured address.",
+ output_path="output.provider_captured_address",
+ expected_output_value="77 Gaslight Way, Decatur, GA 30030",
+ ),
+ CompletionCriterion(
+ id="c_requested_date",
+ outcome="The returned record includes requested date.",
+ output_path="output.requested_date",
+ expected_output_value="2026-06-24",
+ ),
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_value="Submitted / Processing",
+ ),
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "confirmation_number": "QC-2002-DEMO",
+ "provider_captured_address": "100245",
+ "requested_date": "77 Gaslight Way, Decatur, GA 30030",
+ "status": "2026-06-24",
+ "evidence_text": "Submitted / Processing",
+ }
+ }
+ ),
+ )
+
+ verdicts_by_id = {verdict.criterion_id: verdict for verdict in verdicts}
+ assert verdicts_by_id["c_confirmation_number"].reason_code == "evidence_confirms"
+ for criterion_id in ("c_provider_captured_address", "c_requested_date", "c_status"):
+ assert verdicts_by_id[criterion_id].state == "unsatisfied"
+ assert verdicts_by_id[criterion_id].reason_code == "evidence_contradicts"
+
+
[email protected]("status", [None, "None", "null", "n/a", "na", "", "-", "--"])
+def test_requested_output_status_shape_treats_null_placeholders_as_missing(status: str | None) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
+ criteria = [
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_peach_gas_quickconnect": {
+ "status": status,
+ "evidence_text": "Submitted / Processing",
+ }
+ }
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code in {"missing_exact_field", "structurally_abstained"}
+
+
[email protected]("status", ["Submitted / Processing", "Approved", "Pending Review", "Not Credentialed"])
+def test_requested_output_status_shape_structurally_abstains_for_real_labels(status: str) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
+ criteria = [
+ CompletionCriterion(
+ id="c_status",
+ outcome="The returned record includes status.",
+ output_path="output.status",
+ expected_output_shape="status_label",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"utility_peach_gas_quickconnect": {"status": status}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+
+
+ "deposit_amount",
+ [
+ "2026-06-22",
+ "100245",
+ "77 Gaslight Way, Decatur, GA 30030",
+ "Submitted / Processing",
+ "https://utility.example.test/pay",
+ "initiation fee applies",
+ ],
+)
+def test_requested_output_money_amount_shape_structurally_abstains_for_present_neighbors(deposit_amount: str) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("deposit_amount")
+ criteria = [
+ _criterion(
+ "c_deposit_amount",
+ "The output includes the deposit amount.",
+ output_path="output.deposit_amount",
+ expected_output_shape="money_amount",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"deposit_amount": deposit_amount}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+
+
+def test_requested_output_exact_wrong_value_beats_matching_shape() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ expected_output_shape="reference_code",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"confirmation_number": "QC-2002-DEMO"}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+
+
+def test_requested_output_shape_missing_field_is_missing_exact_field() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ CompletionCriterion(
+ id="c_confirmation_number",
+ outcome="The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"utility_citrus_turn_on": {"evidence_text": "WTR-1842-DEMO"}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+
+
+async def test_requested_output_path_requires_expected_value_match(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("value-grounded requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("service_address", "requested_start_date")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ ),
+ CompletionCriterion(
+ id="c_date",
+ outcome="The returned record includes requested start date.",
+ output_path="output.requested_start_date",
+ expected_output_value="2026-06-22",
+ ),
+ ]
+ )
+
+ swapped = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(
+ {
+ "service_address": "2026-06-22",
+ "requested_start_date": "1234 Sample Utility Way",
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert swapped is not None
+ assert swapped.is_fully_satisfied() is False
+ assert {verdict.reason_code for verdict in swapped.verdicts} == {"evidence_contradicts"}
+
+ matched = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(
+ {
+ "service_address": "1234 Sample Utility Way",
+ "requested_start_date": "2026-06-22",
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert matched is not None
+ assert matched.is_fully_satisfied() is True
+ assert {verdict.reason_code for verdict in matched.verdicts} == {"evidence_confirms"}
+
+
+async def test_rehydrated_requested_output_expected_value_blocks_scrambled_output(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("value-grounded requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("service_address", "requested_start_date")
+ criteria = (
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ ),
+ CompletionCriterion(
+ id="c_date",
+ outcome="The returned record includes requested start date.",
+ output_path="output.requested_start_date",
+ expected_output_value="2026-06-22",
+ ),
+ )
+ ctx.request_policy = RequestPolicy(completion_criteria=list(criteria_from_json(criteria_to_json(criteria))))
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(
+ {
+ "service_address": "2026-06-22",
+ "requested_start_date": "1234 Sample Utility Way",
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert {verdict.reason_code for verdict in verification.verdicts} == {"evidence_contradicts"}
+
+
+async def test_requested_output_path_does_not_match_block_label_only(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("requested-output criteria must not reach the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "npi")
+ ctx.code_artifact_metadata = {"npi": _metadata_for_requested_paths("npi")["extract_profile"]}
ctx.request_policy = RequestPolicy(
completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]
)
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_requested_output",
+ "overall_status": "completed",
+ "executed_block_labels": ["npi"],
+ "current_url": "https://example.test/profile",
+ "blocks": [
+ {
+ "label": "npi",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": {"value": "1234567890"},
+ }
+ ],
+ },
+ },
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert verification.verdicts[0].reason_code == "no_evidence"
+
+
+ ("goal_value_path", "requested_path", "runtime_output", "evidence_ref"),
+ [
+ ("$.npi", "output.npi", {"npi": "1234567890"}, "block_outputs:extract_profile.npi"),
+ ("$[*].npi", "output.npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.npi"),
+ ("$[0].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),
+ ("$[].npi", "output.[].npi", [{"npi": "1234567890"}], "block_outputs:extract_profile.[].npi"),
+ (
+ "records[0].npi",
+ "output.records[].npi",
+ {"records": [{"npi": "1234567890"}]},
+ "block_outputs:extract_profile.records[].npi",
+ ),
+ (
+ "output.records[0].npi",
+ "output.records[].npi",
+ {"records": [{"npi": "1234567890"}]},
+ "block_outputs:extract_profile.records[].npi",
+ ),
+ ],
+)
+async def test_requested_output_path_normalizes_jsonpath_and_indexes(
+ monkeypatch: pytest.MonkeyPatch,
+ goal_value_path: str,
+ requested_path: str,
+ runtime_output: Any,
+ evidence_ref: str,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("normalized requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths(goal_value_path)
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path=requested_path,
+ expected_output_value="1234567890",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(runtime_output),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].evidence_ref == evidence_ref
+
+
+async def test_requested_output_path_aliases_authored_contract_to_requested_path(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("aliased requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The nested record NPI is returned.",
+ output_path="output.records[].npi",
+ expected_output_value="1234567890",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"records": [{"npi": "1234567890"}]}),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].evidence_ref == "block_outputs:extract_profile.records[].npi"
+
+
+async def test_requested_output_path_requires_exact_nested_runtime_field(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("requested-output criteria must not reach the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("locations[].address")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_address",
+ outcome="Each listed location includes address.",
+ output_path="output.locations[].address",
+ expected_output_value="100 Main St",
+ )
+ ]
+ )
+
+ missing = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"locations": [{"name": "North Clinic"}]}),
+ time.monotonic(),
+ )
+ assert missing is not None
+ assert missing.is_fully_satisfied() is False
+ assert missing.verdicts[0].reason_code == "missing_exact_field"
+ assert missing.verdicts[0].missing_evidence == (
+ "run output did not include exact structured field output.locations[].address"
+ )
+
+ satisfied = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"locations": [{"address": "100 Main St"}]}),
+ time.monotonic(),
+ )
+ assert satisfied is not None
+ assert satisfied.is_fully_satisfied() is True
+ assert satisfied.verdicts[0].evidence_ref == "block_outputs:extract_profile.locations[].address"
+
+
+async def test_requested_output_path_uses_only_accepted_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("unproducible requested-output criteria must not reach the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.raw_code_artifact_metadata = _metadata_for_requested_paths("npi")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ )
+ ]
+ )
+
rejected = await _maybe_run_completion_verification(
ctx,
_requested_output_result({"npi": "1234567890"}),
@@ -3607,7 +4585,14 @@ async def test_requested_output_path_can_use_static_return_key_contract(
"""
)
ctx.request_policy = RequestPolicy(
- completion_criteria=[CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi")]
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ )
+ ]
)
verification = await _maybe_run_completion_verification(
@@ -3643,7 +4628,12 @@ async def test_requested_output_path_can_use_static_list_return_key_contract(
)
ctx.request_policy = RequestPolicy(
completion_criteria=[
- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.[].npi")
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.[].npi",
+ expected_output_value="1234567890",
+ )
]
)
@@ -3673,7 +4663,12 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyte
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
ctx.request_policy = RequestPolicy(
completion_criteria=[
- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi"),
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ ),
CompletionCriterion(id="c_cart", outcome="The cart contains the selected item."),
]
)
@@ -3691,6 +4686,78 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyte
assert {verdict.criterion_id for verdict in verification.verdicts} == {"c_npi", "c_cart"}
+async def test_present_generic_requested_output_without_expected_value_abstains_without_vetoing_run_success(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def handler(**_: object) -> dict:
+ return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}
+
+ _patch_completion_handler(monkeypatch, handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_customer_name",
+ outcome="The returned record includes customer name.",
+ output_path="output.customer_name",
+ ),
+ _criterion(
+ "c_submit",
+ "The record extraction completes.",
+ kind="terminal_action",
+ terminal_action_family="request",
+ ),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"customer_name": "Sample Customer"}),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
+ assert verdicts["c_customer_name"].reason_code == "structurally_abstained"
+ assert verdicts["c_submit"].reason_code == "evidence_confirms"
+
+
+async def test_all_abstained_requested_outputs_do_not_satisfy_completion(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("requested-output only abstention should not reach the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("customer_name")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_customer_name",
+ outcome="The returned record includes customer name.",
+ output_path="output.customer_name",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result({"customer_name": "Sample Customer"}),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert verification.verdicts[0].reason_code == "structurally_abstained"
+
+
@pytest.mark.asyncio
async def test_requested_output_bypasses_judge_satisfaction_without_exact_field(
monkeypatch: pytest.MonkeyPatch,
@@ -3714,7 +4781,7 @@ async def test_requested_output_bypasses_judge_satisfaction_without_exact_field(
assert verification is not None
assert verification.is_fully_satisfied() is False
- assert verification.verdicts[0].reason_code == "missing_exact_field"
+ assert verification.verdicts[0].reason_code == "no_evidence"
@pytest.mark.asyncio
@@ -3745,13 +4812,13 @@ async def test_requested_output_verdict_survives_unavailable_judge_result(
assert verification.status == "evaluated"
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_npi"].reason_code == "missing_exact_field"
+ assert verdicts["c_npi"].reason_code == "no_evidence"
assert verdicts["c_npi"].satisfied is False
assert verdicts["c_cart"].state == "unknown"
@pytest.mark.asyncio
[email protected]("reason_code", ["missing_exact_field", "unproducible"])
[email protected]("reason_code", ["no_evidence", "unproducible"])
async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_run_criterion(
monkeypatch: pytest.MonkeyPatch,
reason_code: str,
@@ -3762,7 +4829,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_
_patch_completion_handler(monkeypatch, fail_handler)
ctx = _run_ctx()
_set_workflow_labels(ctx, "extract_profile")
- if reason_code == "missing_exact_field":
+ if reason_code == "no_evidence":
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
output = {"status": "DONE", "blocker": False}
else:
@@ -3791,7 +4858,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_
@pytest.mark.asyncio
[email protected]("reason_code", ["missing_exact_field", "unproducible"])
[email protected]("reason_code", ["no_evidence", "unproducible"])
async def test_fired_contingent_requested_output_miss_still_vetoes(
monkeypatch: pytest.MonkeyPatch,
reason_code: str,
@@ -3802,7 +4869,7 @@ async def test_fired_contingent_requested_output_miss_still_vetoes(
_patch_completion_handler(monkeypatch, fail_handler)
ctx = _run_ctx()
_set_workflow_labels(ctx, "extract_profile")
- if reason_code == "missing_exact_field":
+ if reason_code == "no_evidence":
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
output = {"status": "DONE", "blocker": True}
else:
@@ -3876,7 +4943,12 @@ async def test_requested_output_satisfaction_is_not_vetoed_by_fallback_record_ab
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
ctx.request_policy = RequestPolicy(
completion_criteria=[
- CompletionCriterion(id="c_npi", outcome="The NPI is returned.", output_path="output.npi"),
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ ),
*_structured_record_criteria(),
]
)
@@ -4518,7 +5590,7 @@ async def test_download_registered_output_parameter_injects_and_verifies_without
@pytest.mark.asyncio
-async def test_marked_requested_output_id_still_requires_exact_field(
+async def test_marked_requested_output_id_without_expected_value_is_no_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
@@ -4551,13 +5623,13 @@ async def test_marked_requested_output_id_still_requires_exact_field(
assert verification is not None
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert verdicts["c_output_id"].reason_code == "no_evidence"
assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
assert ctx.request_policy.completion_criteria == [requested]
@pytest.mark.asyncio
-async def test_marked_requested_output_npi_still_requires_exact_field(
+async def test_marked_requested_output_npi_without_expected_value_is_no_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
@@ -4590,13 +5662,13 @@ async def test_marked_requested_output_npi_still_requires_exact_field(
assert verification is not None
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_npi"].reason_code == "missing_exact_field"
+ assert verdicts["c_npi"].reason_code == "no_evidence"
assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
assert ctx.request_policy.completion_criteria == [requested]
@pytest.mark.asyncio
-async def test_unmarked_requested_output_id_still_requires_exact_field(
+async def test_unmarked_requested_output_id_without_expected_value_is_no_evidence(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
@@ -4631,7 +5703,7 @@ async def test_unmarked_requested_output_id_still_requires_exact_field(
assert verification is not None
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert verdicts["c_output_id"].reason_code == "no_evidence"
@pytest.mark.asyncio
@@ -4672,8 +5744,8 @@ async def test_marked_download_deliverable_does_not_remove_mixed_extraction_fiel
assert verification is not None
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_output_id"].reason_code == "missing_exact_field"
- assert verdicts["c_npi"].reason_code == "missing_exact_field"
+ assert verdicts["c_output_id"].reason_code == "no_evidence"
+ assert verdicts["c_npi"].reason_code == "no_evidence"
assert REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID in verdicts
@@ -4758,7 +5830,7 @@ async def test_marked_download_deliverable_without_registered_evidence_fails_clo
assert verification is not None
assert verification.is_fully_satisfied() is False
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
- assert verdicts["c_output_id"].reason_code == "missing_exact_field"
+ assert verdicts["c_output_id"].reason_code == "no_evidence"
assert verdicts[REGISTERED_DOWNLOAD_COMPLETION_CRITERION_ID].reason_code == "no_evidence"
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index 4a86d060d..2ac2c71f3 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -63,6 +63,8 @@ def _criterion(
contingent_on: str | None = None,
contingent_antecedent_output_path: str | None = None,
deliverable_kind: str | None = None,
+ expected_output_value: str | None = None,
+ expected_output_shape: str | None = None,
) -> CompletionCriterion:
return CompletionCriterion(
id=cid,
@@ -73,6 +75,8 @@ def _criterion(
contingent_on=contingent_on,
contingent_antecedent_output_path=contingent_antecedent_output_path,
deliverable_kind=deliverable_kind, # type: ignore[arg-type]
+ expected_output_value=expected_output_value,
+ expected_output_shape=expected_output_shape, # type: ignore[arg-type]
)
@@ -260,6 +264,247 @@ async def test_classifier_drift_canonicalizes_to_stable_generic_requested_output
)
+async def test_requested_output_criteria_preserve_stable_paths_without_derived_carriers() -> None:
+ prompt = "Return a final record with service address 1234 Sample Utility Way and requested start date 2026-06-22."
+ classifier_shapes = [
+ [
+ {
+ "outcome": "The returned record includes requested start date 2026-06-22.",
+ "output_path": "output.requested_start_date",
+ },
+ {
+ "outcome": "The returned record includes service address 1234 Sample Utility Way.",
+ "output_path": "output.service_address",
+ },
+ ],
+ [
+ {"outcome": "The returned record includes service address.", "output_path": "output.service_address"},
+ {
+ "outcome": "The returned record includes requested start date.",
+ "output_path": "output.requested_start_date",
+ },
+ {"outcome": "The workflow runs to an expected result.", "method_mandated": True},
+ ],
+ ]
+
+ policies = [await _policy_for_message(prompt, criteria) for criteria in classifier_shapes]
+ requested_output_paths = {"output.service_address", "output.requested_start_date"}
+ canonical_subsets = [_requested_output_subset(policy, requested_output_paths) for policy in policies]
+ canonical_json = [criteria_to_json(subset) for subset in canonical_subsets]
+
+ assert canonical_json == [canonical_json[0], canonical_json[0]]
+ assert [criterion.id for criterion in canonical_subsets[0]] == [
+ "__copilot_requested_output__output_service_address",
+ "__copilot_requested_output__output_requested_start_date",
+ ]
+ assert [criterion.output_path for criterion in canonical_subsets[0]] == [
+ "output.service_address",
+ "output.requested_start_date",
+ ]
+ assert [criterion.outcome for criterion in canonical_subsets[0]] == [
+ "The returned record includes service address.",
+ "The returned record includes requested start date.",
+ ]
+ assert [criterion.expected_output_value for criterion in canonical_subsets[0]] == [
+ None,
+ None,
+ ]
+ assert [criterion.expected_output_shape for criterion in canonical_subsets[0]] == [None, None]
+
+
+async def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with provider captured address, account number, requested date, status, "
+ "confirmation number, deposit amount, and next owner.",
+ [],
+ )
+
+ requested_output_paths = {
+ "output.provider_captured_address",
+ "output.account_number",
+ "output.requested_date",
+ "output.status",
+ "output.confirmation_number",
+ "output.deposit_amount",
+ "output.next_owner",
+ }
+
+ assert {
+ criterion.output_path: criterion.expected_output_shape
+ for criterion in _requested_output_subset(policy, requested_output_paths)
+ } == {
+ "output.provider_captured_address": None,
+ "output.account_number": None,
+ "output.requested_date": None,
+ "output.status": None,
+ "output.confirmation_number": None,
+ "output.deposit_amount": None,
+ "output.next_owner": None,
+ }
+
+
+ ("prompt", "output_path", "_expected_value"),
+ [
+ (
+ "Return a final record with service URL https://example.com/a:b.",
+ "output.service_url",
+ "https://example.com/a:b",
+ ),
+ ("Return a final record with tax rate 1.25.", "output.tax_rate", "1.25"),
+ ("Return a final record with company domain example.com.", "output.company_domain", "example.com"),
+ (
+ "Return a final record with department name Research and Development.",
+ "output.department_name",
+ "Research and Development",
+ ),
+ ],
+)
+async def test_requested_output_expected_values_are_not_synthesized_from_user_message_regex(
+ prompt: str, output_path: str, _expected_value: str
+) -> None:
+ policy = await _policy_for_message(prompt, [])
+
+ criteria = _criteria_for_path(policy, output_path)
+ assert len(criteria) == 1
+ assert (
+ criteria[0].outcome == f"The returned record includes {output_path.removeprefix('output.').replace('_', ' ')}."
+ )
+ assert criteria[0].expected_output_value is None
+
+
+async def test_message_text_cannot_override_classifier_typed_expected_output_value() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with service address 999 Wrong Way.",
+ [
+ {
+ "outcome": "The returned record includes service address.",
+ "output_path": "output.service_address",
+ "expected_output_value": "1234 Sample Utility Way",
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.service_address")
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_value == "1234 Sample Utility Way"
+
+
+async def test_classifier_typed_expected_output_value_is_preserved_without_outcome_carrier() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with service address.",
+ [
+ {
+ "outcome": "The returned record includes service address.",
+ "output_path": "output.service_address",
+ "expected_output_value": "1234 Sample Utility Way",
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.service_address")
+ assert len(criteria) == 1
+ assert criteria[0].outcome == "The returned record includes service address."
+ assert criteria[0].expected_output_value == "1234 Sample Utility Way"
+
+
+async def test_classifier_typed_expected_output_shape_is_preserved_without_outcome_carrier() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with confirmation number.",
+ [
+ {
+ "outcome": "The returned record includes confirmation number.",
+ "output_path": "output.confirmation_number",
+ "expected_output_shape": "reference_code",
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.confirmation_number")
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_shape == "reference_code"
+
+
+async def test_classifier_invalid_expected_output_shape_is_ignored() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with custom value.",
+ [
+ {
+ "outcome": "The returned record includes custom value.",
+ "output_path": "output.custom_value",
+ "expected_output_shape": "anything_non_empty",
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.custom_value")
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_shape is None
+
+
+def test_parse_completion_criteria_accepts_only_declared_expected_output_shapes() -> None:
+ valid, invalid = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record includes confirmation number.",
+ "output_path": "output.confirmation_number",
+ "expected_output_shape": "reference_code",
+ },
+ {
+ "outcome": "The returned record includes custom value.",
+ "output_path": "output.custom_value",
+ "expected_output_shape": "anything_non_empty",
+ },
+ ]
+ )
+
+ assert valid.expected_output_shape == "reference_code"
+ assert invalid.expected_output_shape is None
+
+
+async def test_requested_output_canonicalization_preserves_classifier_shape_first() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with confirmation number.",
+ [
+ {
+ "outcome": "The returned record includes confirmation number.",
+ "output_path": "output.confirmation_number",
+ "expected_output_shape": "numeric_identifier",
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.confirmation_number")
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_shape == "numeric_identifier"
+
+
+async def test_classifier_non_string_expected_output_value_is_ignored() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with service address.",
+ [
+ {
+ "outcome": "The returned record includes service address.",
+ "output_path": "output.service_address",
+ "expected_output_value": {"value": "1234 Sample Utility Way"},
+ }
+ ],
+ )
+
+ criteria = _criteria_for_path(policy, "output.service_address")
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_value is None
+
+
@pytest.mark.asyncio
async def test_method_and_setup_text_do_not_become_completion_criteria() -> None:
policy = await _policy_for_message(
@@ -288,7 +533,7 @@ async def test_reusable_input_id_does_not_cover_requested_output_id() -> None:
@pytest.mark.asyncio
-async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_gate_remains() -> None:
+async def test_unbound_requested_output_narrative_is_replaced_but_specific_run_gate_remains() -> None:
policy = await _policy_for_message(
"Return a final record with record id, status, and phone.",
[
@@ -308,6 +553,34 @@ async def test_unbound_requested_output_narrative_is_replaced_but_unrelated_run_
} == {"output.record_id", "output.status", "output.phone"}
+async def test_generic_ungrounded_run_gate_drops_after_requested_output_canonicalization() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with record id.",
+ [{"outcome": "The workflow runs to its intended end state with the expected output."}],
+ )
+
+ assert "The workflow runs to its intended end state with the expected output." not in _outcomes(policy)
+ assert _criteria_for_path(policy, "output.record_id")
+
+
+async def test_terminal_action_run_gate_survives_requested_output_canonicalization() -> None:
+ policy = await _policy_for_message(
+ "Submit the request and return a final record with confirmation number.",
+ [
+ {
+ "outcome": "The request submission action is completed.",
+ "kind": "terminal_action",
+ "terminal_action_family": "request",
+ },
+ ],
+ )
+
+ assert "The request submission action is completed." in _outcomes(policy)
+ assert _criteria_for_path(policy, "output.confirmation_number")
+
+
@pytest.mark.asyncio
async def test_requested_output_coverage_uses_whole_token_sequence_matching() -> None:
policy = await _policy_for_message(
@@ -592,6 +865,58 @@ def test_active_criteria_rendering_includes_deliverable_kind() -> None:
]
+def test_active_criteria_rendering_includes_expected_output_value_as_structured_data() -> None:
+ rendered = _render_active_criteria_for_prompt(
+ [
+ _criterion(
+ "c0",
+ "The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ )
+ ]
+ )
+
+ assert json.loads(rendered) == [
+ {
+ "outcome": "The returned record includes service address.",
+ "implicit": False,
+ "method_mandated": False,
+ "level": "run",
+ "kind": "outcome",
+ "terminal_action_family": None,
+ "output_path": "output.service_address",
+ "expected_output_value": "1234 Sample Utility Way",
+ }
+ ]
+
+
+def test_active_criteria_rendering_includes_expected_output_shape_as_structured_data() -> None:
+ rendered = _render_active_criteria_for_prompt(
+ [
+ _criterion(
+ "c0",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ )
+ ]
+ )
+
+ assert json.loads(rendered) == [
+ {
+ "outcome": "The returned record includes confirmation number.",
+ "implicit": False,
+ "method_mandated": False,
+ "level": "run",
+ "kind": "outcome",
+ "terminal_action_family": None,
+ "output_path": "output.confirmation_number",
+ "expected_output_shape": "reference_code",
+ }
+ ]
+
+
def test_criteria_json_round_trips_contingent_on() -> None:
criteria = (
_criterion(
@@ -606,6 +931,131 @@ def test_criteria_json_round_trips_contingent_on() -> None:
assert restored == criteria
+def test_criteria_json_round_trips_expected_output_value() -> None:
+ criteria = (
+ _criterion(
+ "c0",
+ "The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ ),
+ )
+
+ restored = criteria_from_json(criteria_to_json(criteria))
+
+ assert restored == criteria
+
+
+def test_criteria_json_round_trips_expected_output_shape() -> None:
+ criteria = (
+ _criterion(
+ "c0",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ )
+
+ restored = criteria_from_json(criteria_to_json(criteria))
+
+ assert restored == criteria
+
+
+def test_request_policy_trace_exposes_requested_output_grounding_contract_without_values() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_address",
+ "The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ ),
+ _criterion(
+ "c_confirmation",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ _criterion("c_status", "The returned record includes status.", output_path="output.status"),
+ ]
+ )
+
+ trace = policy.to_trace_data()
+
+ assert trace["requested_output_criteria_count"] == 3
+ assert trace["requested_output_criterion_0_id"] == "c_address"
+ assert trace["requested_output_criterion_0_output_path"] == "output.service_address"
+ assert trace["requested_output_criterion_0_grounding_mode"] == "exact_value"
+ assert trace["requested_output_criterion_0_has_exact_value"] is True
+ assert "requested_output_criterion_0_expected_output_shape" not in trace
+ assert trace["requested_output_criterion_1_id"] == "c_confirmation"
+ assert trace["requested_output_criterion_1_output_path"] == "output.confirmation_number"
+ assert trace["requested_output_criterion_1_grounding_mode"] == "shape"
+ assert trace["requested_output_criterion_1_expected_output_shape"] == "reference_code"
+ assert trace["requested_output_criterion_1_has_exact_value"] is False
+ assert trace["requested_output_criterion_2_id"] == "c_status"
+ assert trace["requested_output_criterion_2_output_path"] == "output.status"
+ assert trace["requested_output_criterion_2_grounding_mode"] == "missing"
+ assert trace["requested_output_criterion_2_has_exact_value"] is False
+ assert "1234 Sample Utility Way" not in repr(trace)
+
+
+def test_reconcile_supersedes_same_output_path_with_different_expected_output_value() -> None:
+ stored = _stored(
+ _criterion(
+ "s0",
+ "The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="1234 Sample Utility Way",
+ )
+ )
+ fresh = [
+ _criterion(
+ "c0",
+ "The returned record includes service address.",
+ output_path="output.service_address",
+ expected_output_value="7890 Changed Ave",
+ )
+ ]
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ fresh,
+ actionable=True,
+ )
+
+ assert decision.action == "create"
+ assert decision.criteria == tuple(fresh)
+
+
+def test_reconcile_supersedes_same_output_path_with_different_expected_output_shape() -> None:
+ stored = _stored(
+ _criterion(
+ "s0",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ )
+ )
+ fresh = [
+ _criterion(
+ "c0",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_shape="numeric_identifier",
+ )
+ ]
+
+ decision = reconcile_completion_criteria(
+ StoredCriteriaSnapshot(active=stored, next_epoch=2),
+ fresh,
+ actionable=True,
+ )
+
+ assert decision.action == "create"
+ assert decision.criteria == tuple(fresh)
+
+
def test_criteria_json_round_trips_contingent_antecedent_output_path() -> None:
criteria = (
_criterion(
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index b755739bc..21068227a 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -89,8 +89,11 @@ class TestRequestPolicyPromptStructure:
rendered = _render()
assert (
"{outcome, contingent_on, contingent_antecedent_output_path, "
- "deliverable_kind, implicit, method_mandated, level, kind, terminal_action_family}"
+ "deliverable_kind, implicit, method_mandated, level, output_path, expected_output_value, "
+ "expected_output_shape, kind, terminal_action_family}"
) in rendered
+ assert "never hide it in outcome prose" in rendered
+ assert "reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label" in rendered
assert "kind=outcome|terminal_action" in rendered
assert "terminal_action_family=request|application|form|order|null" in rendered
assert 'The only supported non-null value is "registered_download"' in rendered
@@ -101,6 +104,8 @@ class TestRequestPolicyPromptStructure:
CompletionCriterion(
id="c0",
outcome="a commercial water service request is started",
+ expected_output_value="WTR-1842-DEMO",
+ expected_output_shape="reference_code",
kind="terminal_action",
terminal_action_family="request",
)
@@ -111,6 +116,8 @@ class TestRequestPolicyPromptStructure:
assert '"kind": "terminal_action"' in rendered
assert '"terminal_action_family": "request"' in rendered
+ assert '"expected_output_value": "WTR-1842-DEMO"' in rendered
+ assert '"expected_output_shape": "reference_code"' in rendered
class TestRawSecretBackstop: