Skip to content

Automate browser based workflows with AI

SKY-11833: type judgment-boolean provenance through Copilot completion grading (#7086)

Andrew Neilson973f67b0f88c
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
index 1b235e322..3de9fb10b 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-policy.j2
@@ -30,8 +30,8 @@ completion_criteria:
- contingent_on: null for ordinary criteria. Use a short antecedent string only when this criterion should be graded only if that antecedent happens, for example {"contingent_on": "the provider site blocks online submission", "contingent_antecedent_output_path": "output.blocker", "outcome": "the blocker is reported to the user"}. The antecedent goes in contingent_on; the consequent to grade goes in outcome. Do not encode if/then criteria only in prose.
- contingent_antecedent_output_path: null unless the antecedent is represented by a structured run output field. The only valid form is output.<field> (for example output.blocker). Do not emit transcript paths, selectors, regexes, JSONPath, screenshots, nested paths, or prose conditions here.
- deliverable_kind: null unless this criterion's requested output is satisfied by execution-owned browser-download registration rather than a user-authored extraction field. The only supported non-null value is "registered_download".
-- expected_output_value: null unless this criterion requires an exact structured output value the user explicitly provided. Put that value here as data; never hide it in outcome prose.
-- expected_output_shape: null unless this criterion requires a generated/provider structured output value whose exact value is not known before the run. Use only one of: reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label. Do not use this as a non-empty-value proxy; leave null when the field's shape is not one of those declared types.
+- expected_output_value: null unless this criterion requires an exact structured output value the user explicitly provided. Put that value here as data; never hide it in outcome prose. For a goal-judgment field whose expected answer is a boolean (e.g. selected_highest_priority), provide the JSON true or false here.
+- expected_output_shape: null unless this criterion requires a generated/provider structured output value whose exact value is not known before the run. Use only one of: reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label, goal_judgment_boolean. goal_judgment_boolean declares that the field is an artifact-computed true/false judgment; the typed boolean expected_output_value is the deterministic backstop. Do not use this as a non-empty-value proxy; leave null when the field's shape is not one of those declared types.
- requested_output_evidence_source: runtime_output|independent_run_evidence|registered_output_parameter|registered_artifact_content. Use runtime_output for site-produced values the code block directly reads from the page or browser state, such as dates, amounts, statuses, identifiers, names, or downloaded-file descriptors. Use independent_run_evidence only for artifact-computed judgments, selections, superlatives, rankings, validation booleans, or "best/correct/cheapest/highest" claims that cannot prove themselves by returning their own answer. Use registered_output_parameter or registered_artifact_content only when the requested datum must be proven by execution-owned registered output parameters or registered artifact content. Do not choose this from output_path names, fixture names, domain keywords, or prose labels; choose it from the user's requested evidence contract.
- When a selection, priority, ranking, or superlative criterion governs a returned requested-output field, set output_path for that returned field and requested_output_evidence_source=independent_run_evidence; the selected value cannot prove its own correctness.
- Example: if the user asks to return the selected or highest-priority document name, emit the criterion with output_path=output.document_name and requested_output_evidence_source=independent_run_evidence.
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 2160adc5e..53902574c 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -1283,7 +1283,7 @@ def _output_path_value_status(
return "empty_typed_value"
grounding_mode = getattr(verdict, "grounding_mode", None)
has_exact_value = getattr(verdict, "has_exact_value", False)
- if grounding_mode == "shape" and not has_exact_value:
+ if grounding_mode in {"shape", "judgment_boolean"} and not has_exact_value:
return "presence_only_evidence"
return "typed_value_unverified"
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 7bb29245b..2b0d9f197 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -21,15 +21,18 @@ from skyvern.forge.sdk.copilot.request_policy import (
ExpectedOutputShape,
RequestedOutputEvidenceSource,
TerminalActionFamily,
+ _canonical_bool_string,
_coerce_classification_output_key,
_coerce_expected_classification,
_coerce_expected_output_shape,
+ _coerce_expected_output_value,
_coerce_requested_output_evidence_source,
_normalize_contingent_antecedent_output_path,
_normalize_deliverable_kind,
is_fallback_floor_criterion,
normalized_criterion_outcome_key,
requested_output_path_for_field,
+ typed_expected_output_value_key,
)
ReconcileAction = Literal["create", "adopt_stored", "none"]
@@ -175,17 +178,22 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
family_raw if kind == "terminal_action" and family_raw in _TERMINAL_ACTION_FAMILIES else None
)
stored_output_path = output_path.strip() if isinstance(output_path, str) and output_path.strip() else None
- stored_expected_output_value = (
- expected_output_value.strip()
- if isinstance(expected_output_value, str) and expected_output_value.strip()
- else None
- )
+ stored_expected_output_value = _coerce_expected_output_value(expected_output_value)
stored_expected_output_shape = cast(ExpectedOutputShape | None, expected_output_shape)
+ if isinstance(stored_expected_output_value, str) and (
+ requested_output_evidence_source == "independent_run_evidence"
+ or stored_expected_output_shape == "goal_judgment_boolean"
+ ):
+ coerced_judgment_bool = _canonical_bool_string(stored_expected_output_value)
+ if coerced_judgment_bool is not None:
+ stored_expected_output_value = coerced_judgment_bool
if kind == "validation_classification":
stored_output_path = None
stored_expected_output_value = None
stored_expected_output_shape = None
requested_output_evidence_source = "runtime_output"
+ elif isinstance(stored_expected_output_value, bool) or stored_expected_output_shape == "goal_judgment_boolean":
+ requested_output_evidence_source = "independent_run_evidence"
criteria.append(
CompletionCriterion(
id=criterion_id,
@@ -216,7 +224,7 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
deliverable_kind_key = criterion.deliverable_kind or ""
- expected_output_value_key = criterion.expected_output_value or ""
+ expected_output_value_key = typed_expected_output_value_key(criterion.expected_output_value)
expected_output_shape_key = criterion.expected_output_shape or ""
requested_output_evidence_source_key = criterion.requested_output_evidence_source
classification_output_key = criterion.classification_output_key or ""
diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.py
index 11ef67d45..f90a7c4ce 100644
--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py
+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py
@@ -12,6 +12,9 @@ import yaml
from skyvern.forge.sdk.copilot.completion_verification import CriterionVerdict, EvidenceSourceKind, RunEvidenceSnapshot
from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
+ ExpectedOutputValue,
+ RequestedOutputEvidenceSource,
+ _is_judgment_boolean_criterion,
lookup_requested_output_path_alias,
schema_output_path_aliases_from_criteria,
)
@@ -22,6 +25,9 @@ _REQUESTED_OUTPUT_PREFIX = "output."
_STRUCTURAL_ABSTENTION_REASON_CODE = "structurally_abstained"
# Ignored at any requested-output traversal depth, including schema-declared nested fields.
_IGNORED_RUNTIME_FIELD_NAMES = frozenset({"evidence_text"})
+_INDEPENDENT_EVIDENCE_SOURCES: frozenset[EvidenceSourceKind] = frozenset(
+ {"independent_page_evidence", "registered_output_parameter", "registered_artifact_content"}
+)
class _GroundingCtx(Protocol):
@@ -57,6 +63,7 @@ def grade_requested_output_criteria(
requested_path_aliases = schema_output_path_aliases_from_criteria(criteria)
raw_authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx)
authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, requested_path_aliases)
+ declared_judgment_output_paths = _producer_declared_judgment_output_paths(copilot_ctx.code_artifact_metadata)
verdicts: list[CriterionVerdict] = []
for criterion in criteria:
path = _normalize_output_path(criterion.output_path)
@@ -73,7 +80,12 @@ def grade_requested_output_criteria(
)
)
continue
- accepted_runtime_outputs = dict(_accepted_runtime_outputs(snapshot.block_outputs, accepted_labels))
+ independent_labels = {
+ label for label, source in snapshot.block_output_sources.items() if source in _INDEPENDENT_EVIDENCE_SOURCES
+ }
+ accepted_runtime_outputs = dict(
+ _accepted_runtime_outputs(snapshot.block_outputs, accepted_labels, independent_labels)
+ )
accepted_output_sources = {
label: source
for label, source in snapshot.block_output_sources.items()
@@ -87,7 +99,16 @@ def grade_requested_output_criteria(
expected_value = criterion.expected_output_value
expected_shape = criterion.expected_output_shape
grounding_mode = _criterion_grounding_mode(criterion)
- trace_fields = _criterion_trace_fields(criterion, grounding_mode)
+ requires_declared_independent_evidence = bool(path) and path in declared_judgment_output_paths
+ effective_evidence_source: RequestedOutputEvidenceSource = (
+ "independent_run_evidence"
+ if requires_declared_independent_evidence
+ else criterion.requested_output_evidence_source
+ )
+ judgment_grounding_bars_self_emission = (
+ _is_judgment_boolean_criterion(criterion) or requires_declared_independent_evidence
+ )
+ trace_fields = _criterion_trace_fields(criterion, grounding_mode, effective_evidence_source)
if expected_value is not None:
refutation = _independent_evidence_text_refutation(
accepted_runtime_outputs,
@@ -118,6 +139,7 @@ def grade_requested_output_criteria(
else:
match_state, evidence_ref = _runtime_output_path_presence(accepted_runtime_outputs, path, projection_roots)
if match_state == "present" and evidence_ref is not None:
+ present_source = _evidence_source_for_ref(accepted_output_sources, evidence_ref)
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -128,7 +150,11 @@ def grade_requested_output_criteria(
"requested-output field is present, but the criterion lacks typed expected_output_value "
"or expected_output_shape to prove the value"
),
- evidence_source=_evidence_source_for_ref(accepted_output_sources, evidence_ref),
+ evidence_source=present_source,
+ self_emitted_judgment_not_independent=(
+ judgment_grounding_bars_self_emission
+ and present_source not in _INDEPENDENT_EVIDENCE_SOURCES
+ ),
**trace_fields,
)
)
@@ -147,6 +173,7 @@ def grade_requested_output_criteria(
)
continue
if match_state == "present" and evidence_ref is not None:
+ present_source = _evidence_source_for_ref(accepted_output_sources, evidence_ref)
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -157,13 +184,20 @@ def grade_requested_output_criteria(
"requested-output field is present with a typed expected_output_shape, but no exact "
"expected_output_value can prove or refute the value"
),
- evidence_source=_evidence_source_for_ref(accepted_output_sources, evidence_ref),
+ evidence_source=present_source,
+ self_emitted_judgment_not_independent=(
+ judgment_grounding_bars_self_emission and present_source not in _INDEPENDENT_EVIDENCE_SOURCES
+ ),
**trace_fields,
)
)
continue
if match_state == "satisfied" and evidence_ref is not None:
- if criterion.requested_output_evidence_source == "independent_run_evidence":
+ satisfied_source = _evidence_source_for_ref(accepted_output_sources, evidence_ref)
+ requires_independent_evidence = (
+ effective_evidence_source == "independent_run_evidence" or _is_judgment_boolean_criterion(criterion)
+ )
+ if requires_independent_evidence and satisfied_source not in _INDEPENDENT_EVIDENCE_SOURCES:
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -173,7 +207,7 @@ def grade_requested_output_criteria(
missing_evidence=(
"self-emitted requested output is not independent evidence for this criterion"
),
- evidence_source=_evidence_source_for_ref(accepted_output_sources, evidence_ref),
+ evidence_source=satisfied_source,
self_emitted_judgment_not_independent=True,
**trace_fields,
)
@@ -237,7 +271,7 @@ def _independent_evidence_text_refutation(
block_outputs: Mapping[str, Any],
output_sources: Mapping[str, EvidenceSourceKind],
path: str,
- expected_value: str,
+ expected_value: ExpectedOutputValue,
projection_roots: set[str],
) -> tuple[str, EvidenceSourceKind] | None:
parts = _path_parts(path)
@@ -245,6 +279,16 @@ def _independent_evidence_text_refutation(
source = output_sources.get(label)
if source != "independent_page_evidence" or not isinstance(payload, Mapping):
continue
+ if isinstance(expected_value, bool):
+ values, source_path = _runtime_path_values(payload, parts, path, projection_roots)
+ grounded_booleans = [_bool_canonical(value) for value in values]
+ if (
+ grounded_booleans
+ and all(value is not None for value in grounded_booleans)
+ and expected_value not in grounded_booleans
+ ):
+ return f"block_outputs:{label}.{source_path}", source
+ continue
evidence_text = payload.get("evidence_text")
if not isinstance(evidence_text, str) or not _expected_value_in_text(evidence_text, expected_value):
continue
@@ -260,7 +304,11 @@ def _expected_value_in_text(text: str, expected: str) -> bool:
return bool(target) and (target in observed or _compact_expected_text(target) in _compact_expected_text(observed))
-def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:
+def _criterion_grounding_mode(
+ criterion: CompletionCriterion,
+) -> Literal["exact_value", "shape", "missing", "judgment_boolean"]:
+ if _is_judgment_boolean_criterion(criterion):
+ return "judgment_boolean"
if criterion.expected_output_value is not None:
return "exact_value"
if criterion.expected_output_shape is not None:
@@ -268,16 +316,68 @@ def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_
return "missing"
-def _criterion_trace_fields(criterion: CompletionCriterion, grounding_mode: str) -> dict[str, Any]:
+def _criterion_trace_fields(
+ criterion: CompletionCriterion,
+ grounding_mode: str,
+ evidence_source: RequestedOutputEvidenceSource | None = None,
+) -> dict[str, Any]:
return {
"output_path": criterion.output_path,
"grounding_mode": grounding_mode,
"expected_output_shape": criterion.expected_output_shape,
"has_exact_value": criterion.expected_output_value is not None,
- "requested_output_evidence_source": criterion.requested_output_evidence_source,
+ "requested_output_evidence_source": evidence_source or criterion.requested_output_evidence_source,
}
+def _producer_declared_judgment_output_paths(metadata: object) -> set[str]:
+ if not isinstance(metadata, Mapping):
+ return set()
+ paths: set[str] = set()
+ for artifact in metadata.values():
+ if not isinstance(artifact, Mapping):
+ continue
+ declared_criteria = artifact.get("completion_criteria")
+ if isinstance(declared_criteria, list):
+ for declared in declared_criteria:
+ if not isinstance(declared, Mapping):
+ continue
+ if declared.get("requested_output_evidence_source") != "independent_run_evidence":
+ continue
+ normalized = _normalize_output_path(declared.get("output_path"))
+ if normalized:
+ paths.add(normalized)
+ for row_group in (artifact.get("claimed_outcomes"), artifact.get("terminal_verifier_expectations")):
+ rows = [row for row in row_group if isinstance(row, Mapping)] if isinstance(row_group, list) else []
+ for row in rows:
+ paths.update(_schema_boolean_output_paths(_parse_extraction_schema(row.get("extraction_schema"))))
+ return paths
+
+
+def _schema_boolean_output_paths(schema: Mapping[str, Any] | None, prefix: str = "") -> set[str]:
+ if not isinstance(schema, Mapping):
+ return set()
+ if schema.get("type") == "array":
+ items = schema.get("items")
+ array_prefix = f"{prefix}[]" if prefix else ""
+ return _schema_boolean_output_paths(items, array_prefix) if isinstance(items, Mapping) else set()
+ properties = schema.get("properties")
+ if not isinstance(properties, Mapping):
+ if schema.get("type") == "boolean" and prefix:
+ return {_normalize_output_path(prefix)}
+ return set()
+ paths: set[str] = set()
+ for raw_name, child_schema in properties.items():
+ name = str(raw_name).strip()
+ if not name or not isinstance(child_schema, Mapping):
+ continue
+ child_prefix = f"{prefix}.{name}" if prefix else name
+ if child_schema.get("type") == "boolean":
+ paths.add(_normalize_output_path(child_prefix))
+ paths.update(_schema_boolean_output_paths(child_schema, child_prefix))
+ return paths
+
+
def _authored_output_contract_paths_by_label(
copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None
) -> dict[str, set[str]]:
@@ -415,14 +515,16 @@ def _runtime_projection_roots(authored_paths: set[str]) -> set[str]:
return {parts[0] for path in authored_paths if len(parts := _path_parts(path)) > 1}
-def _accepted_runtime_outputs(block_outputs: Mapping[str, Any], accepted_labels: set[str]) -> list[tuple[str, Any]]:
- if not accepted_labels:
+def _accepted_runtime_outputs(
+ block_outputs: Mapping[str, Any], accepted_labels: set[str], independent_labels: set[str]
+) -> list[tuple[str, Any]]:
+ if not accepted_labels and not independent_labels:
return []
accepted_output_keys = {f"{label}_output" for label in accepted_labels}
return [
(label, payload)
for label, payload in block_outputs.items()
- if label in accepted_labels or label in accepted_output_keys
+ if label in accepted_labels or label in accepted_output_keys or label in independent_labels
]
@@ -457,7 +559,10 @@ def _dict_keys(node: ast.Dict) -> set[str]:
def _runtime_output_path_match(
- block_outputs: Mapping[str, Any], path: str, expected_value: str | None, projection_roots: set[str] | None = None
+ block_outputs: Mapping[str, Any],
+ path: str,
+ expected_value: ExpectedOutputValue | None,
+ projection_roots: set[str] | None = None,
) -> tuple[str, str | None]:
parts = _path_parts(path)
if not parts or expected_value is None:
@@ -580,7 +685,22 @@ def _is_meaningful_requested_value(value: Any) -> bool:
return True
-def _value_matches_expected(value: Any, expected: str) -> bool:
+def _bool_canonical(value: Any) -> bool | None:
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str):
+ collapsed = value.strip().casefold()
+ if collapsed == "true":
+ return True
+ if collapsed == "false":
+ return False
+ return None
+
+
+def _value_matches_expected(value: Any, expected: ExpectedOutputValue) -> bool:
+ if isinstance(expected, bool):
+ canonical = _bool_canonical(value)
+ return canonical is not None and canonical == expected
if isinstance(value, Mapping):
return any(_value_matches_expected(item, expected) for item in value.values())
if isinstance(value, list):
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index fcd4d5f81..0a8bf7827 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -13,8 +13,8 @@ from __future__ import annotations
import asyncio
import json
import re
-from collections.abc import Collection, Iterable
-from dataclasses import dataclass, field
+from collections.abc import Collection, Iterable, Mapping
+from dataclasses import dataclass, field, replace
from typing import Any, Literal
import structlog
@@ -89,7 +89,7 @@ class CriterionVerdict:
evidence_ref: str | None = None
missing_evidence: str | None = None
output_path: str | None = None
- grounding_mode: Literal["exact_value", "shape", "missing", "terminal_record"] | None = None
+ grounding_mode: Literal["exact_value", "shape", "missing", "terminal_record", "judgment_boolean"] | None = None
expected_output_shape: str | None = None
has_exact_value: bool = False
requested_output_evidence_source: str | None = None
@@ -574,6 +574,13 @@ def _is_structural_no_blocker_marker(value: Any) -> bool:
}
+def _evidence_source_from_ref(
+ block_output_sources: Mapping[str, EvidenceSourceKind], evidence_ref: str | None
+) -> EvidenceSourceKind | None:
+ label = _evidence_ref_record_label(evidence_ref)
+ return block_output_sources.get(label) if label else None
+
+
def _coerce_result(
raw: Any,
criterion_ids: list[str],
@@ -582,6 +589,7 @@ def _coerce_result(
contingent_on_by_criterion_id: dict[str, str] | None = None,
contingent_antecedent_output_path_by_criterion_id: dict[str, str] | None = None,
structural_unfired_criterion_ids: Iterable[str] = (),
+ block_output_sources: Mapping[str, EvidenceSourceKind] | None = None,
) -> CompletionVerificationResult:
if isinstance(raw, bytes):
raw = raw.decode("utf-8", errors="replace")
@@ -624,6 +632,7 @@ def _coerce_result(
reason_code=reason_code,
evidence_ref=evidence_ref,
missing_evidence=missing_evidence,
+ evidence_source=_evidence_source_from_ref(block_output_sources or {}, evidence_ref),
)
verdicts = [by_id.get(criterion_id, _missing_verdict(criterion_id)) for criterion_id in criterion_ids]
@@ -1903,6 +1912,7 @@ _REPERCEPTION_CONTRADICTION_EVIDENCE_REFS = frozenset({"scout_synthesized_browse
_INDEPENDENT_REQUESTED_OUTPUT_CORROBORATOR_SOURCES = frozenset(
{"independent_page_evidence", "registered_output_parameter", "registered_artifact_content"}
)
+_SELF_EMITTED_EVIDENCE_SOURCES = frozenset({"runtime_output", "same_record_context"})
def _is_satisfied_observed_end_state_verdict(verdict: CriterionVerdict) -> bool:
@@ -1943,12 +1953,17 @@ def _is_corroborated_structural_requested_output_abstention(
criterion_id: str,
has_corroboration: bool,
) -> bool:
+ if verdict.requested_output_evidence_source == "independent_run_evidence":
+ corroborated = _has_independent_satisfied_requested_output_corroborator(verdict_by_id, criterion_id)
+ else:
+ corroborated = has_corroboration or _has_satisfied_requested_output_corroborator(verdict_by_id, criterion_id)
return (
- (has_corroboration or _has_satisfied_requested_output_corroborator(verdict_by_id, criterion_id))
+ corroborated
and _is_structural_requested_output_abstention(verdict)
and bool(verdict.evidence_ref)
and bool(verdict.output_path)
and not verdict.has_exact_value
+ and verdict.grounding_mode != "judgment_boolean"
)
@@ -1983,6 +1998,79 @@ def _is_requested_output_corroborator_id(candidate_id: str, criterion_id: str) -
return candidate_id.removeprefix(f"{prefix}_").isdigit()
+_REQUESTED_OUTPUT_CORROBORATOR_MARKER = "__requested_output_corroborator"
+_TASK_OUTPUT_PARAMETER_SUFFIX = "_output"
+_SAME_RECORD_CORROBORATOR_MISSING_EVIDENCE = (
+ "requested-output corroborator cited the criterion's own emitted output; independent corroboration is required"
+)
+
+
+def _requested_output_corroborator_source_id(candidate_id: str) -> str | None:
+ index = candidate_id.find(_REQUESTED_OUTPUT_CORROBORATOR_MARKER)
+ if index <= 0:
+ return None
+ suffix = candidate_id[index + len(_REQUESTED_OUTPUT_CORROBORATOR_MARKER) :]
+ if suffix and not (suffix.startswith("_") and suffix[1:].isdigit()):
+ return None
+ return candidate_id[:index]
+
+
+def _evidence_ref_record_label(evidence_ref: str | None) -> str | None:
+ if not evidence_ref:
+ return None
+ ref = evidence_ref.strip().removeprefix("block_outputs:")
+ label = ref.split(".", 1)[0].strip()
+ return label or None
+
+
+def _output_record_root(label: str) -> str:
+ return label[: -len(_TASK_OUTPUT_PARAMETER_SUFFIX)] if label.endswith(_TASK_OUTPUT_PARAMETER_SUFFIX) else label
+
+
+def _is_self_emitted_source_verdict(verdict: CriterionVerdict) -> bool:
+ return verdict.evidence_source is None or verdict.evidence_source in _SELF_EMITTED_EVIDENCE_SOURCES
+
+
+def _corroborator_draws_only_from_source_record(
+ corroborator: CriterionVerdict,
+ verdict_by_id: Mapping[str, CriterionVerdict],
+) -> bool:
+ source_id = _requested_output_corroborator_source_id(corroborator.criterion_id)
+ source = verdict_by_id.get(source_id) if source_id else None
+ if source is None or not _is_self_emitted_source_verdict(source):
+ return False
+ source_label = _evidence_ref_record_label(source.evidence_ref)
+ corroborator_label = _evidence_ref_record_label(corroborator.evidence_ref)
+ if not source_label or not corroborator_label:
+ return False
+ return _output_record_root(source_label) == _output_record_root(corroborator_label)
+
+
+def _enforce_requested_output_corroborator_independence(
+ verdicts: list[CriterionVerdict],
+) -> list[CriterionVerdict]:
+ verdict_by_id = {verdict.criterion_id: verdict for verdict in verdicts}
+ updated: list[CriterionVerdict] = []
+ for verdict in verdicts:
+ if (
+ verdict.satisfied
+ and verdict.reason_code == "evidence_confirms"
+ and _requested_output_corroborator_source_id(verdict.criterion_id) is not None
+ and _corroborator_draws_only_from_source_record(verdict, verdict_by_id)
+ ):
+ updated.append(
+ replace(
+ verdict,
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ missing_evidence=_SAME_RECORD_CORROBORATOR_MISSING_EVIDENCE,
+ )
+ )
+ continue
+ updated.append(verdict)
+ return updated
+
+
def only_structural_requested_output_abstentions(result: CompletionVerificationResult) -> bool:
if result.status != "evaluated" or not result.criterion_ids or result.is_fully_satisfied():
return False
@@ -2056,6 +2144,11 @@ def combine_verification_results(
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
structural_unfired_criterion_ids=structural_unfired_ids,
)
+ if run_result is not None:
+ run_result = replace(
+ run_result,
+ verdicts=_enforce_requested_output_corroborator_independence(run_result.verdicts),
+ )
verdict_by_id = {verdict.criterion_id: verdict for verdict in definition_verdicts}
if run_result is not None:
verdict_by_id.update({verdict.criterion_id: verdict for verdict in run_result.verdicts})
@@ -2139,4 +2232,5 @@ async def evaluate_completion_criteria(
contingent_on_by_criterion_id=contingent_on_by_id,
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
structural_unfired_criterion_ids=structural_unfired_ids,
+ block_output_sources=snapshot.block_output_sources,
)
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index e9d4b2569..f22697499 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -178,6 +178,7 @@ _CRITERION_LEVELS: frozenset[str] = frozenset({"definition", "run"})
CriterionKind = Literal["outcome", "terminal_action", "validation_classification"]
TerminalActionFamily = Literal["request", "application", "form", "order"]
ClassificationTarget = str | bool
+ExpectedOutputValue = str | bool
ExpectedOutputShape = Literal[
"reference_code",
"numeric_identifier",
@@ -186,6 +187,7 @@ ExpectedOutputShape = Literal[
"status_label",
"money_amount",
"owner_label",
+ "goal_judgment_boolean",
]
RequestedOutputEvidenceSource = Literal[
"runtime_output",
@@ -257,7 +259,7 @@ class CompletionCriterion:
# YAML; "run": an end state only a run can evidence. Invalid input coerces to "run".
level: CriterionLevel = "run"
output_path: str | None = None
- expected_output_value: str | None = None
+ expected_output_value: ExpectedOutputValue | None = None
expected_output_shape: ExpectedOutputShape | None = None
requested_output_evidence_source: RequestedOutputEvidenceSource = "runtime_output"
kind: CriterionKind = "outcome"
@@ -400,7 +402,25 @@ def request_policy_has_present_completion_contract(request_policy: RequestPolicy
return request_policy.completion_contract_status == "present" or bool(request_policy.completion_criteria)
-def _criterion_grounding_mode(criterion: CompletionCriterion) -> Literal["exact_value", "shape", "missing"]:
+def _is_judgment_boolean_criterion(criterion: CompletionCriterion) -> bool:
+ return (
+ isinstance(criterion.expected_output_value, bool) or criterion.expected_output_shape == "goal_judgment_boolean"
+ )
+
+
+def typed_expected_output_value_key(value: ExpectedOutputValue | None) -> str:
+ if isinstance(value, bool):
+ return f"bool:{value}"
+ if isinstance(value, str):
+ return f"str:{value}"
+ return ""
+
+
+def _criterion_grounding_mode(
+ criterion: CompletionCriterion,
+) -> Literal["exact_value", "shape", "missing", "judgment_boolean"]:
+ if _is_judgment_boolean_criterion(criterion):
+ return "judgment_boolean"
if criterion.expected_output_value is not None:
return "exact_value"
if criterion.expected_output_shape is not None:
@@ -609,6 +629,24 @@ def _coerce_terminal_action_family(value: Any, kind: CriterionKind) -> TerminalA
return None
+def _coerce_expected_output_value(value: Any) -> ExpectedOutputValue | None:
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str):
+ collapsed = " ".join(value.split())[:_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS].strip()
+ return collapsed or None
+ return None
+
+
+def _canonical_bool_string(value: str) -> bool | None:
+ collapsed = value.strip().casefold()
+ if collapsed == "true":
+ return True
+ if collapsed == "false":
+ return False
+ return None
+
+
def _coerce_expected_output_shape(value: Any) -> ExpectedOutputShape | None:
if isinstance(value, str) and value in _EXPECTED_OUTPUT_SHAPES:
return cast(ExpectedOutputShape, value)
@@ -710,7 +748,7 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
if not isinstance(raw, list):
return []
criteria: list[CompletionCriterion] = []
- seen: set[tuple[str, str, str, str, str, str]] = set()
+ seen: set[tuple[str, ...]] = set()
for item in raw:
if not isinstance(item, dict):
continue
@@ -722,17 +760,20 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
continue
output_path_raw = item.get("output_path")
output_path = output_path_raw.strip() if isinstance(output_path_raw, str) and output_path_raw.strip() else None
- expected_output_value_raw = item.get("expected_output_value")
- expected_output_value = (
- " ".join(expected_output_value_raw.split())[:_COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS].strip()
- if isinstance(expected_output_value_raw, str)
- else None
+ expected_output_value: ExpectedOutputValue | None = _coerce_expected_output_value(
+ item.get("expected_output_value")
)
- expected_output_value = expected_output_value or None
expected_output_shape = _coerce_expected_output_shape(item.get("expected_output_shape"))
requested_output_evidence_source = _coerce_requested_output_evidence_source(
item.get("requested_output_evidence_source")
)
+ if isinstance(expected_output_value, str) and (
+ requested_output_evidence_source == "independent_run_evidence"
+ or expected_output_shape == "goal_judgment_boolean"
+ ):
+ coerced_judgment_bool = _canonical_bool_string(expected_output_value)
+ if coerced_judgment_bool is not None:
+ expected_output_value = coerced_judgment_bool
contingent_on_raw = item.get("contingent_on")
contingent_on = (
" ".join(contingent_on_raw.split())[:_COMPLETION_CRITERION_CONTINGENT_ON_MAX_CHARS].strip()
@@ -760,6 +801,8 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
expected_output_value = None
expected_output_shape = None
requested_output_evidence_source = "runtime_output"
+ elif isinstance(expected_output_value, bool) or expected_output_shape == "goal_judgment_boolean":
+ requested_output_evidence_source = "independent_run_evidence"
key = (
contingent_on or "",
contingent_antecedent_output_path or "",
@@ -767,6 +810,9 @@ def _parse_completion_criteria(raw: Any) -> list[CompletionCriterion]:
deliverable_kind or "",
kind,
str(expected_classification) if expected_classification is not None else "",
+ typed_expected_output_value_key(expected_output_value),
+ expected_output_shape or "",
+ requested_output_evidence_source,
)
if key in seen:
continue
@@ -1073,8 +1119,8 @@ def _criterion_text_covers_any_requested_output(criterion: CompletionCriterion,
def _requested_output_expected_values_from_criteria(
criteria: list[CompletionCriterion],
requested_specs: list[tuple[str, str, str]],
-) -> dict[str, str]:
- values: dict[str, str] = {}
+) -> dict[str, ExpectedOutputValue]:
+ values: dict[str, ExpectedOutputValue] = {}
for criterion in criteria:
if criterion.level == "definition" or criterion.method_mandated:
continue
@@ -1084,7 +1130,7 @@ def _requested_output_expected_values_from_criteria(
):
continue
value = criterion.expected_output_value
- if value:
+ if value is not None:
values.setdefault(output_path, value)
return values
@@ -1446,6 +1492,7 @@ def _apply_validation_classification_completion_criteria(policy: RequestPolicy)
output_path=None,
expected_output_value=None,
expected_output_shape=None,
+ requested_output_evidence_source="runtime_output",
deliverable_kind=None,
terminal_action_family=None,
classification_output_key=output_key,
@@ -1484,7 +1531,7 @@ def _render_active_criteria_for_prompt(criteria: list[CompletionCriterion] | Non
item["deliverable_kind"] = criterion.deliverable_kind
if criterion.output_path:
item["output_path"] = criterion.output_path
- if criterion.expected_output_value:
+ if criterion.expected_output_value is not None:
item["expected_output_value"] = criterion.expected_output_value
if criterion.expected_output_shape:
item["expected_output_shape"] = criterion.expected_output_shape
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index c4e06e5d1..6985be0e0 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -18,6 +18,8 @@ from skyvern.forge.sdk.copilot.agent import (
)
from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criteria_to_json
from skyvern.forge.sdk.copilot.completion_output_grounding import (
+ _schema_boolean_output_paths,
+ _value_matches_expected,
grade_requested_output_criteria,
split_requested_output_criteria,
)
@@ -5021,6 +5023,297 @@ def test_requested_output_independent_evidence_source_does_not_self_attest() ->
assert trace.to_trace_data()["verdict_0_self_emitted_judgment_not_independent"] is True
+def test_shape_judgment_self_emitted_from_runtime_engages_veto_without_exact_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("selected_highest_priority")
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"extract_profile": {"selected_highest_priority": True}},
+ block_output_sources={"extract_profile": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].grounding_mode == "judgment_boolean"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+
+
+def test_presence_only_extraction_typed_independent_by_classifier_stays_veto_exempt() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("document_name")
+ criteria = [
+ _criterion(
+ "c_document_name",
+ "The highest-priority document name is returned.",
+ output_path="output.document_name",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"extract_profile": {"document_name": "Selected Document.pdf"}},
+ block_output_sources={"extract_profile": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].grounding_mode == "missing"
+ assert verdicts[0].self_emitted_judgment_not_independent is False
+
+
+def _metadata_with_declared_independent_criterion(label: str, output_path: str) -> dict[str, Any]:
+ normalized = output_path.removeprefix("output.")
+ return {
+ label: {
+ "claimed_outcomes": [{"goal_value_paths": [normalized]}],
+ "completion_criteria": [
+ {"output_path": output_path, "requested_output_evidence_source": "independent_run_evidence"}
+ ],
+ }
+ }
+
+
+def test_producer_declared_independent_source_engages_veto_despite_classifier_string_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_with_declared_independent_criterion(
+ "select_document", "output.selected_highest_priority"
+ )
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_value="true",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"select_document": {"selected_highest_priority": True}},
+ block_output_sources={"select_document": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+ assert verdicts[0].requested_output_evidence_source == "independent_run_evidence"
+
+
+def test_presence_only_producer_declared_independent_path_engages_veto() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_with_declared_independent_criterion(
+ "select_document", "output.selected_highest_priority"
+ )
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"select_document": {"selected_highest_priority": True}},
+ block_output_sources={"select_document": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].grounding_mode == "missing"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+
+
+def test_producer_declared_boolean_schema_engages_veto_despite_classifier_string_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {
+ "select_document": {
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": ["selected_highest_priority"],
+ "extraction_schema": {
+ "type": "object",
+ "properties": {"selected_highest_priority": {"type": "boolean"}},
+ },
+ }
+ ],
+ }
+ }
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_value="true",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"select_document": {"selected_highest_priority": True}},
+ block_output_sources={"select_document": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+
+
+def test_schema_boolean_output_paths_captures_array_of_boolean_leaf() -> None:
+ schema = {
+ "type": "object",
+ "properties": {"judgments": {"type": "array", "items": {"type": "boolean"}}},
+ }
+ assert _schema_boolean_output_paths(schema) == {"judgments[]"}
+
+
+def test_array_of_boolean_judgment_path_self_emitted_from_runtime_abstains() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {
+ "select_document": {
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": ["judgments[]"],
+ "extraction_schema": {
+ "type": "object",
+ "properties": {"judgments": {"type": "array", "items": {"type": "boolean"}}},
+ },
+ }
+ ],
+ }
+ }
+ criteria = [
+ _criterion(
+ "c_selected",
+ "Each candidate was judged against the highest-priority rule.",
+ output_path="output.judgments[]",
+ expected_output_value="true",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"select_document": {"judgments": [True]}},
+ block_output_sources={"select_document": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+
+
+def test_producer_declared_independent_criterion_stays_in_requested_output_grader_split() -> None:
+ criterion = _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_value="true",
+ requested_output_evidence_source="runtime_output",
+ )
+ requested, remaining = split_requested_output_criteria([criterion])
+ assert requested == [criterion]
+ assert remaining == []
+
+
+def test_producer_declared_independent_source_still_satisfies_with_independent_corroboration() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_with_declared_independent_criterion(
+ "select_document", "output.selected_highest_priority"
+ )
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_value="true",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"page_probe": {"selected_highest_priority": True}},
+ block_output_sources={"page_probe": "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def test_runtime_site_datum_still_self_confirms_when_other_path_declared_independent() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {
+ "extract_profile": {
+ "claimed_outcomes": [{"goal_value_paths": ["status", "selected_highest_priority"]}],
+ "completion_criteria": [
+ {
+ "output_path": "output.selected_highest_priority",
+ "requested_output_evidence_source": "independent_run_evidence",
+ }
+ ],
+ }
+ }
+ criteria = [
+ _criterion(
+ "c_status",
+ "The returned record includes status.",
+ output_path="output.status",
+ expected_output_value="Active",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"extract_profile": {"status": "Active"}},
+ block_output_sources={"extract_profile": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].requested_output_evidence_source == "runtime_output"
+
+
@pytest.mark.asyncio
async def test_admitted_code_artifact_metadata_propagates_requested_output_evidence_source() -> None:
ctx = _run_ctx()
@@ -5524,6 +5817,72 @@ def test_requested_output_ignores_unrelated_block_outputs_when_matching_values(
assert verdicts[0].evidence_ref == expected_ref
+def test_requested_output_string_distinct_independent_page_evidence_refutes_matching_self_emission() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "extract_profile": {"confirmation_number": "WTR-1842-DEMO"},
+ "current_page_observation": {"confirmation_number": "WTR-9999-WRONG"},
+ },
+ block_output_sources={
+ "extract_profile": "runtime_output",
+ "current_page_observation": "independent_page_evidence",
+ },
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:current_page_observation.confirmation_number"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def test_requested_output_string_distinct_independent_corroborator_mismatch_contradicts() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-0000-BAD",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "utility_citrus_turn_on": {"confirmation_number": "WTR-0000-BAD"},
+ "current_page_observation": {"confirmation_number": "WTR-1842-DEMO"},
+ },
+ block_output_sources={
+ "utility_citrus_turn_on": "runtime_output",
+ "current_page_observation": "independent_page_evidence",
+ },
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:current_page_observation.confirmation_number"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
def test_requested_output_exact_value_unrelated_sibling_does_not_refute_matching_target() -> None:
ctx = _run_ctx()
ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
@@ -8724,3 +9083,387 @@ def test_completion_contract_not_violated_unavailable_blocks_surfacing() -> None
# An unavailable verdict means the outcome could not be verified: do not surface
# the workflow as verified on run status alone.
assert _completion_contract_not_violated(ctx) is False # type: ignore[arg-type]
+
+
+def test_judgment_boolean_self_emitted_from_runtime_source_abstains() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("selected_highest_priority")
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The returned record reports the highest-priority selection judgment.",
+ output_path="output.selected_highest_priority",
+ expected_output_value=True,
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"extract_profile": {"selected_highest_priority": True}},
+ block_output_sources={"extract_profile": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "structurally_abstained"
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+ assert verdicts[0].grounding_mode == "judgment_boolean"
+
+
+def test_judgment_boolean_confirmed_by_independent_page_evidence() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("selected_highest_priority")
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The returned record reports the highest-priority selection judgment.",
+ output_path="output.selected_highest_priority",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"current_page_observation": {"selected_highest_priority": True}},
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def test_judgment_boolean_refuted_by_independent_page_evidence_at_structured_path() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("selected_highest_priority")
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The returned record reports the highest-priority selection judgment.",
+ output_path="output.selected_highest_priority",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"current_page_observation": {"selected_highest_priority": False}},
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:current_page_observation.selected_highest_priority"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def test_bool_expected_never_matches_container_with_unrelated_nested_boolean() -> None:
+ assert _value_matches_expected({"answer": False, "reviewed": True}, True) is False
+ assert _value_matches_expected([{"answer": False}, {"other": True}], True) is False
+ assert _value_matches_expected(True, True) is True
+ assert _value_matches_expected("true", True) is True
+ assert _value_matches_expected(False, True) is False
+
+
+def test_judgment_boolean_refuted_when_independent_evidence_resolves_false_leaf() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("answer")
+ criteria = [
+ _criterion(
+ "c_answer",
+ "The returned record reports the affirmative judgment.",
+ output_path="output.answer",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"current_page_observation": {"answer": False, "reviewed": True}},
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+ assert verdicts[0].evidence_ref == "block_outputs:current_page_observation.answer"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def _judgment_shape_abstention() -> CriterionVerdict:
+ return CriterionVerdict(
+ criterion_id="c_judgment",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:extract_profile.selected_highest_priority",
+ output_path="output.selected_highest_priority",
+ grounding_mode="judgment_boolean",
+ has_exact_value=False,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+
+
+def test_judgment_abstention_gets_no_floor_credit_from_generic_corroboration() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_judgment"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ _judgment_shape_abstention(),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_value_less_judgment_abstention_never_certifies_via_independent_corroborator() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_judgment"],
+ verdicts=[
+ _judgment_shape_abstention(),
+ CriterionVerdict(
+ criterion_id="c_judgment__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_source="independent_page_evidence",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_judgment_abstention_rejects_non_independent_corroborator() -> None:
+ result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_judgment"],
+ verdicts=[
+ _judgment_shape_abstention(),
+ CriterionVerdict(
+ criterion_id="c_judgment__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_source="runtime_output",
+ ),
+ ],
+ )
+
+ assert result.is_fully_satisfied() is False
+
+
+def test_coerce_result_stamps_evidence_source_from_snapshot_sources() -> None:
+ raw = {
+ "verdicts": [
+ {
+ "criterion_id": "c_corroborator",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "block_outputs:current_page_observation.selected_highest_priority",
+ }
+ ]
+ }
+
+ result = _coerce_result(
+ raw,
+ ["c_corroborator"],
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ )
+
+ assert result.verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def _verdict_for(result: CompletionVerificationResult, criterion_id: str) -> CriterionVerdict:
+ return next(verdict for verdict in result.verdicts if verdict.criterion_id == criterion_id)
+
+
+def test_requested_output_corroborator_same_record_self_confirmation_fails_closed() -> None:
+ run_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c0", "c1", "c1__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c0",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_turn_on_request",
+ grounding_mode="terminal_record",
+ evidence_source="terminal_record",
+ ),
+ CriterionVerdict(
+ criterion_id="c1",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:submit_turn_on_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ requested_output_evidence_source="runtime_output",
+ evidence_source="runtime_output",
+ ),
+ CriterionVerdict(
+ criterion_id="c1__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="submit_turn_on_request_output",
+ ),
+ ],
+ )
+
+ result = combine_verification_results(["c0", "c1", "c1__requested_output_corroborator"], run_result, [])
+
+ corroborator = _verdict_for(result, "c1__requested_output_corroborator")
+ assert corroborator.state == "unsatisfied"
+ assert corroborator.reason_code == "evidence_contradicts"
+ assert result.is_fully_satisfied() is False
+
+
+def test_requested_output_corroborator_distinct_record_still_satisfies() -> None:
+ run_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c1", "c1__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c1",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:submit_turn_on_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ requested_output_evidence_source="runtime_output",
+ evidence_source="runtime_output",
+ ),
+ CriterionVerdict(
+ criterion_id="c1__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ evidence_source="independent_page_evidence",
+ ),
+ ],
+ )
+
+ result = combine_verification_results(["c1", "c1__requested_output_corroborator"], run_result, [])
+
+ corroborator = _verdict_for(result, "c1__requested_output_corroborator")
+ assert corroborator.state == "satisfied"
+ assert corroborator.reason_code == "evidence_confirms"
+
+
+def test_coerce_result_stamps_evidence_source_from_bare_label_ref() -> None:
+ raw = {
+ "verdicts": [
+ {
+ "criterion_id": "c_corroborator",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation",
+ }
+ ]
+ }
+
+ result = _coerce_result(
+ raw,
+ ["c_corroborator"],
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ )
+
+ assert result.verdicts[0].evidence_source == "independent_page_evidence"
+
+
+def test_bare_label_corroborator_certifies_self_emitted_judgment() -> None:
+ coerced_corroborator = _coerce_result(
+ {
+ "verdicts": [
+ {
+ "criterion_id": "c_judgment__requested_output_corroborator",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation",
+ }
+ ]
+ },
+ ["c_judgment__requested_output_corroborator"],
+ block_output_sources={"current_page_observation": "independent_page_evidence"},
+ ).verdicts[0]
+ run_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c_reach", "c_judgment", "c_judgment__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c_reach",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="observed_end_state_url",
+ ),
+ CriterionVerdict(
+ criterion_id="c_judgment",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ evidence_ref="block_outputs:select_document.selected_highest_priority",
+ output_path="output.selected_highest_priority",
+ grounding_mode="judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_source="runtime_output",
+ self_emitted_judgment_not_independent=True,
+ ),
+ coerced_corroborator,
+ ],
+ )
+
+ result = combine_verification_results(
+ ["c_reach", "c_judgment", "c_judgment__requested_output_corroborator"], run_result, []
+ )
+
+ assert coerced_corroborator.evidence_source == "independent_page_evidence"
+ assert result.is_fully_satisfied() is True
+
+
+def test_requested_output_corroborator_same_record_not_flipped_when_source_is_independent() -> None:
+ run_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["c1", "c1__requested_output_corroborator"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="c1",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="block_outputs:submit_turn_on_request.confirmation_number",
+ output_path="output.confirmation_number",
+ grounding_mode="shape",
+ requested_output_evidence_source="independent_run_evidence",
+ evidence_source="independent_page_evidence",
+ ),
+ CriterionVerdict(
+ criterion_id="c1__requested_output_corroborator",
+ state="satisfied",
+ reason_code="evidence_confirms",
+ evidence_ref="submit_turn_on_request_output",
+ evidence_source="independent_page_evidence",
+ ),
+ ],
+ )
+
+ result = combine_verification_results(["c1", "c1__requested_output_corroborator"], run_result, [])
+
+ corroborator = _verdict_for(result, "c1__requested_output_corroborator")
+ assert corroborator.state == "satisfied"
+ assert corroborator.reason_code == "evidence_confirms"
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index e771443ae..624bbe8bb 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -9,6 +9,7 @@ import pytest
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSet,
StoredCriteriaSnapshot,
+ _criterion_reconcile_key,
criteria_from_json,
criteria_to_json,
reconcile_completion_criteria,
@@ -18,7 +19,9 @@ from skyvern.forge.sdk.copilot.request_policy import (
CompletionCriterion,
RequestPolicy,
_apply_requested_output_completion_criteria,
+ _apply_validation_classification_completion_criteria,
_classify_request,
+ _criterion_grounding_mode,
_parse_completion_criteria,
_render_active_criteria_for_prompt,
build_classifier_fallback_floor,
@@ -1680,3 +1683,202 @@ def test_requested_output_criteria_can_exceed_cap_without_dropping_requested_fie
"output.specialty",
"output.date",
}
+
+
+def test_parser_preserves_boolean_expected_value_and_promotes_independent_source() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record selects the highest-priority document.",
+ "output_path": "output.selected_highest_priority",
+ "expected_output_value": True,
+ }
+ ]
+ )
+
+ assert len(criteria) == 1
+ assert criteria[0].expected_output_value is True
+ assert criteria[0].requested_output_evidence_source == "independent_run_evidence"
+ assert _criterion_grounding_mode(criteria[0]) == "judgment_boolean"
+
+
+def test_parser_promotes_goal_judgment_boolean_shape_without_explicit_value() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record reports whether the highest-priority item was selected.",
+ "output_path": "output.selected_highest_priority",
+ "expected_output_shape": "goal_judgment_boolean",
+ }
+ ]
+ )
+
+ assert criteria[0].expected_output_shape == "goal_judgment_boolean"
+ assert criteria[0].requested_output_evidence_source == "independent_run_evidence"
+ assert _criterion_grounding_mode(criteria[0]) == "judgment_boolean"
+
+
+def test_parser_dedup_keeps_runtime_string_and_judgment_boolean_on_same_path() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record includes the selection label.",
+ "output_path": "output.selection",
+ "expected_output_value": "true",
+ },
+ {
+ "outcome": "The returned record reports the selection judgment.",
+ "output_path": "output.selection",
+ "expected_output_value": True,
+ },
+ ]
+ )
+
+ assert len(criteria) == 2
+ modes = {_criterion_grounding_mode(criterion) for criterion in criteria}
+ assert modes == {"exact_value", "judgment_boolean"}
+
+
+def test_declared_judgment_coerces_stringy_boolean_to_typed_bool() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record reports the highest-priority selection judgment.",
+ "output_path": "output.selected_highest_priority",
+ "expected_output_value": "true",
+ "requested_output_evidence_source": "independent_run_evidence",
+ }
+ ]
+ )
+
+ assert criteria[0].expected_output_value is True
+ assert _criterion_grounding_mode(criteria[0]) == "judgment_boolean"
+
+
+def test_bare_stringy_boolean_without_judgment_signal_stays_exact_value() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record includes the selection label.",
+ "output_path": "output.selection",
+ "expected_output_value": "true",
+ }
+ ]
+ )
+
+ assert criteria[0].expected_output_value == "true"
+ assert _criterion_grounding_mode(criteria[0]) == "exact_value"
+
+
+def test_store_round_trip_preserves_boolean_expected_value_and_source() -> None:
+ criteria = _parse_completion_criteria(
+ [
+ {
+ "outcome": "The returned record reports the false judgment.",
+ "output_path": "output.is_duplicate",
+ "expected_output_value": False,
+ }
+ ]
+ )
+
+ round_tripped = criteria_from_json(criteria_to_json(list(criteria)))
+
+ assert round_tripped[0].expected_output_value is False
+ assert round_tripped[0].requested_output_evidence_source == "independent_run_evidence"
+ assert _criterion_grounding_mode(round_tripped[0]) == "judgment_boolean"
+
+
+def test_expected_false_survives_active_criteria_rendering() -> None:
+ criterion = _criterion(
+ "c_bool",
+ "The returned record reports the false judgment.",
+ output_path="output.is_duplicate",
+ expected_output_value=False,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+
+ rendered = json.loads(_render_active_criteria_for_prompt([criterion]))
+
+ assert rendered[0]["expected_output_value"] is False
+
+
+def test_reconcile_key_distinguishes_false_from_absent_and_string_false() -> None:
+ false_bool = _criterion(
+ "c_false_bool",
+ "outcome",
+ output_path="output.flag",
+ expected_output_value=False,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ absent = _criterion("c_absent", "outcome", output_path="output.flag")
+ string_false = _criterion(
+ "c_string_false",
+ "outcome",
+ output_path="output.flag",
+ expected_output_value="false",
+ )
+
+ keys = {
+ _criterion_reconcile_key(false_bool),
+ _criterion_reconcile_key(absent),
+ _criterion_reconcile_key(string_false),
+ }
+ assert len(keys) == 3
+
+
+def test_criteria_from_json_coerces_stored_stringy_judgment_boolean_to_typed_bool() -> None:
+ raw_outcome = "The returned record reports the highest-priority selection judgment."
+ raw_criterion = {
+ "outcome": raw_outcome,
+ "output_path": "output.selected_highest_priority",
+ "expected_output_value": "true",
+ "requested_output_evidence_source": "independent_run_evidence",
+ }
+ restored = criteria_from_json([{"id": "c0", **raw_criterion}])
+ parsed = _parse_completion_criteria([raw_criterion])
+
+ assert restored[0].expected_output_value is True
+ assert _criterion_reconcile_key(restored[0]) == _criterion_reconcile_key(parsed[0])
+
+
+def test_scope_boundary_boolean_on_login_only_rekinds_and_drops_judgment_invariant() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_login_only",
+ "The workflow only reaches the login page.",
+ output_path="output.login_only",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+ )
+
+ _apply_validation_classification_completion_criteria(policy)
+
+ promoted = policy.completion_criteria[0]
+ assert promoted.kind == "validation_classification"
+ assert promoted.classification_output_key == "login_only"
+ assert promoted.expected_output_value is None
+ assert promoted.requested_output_evidence_source == "runtime_output"
+
+
+def test_scope_boundary_boolean_on_non_enumerated_path_keeps_judgment_invariant() -> None:
+ policy = RequestPolicy(
+ completion_criteria=[
+ _criterion(
+ "c_judgment",
+ "The returned record reports the selection judgment.",
+ output_path="output.selected_highest_priority",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+ )
+
+ _apply_validation_classification_completion_criteria(policy)
+
+ kept = policy.completion_criteria[0]
+ assert kept.kind == "outcome"
+ assert kept.expected_output_value is True
+ assert kept.requested_output_evidence_source == "independent_run_evidence"
diff --git a/tests/unit/test_copilot_request_policy_prompt.py b/tests/unit/test_copilot_request_policy_prompt.py
index 73c321af5..05f283ec1 100644
--- a/tests/unit/test_copilot_request_policy_prompt.py
+++ b/tests/unit/test_copilot_request_policy_prompt.py
@@ -94,7 +94,11 @@ class TestRequestPolicyPromptStructure:
"classification_output_key, expected_classification}"
) in rendered
assert "never hide it in outcome prose" in rendered
- assert "reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label" in rendered
+ assert (
+ "reference_code, numeric_identifier, date, address, status_label, money_amount, owner_label, "
+ "goal_judgment_boolean" in rendered
+ )
+ assert "goal_judgment_boolean declares that the field is an artifact-computed true/false judgment" in rendered
assert "kind=outcome|terminal_action|validation_classification" in rendered
assert "terminal_action_family=request|application|form|order|null" in rendered
assert (