Skip to content

Automate browser based workflows with AI

SKY-12733 Stop correct “no” answers from failing completed workflows (#7688)

Andrew Neilsond0fe4ddcaf90
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2
index dc4a06849..6c9ae5a98 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-completion-verification.j2
@@ -10,7 +10,7 @@ Rules:
- If you cannot tell from the evidence, set satisfied=false and reason_code=unknown.
- A criterion may include [contingent_on=...] and [contingent_antecedent_output_path=output.<field>]. The server handles whether the antecedent fired from structured run output; you must grade the listed outcome normally against the produced evidence. Use evidence_confirms only when the consequent is shown, no_evidence when the required blocker/report evidence is missing, and evidence_contradicts when the evidence refutes it.
- For an implicit constraint (e.g. "added exactly once"), check the end-state value (e.g. a cart quantity), not whether an add step ran.
-- evidence_ref: a short pointer to the specific evidence you used (a block label or the observed URL), or null.
+- evidence_ref: when satisfied=true, use an exact evidence label shown above, optionally followed by a dotted/indexed path into that record (for example `confirm_cart.items[0].quantity` or `observed_end_state_url`). Never return a literal URL. If a structured record is truncated so its nested path is not completely visible, cite the record label without a path. Use null only when satisfied=false.
reason_code is one of: evidence_confirms (satisfied=true), no_evidence, evidence_contradicts, unknown.
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
index 5fac9048d..cf27dc61e 100644
--- a/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-request-slots.j2
@@ -27,6 +27,7 @@ Rules:
- Include every request-owned output member exactly once. A conditional fallback is still an output member even when another classifier might omit its condition. Do not emit paths, path segments, IDs, aliases, labels, counts, expected values, explanations, or unknown fields.
- Do not infer slots from sampled page shapes or non-user context. Do not normalize, rename, or paraphrase source quotes.
- Return datum_bindings as an array. When DATUM TARGETS is empty, return an empty array. Otherwise answer every target exactly once and no others, identifying each target only by its criterion_index and datum_field. When user text names the datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":false,"source_id":<source ID>,"source_quote":<smallest unique exact quote>} whose independently declared slot names that target datum. Every accepted binding must name one exact slot in your slots array, and two targets may not bind the same slot. When no source quote actually names a target datum, return exactly {"criterion_index":<target integer>,"datum_field":<target field>,"declined":true}. The server owns datum_value and criterion_outcome_sha256; never return either field. Decline instead of guessing or borrowing an adjacent datum.
+- When a target's datum_value leaf directly matches a `whether ...` output phrase (underscores are spaces; articles and forms of "be" are insignificant), declare and bind that exact full `whether ...` phrase. Never bind the target to an earlier, broader task goal or evidence-gathering instruction.
Ordered, deduplicated user request sources:
```{{ request_sources }}```
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 50aa2f01a..77dc75374 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -8,7 +8,7 @@ stays at the route/repository seam; everything here is side-effect free.
from __future__ import annotations
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field, replace
from typing import Any, Literal, cast, get_args
@@ -45,8 +45,10 @@ from skyvern.forge.sdk.copilot.request_policy import (
_normalize_deliverable_kind,
is_defer_authoring_durable_fill_criterion,
is_fallback_floor_criterion,
+ is_neutral_reported_boolean_criterion,
is_presence_only_requested_output_criterion,
judgment_truth_condition_key,
+ normalize_neutral_reported_boolean_criterion,
normalized_criterion_outcome_key,
requested_output_path_for_field,
resolve_mint_degrade,
@@ -153,6 +155,7 @@ class PersistencePlan:
def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[CompletionCriterion]) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for criterion in criteria:
+ criterion = normalize_neutral_reported_boolean_criterion(criterion)
item = {
"id": criterion.id,
"outcome": criterion.outcome,
@@ -204,12 +207,17 @@ def criteria_to_json(criteria: tuple[CompletionCriterion, ...] | list[Completion
return items
-def criterion_authority_projection(criterion: CompletionCriterion) -> dict[str, Any]:
+def criterion_authority_projection(
+ criterion: CompletionCriterion,
+ *,
+ stored_item: Mapping[str, object] | None = None,
+) -> dict[str, Any]:
"""Canonical values for persisted fields that can change grading authority.
Repository admission compares only fields present in a stored row, preserving
omitted legacy defaults while rejecting values the tolerant decoder normalized.
"""
+ criterion = normalize_neutral_reported_boolean_criterion(criterion)
item = criteria_to_json([criterion])[0]
floor_rekeyed, floor_rekeyed_from_path = _normalize_floor_rekeyed_association(
criterion.requested_output_floor_rekeyed,
@@ -236,6 +244,17 @@ def criterion_authority_projection(criterion: CompletionCriterion) -> dict[str,
"mint_disposition": criterion.mint_disposition,
}
)
+ if (
+ stored_item is not None
+ and is_neutral_reported_boolean_criterion(criterion)
+ and stored_item.get("requested_output_floor_rekeyed") is True
+ and stored_item.get("floor_rekeyed_from_path") == f"output.{criterion.classification_output_key}"
+ ):
+ # The one historical coherent neutral tuple normalizes to the current unmarked tuple.
+ # Preserve its exact marker pair only for stored-authority comparison so malformed
+ # variants remain non-adoptable instead of being blessed by tolerant decoding.
+ item["requested_output_floor_rekeyed"] = True
+ item["floor_rekeyed_from_path"] = stored_item["floor_rekeyed_from_path"]
return item
@@ -347,49 +366,47 @@ def criteria_from_json(raw: Any) -> tuple[CompletionCriterion, ...]:
or stored_method_mandated
):
deliverable_confirmation_criterion_id = None
- criteria.append(
- CompletionCriterion(
- id=criterion_id,
- outcome=outcome,
- contingent_on=contingent_on,
- contingent_antecedent_output_path=contingent_antecedent_output_path,
- antecedent_family=antecedent_family,
- deliverable_kind=stored_deliverable_kind,
- deliverable_confirmation_criterion_id=deliverable_confirmation_criterion_id,
- declared_deliverable_kind=_normalize_deliverable_kind(item.get("declared_deliverable_kind")),
- implicit=bool(item.get("implicit")),
- method_mandated=stored_method_mandated,
- level=stored_level, # type: ignore[arg-type]
- output_path=stored_output_path,
- expected_output_value=stored_expected_output_value,
- expected_output_shape=stored_expected_output_shape,
- requested_output_evidence_source=cast(RequestedOutputEvidenceSource, requested_output_evidence_source),
- requested_output_path_mint_source=requested_output_path_mint_source,
- kind=cast(CriterionKind, kind),
- terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),
- terminal_action_verification_mode=cast(
- TerminalActionVerificationMode, terminal_action_verification_mode
- ),
- classification_output_key=classification_output_key,
- expected_classification=expected_classification,
- requested_output_corroborator=bool(item.get("requested_output_corroborator")),
- requested_output_floor_rekeyed=requested_output_floor_rekeyed,
- floor_rekeyed_from_path=floor_rekeyed_from_path,
- mint_degrade=resolve_mint_degrade(
- item.get("mint_degrade"), contingent_on, contingent_antecedent_output_path
- ),
- judgment_truth_condition=_coerce_judgment_truth_condition(
- item.get("judgment_predicate"), item.get("judgment_polarity_when_holds")
- ),
- request_slot_id=request_slot_id,
- pinability=cast(Pinability | None, pinability),
- mint_disposition=cast(MintDisposition, mint_disposition),
- )
+ criterion = CompletionCriterion(
+ id=criterion_id,
+ outcome=outcome,
+ contingent_on=contingent_on,
+ contingent_antecedent_output_path=contingent_antecedent_output_path,
+ antecedent_family=antecedent_family,
+ deliverable_kind=stored_deliverable_kind,
+ deliverable_confirmation_criterion_id=deliverable_confirmation_criterion_id,
+ declared_deliverable_kind=_normalize_deliverable_kind(item.get("declared_deliverable_kind")),
+ implicit=bool(item.get("implicit")),
+ method_mandated=stored_method_mandated,
+ level=stored_level, # type: ignore[arg-type]
+ output_path=stored_output_path,
+ expected_output_value=stored_expected_output_value,
+ expected_output_shape=stored_expected_output_shape,
+ requested_output_evidence_source=cast(RequestedOutputEvidenceSource, requested_output_evidence_source),
+ requested_output_path_mint_source=requested_output_path_mint_source,
+ kind=cast(CriterionKind, kind),
+ terminal_action_family=cast(TerminalActionFamily | None, terminal_action_family),
+ terminal_action_verification_mode=cast(TerminalActionVerificationMode, terminal_action_verification_mode),
+ classification_output_key=classification_output_key,
+ expected_classification=expected_classification,
+ requested_output_corroborator=bool(item.get("requested_output_corroborator")),
+ requested_output_floor_rekeyed=requested_output_floor_rekeyed,
+ floor_rekeyed_from_path=floor_rekeyed_from_path,
+ mint_degrade=resolve_mint_degrade(
+ item.get("mint_degrade"), contingent_on, contingent_antecedent_output_path
+ ),
+ judgment_truth_condition=_coerce_judgment_truth_condition(
+ item.get("judgment_predicate"), item.get("judgment_polarity_when_holds")
+ ),
+ request_slot_id=request_slot_id,
+ pinability=cast(Pinability | None, pinability),
+ mint_disposition=cast(MintDisposition, mint_disposition),
)
+ criteria.append(normalize_neutral_reported_boolean_criterion(criterion))
return tuple(criteria)
def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
+ criterion = normalize_neutral_reported_boolean_criterion(criterion)
contingent_key = criterion.contingent_on or ""
contingent_path_key = criterion.contingent_antecedent_output_path or ""
antecedent_family_key = criterion.antecedent_family or ""
@@ -415,6 +432,13 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
floor_rekeyed_key = (
f"\x1ffloor_rekeyed:{str(floor_rekeyed).lower()}\x1ffloor_rekeyed_from_path:{floor_rekeyed_from_path or ''}"
)
+ neutral_reported_boolean_key = ""
+ if is_neutral_reported_boolean_criterion(criterion):
+ neutral_reported_boolean_key = (
+ f"\x1fneutral_reported_boolean_key:{criterion.classification_output_key}"
+ f"\x1fneutral_expected_output_shape:{criterion.expected_output_shape}"
+ f"\x1fneutral_evidence_source:{criterion.requested_output_evidence_source}"
+ )
if criterion.output_path:
return (
f"contingent:{contingent_key}\x1fantecedent_path:{contingent_path_key}"
@@ -447,6 +471,7 @@ def _criterion_reconcile_key(criterion: CompletionCriterion) -> str:
f"\x1fkind:{criterion.kind}"
f"\x1fterminal_action_verification_mode:{terminal_action_verification_mode_key}"
f"\x1foutcome:{normalized_criterion_outcome_key(criterion.outcome)}"
+ f"{neutral_reported_boolean_key}"
f"{floor_rekeyed_key}"
)
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 78c207dab..4cb39ad9f 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
is_contingent_missing_antecedent_degraded,
is_fallback_floor_base_criterion,
is_judgment_finalization_candidate,
+ is_neutral_reported_boolean_criterion,
is_turn_unsatisfiable_fallback_degraded,
redact_raw_secrets_for_prompt,
)
@@ -386,6 +387,9 @@ class RunEvidenceSnapshot:
registered_blocker_evidence_by_request_slot_id: dict[str, tuple[RegisteredBlockerEvidence, ...]] = field(
default_factory=dict
)
+ registered_output_evidence_by_request_slot_id: dict[str, tuple[RegisteredBlockerEvidence, ...]] = field(
+ default_factory=dict
+ )
current_url: str | None = None
page_title: str | None = None
run_terminal_status: str | None = None
@@ -436,8 +440,7 @@ class RunEvidenceSnapshot:
if self.block_outputs:
lines.append("produced_block_outputs:")
for label, payload in list(self.block_outputs.items())[:_MAX_BLOCK_OUTPUTS]:
- serialized = payload if isinstance(payload, str) else json.dumps(payload, default=str)
- serialized = " ".join(serialized.split())[:_EVIDENCE_VALUE_MAX_CHARS]
+ serialized = _render_evidence_value(payload)
source = self.block_output_sources.get(label)
source_detail = f" [{source}]" if source else ""
lines.append(f" - {label}{source_detail}: {serialized}")
@@ -447,11 +450,95 @@ class RunEvidenceSnapshot:
if key in self.page_evidence and self.page_evidence[key] not in (None, "", [], {})
}
if page_evidence:
- serialized = json.dumps(page_evidence, default=str)
- serialized = " ".join(serialized.split())[:_EVIDENCE_VALUE_MAX_CHARS]
- lines.append(f"page_evidence: {serialized}")
+ lines.append(f"page_evidence: {_render_evidence_value(page_evidence)}")
return redact_raw_secrets_for_prompt("\n".join(lines))
+ def rendered_evidence_catalog(self) -> dict[str, RenderedEvidenceRecord]:
+ catalog = {
+ label: _rendered_evidence_record(payload, source=self.block_output_sources.get(label))
+ for label, payload in list(self.block_outputs.items())[:_MAX_BLOCK_OUTPUTS]
+ }
+ if self.current_url:
+ _add_reserved_evidence_record(
+ catalog,
+ "observed_end_state_url",
+ _rendered_evidence_record(self.current_url),
+ )
+ if self.page_title:
+ _add_reserved_evidence_record(
+ catalog,
+ "observed_end_state_page_title",
+ _rendered_evidence_record(self.page_title),
+ )
+ scalar_and_list_records: tuple[tuple[str, Any], ...] = (
+ ("run_terminal_status", self.run_terminal_status),
+ ("verified_context_block_labels", self.verified_context_block_labels),
+ ("executed_block_labels", self.executed_block_labels),
+ ("failed_block_labels", self.failed_block_labels),
+ ("failure_classes", self.failure_classes),
+ ("failure_reasons", self.failure_reasons),
+ )
+ for label, value in scalar_and_list_records:
+ if value:
+ _add_reserved_evidence_record(catalog, label, _rendered_evidence_record(value))
+ rendered_page_evidence = {
+ key: self.page_evidence[key]
+ for key in _PAGE_EVIDENCE_KEYS
+ if key in self.page_evidence and self.page_evidence[key] not in (None, "", [], {})
+ }
+ if rendered_page_evidence:
+ _add_reserved_evidence_record(
+ catalog,
+ "page_evidence",
+ _rendered_evidence_record(
+ rendered_page_evidence,
+ source="independent_page_evidence",
+ ),
+ )
+ return catalog
+
+
+@dataclass(frozen=True)
+class RenderedEvidenceRecord:
+ value: Any
+ source: EvidenceSourceKind | None = None
+
+
+def _add_reserved_evidence_record(
+ catalog: dict[str, RenderedEvidenceRecord],
+ label: str,
+ record: RenderedEvidenceRecord,
+) -> None:
+ if label in catalog:
+ # A runtime output must never shadow, or be shadowed by, server-owned
+ # evidence. Remove the ambiguous label so any citation fails closed.
+ catalog.pop(label)
+ return
+ catalog[label] = record
+
+
+def _render_evidence_value(value: Any) -> str:
+ serialized = value if isinstance(value, str) else json.dumps(value, default=str)
+ bounded = " ".join(serialized.split())[:_EVIDENCE_VALUE_MAX_CHARS]
+ return redact_raw_secrets_for_prompt(bounded)
+
+
+def _rendered_evidence_record(
+ value: Any,
+ *,
+ source: EvidenceSourceKind | None = None,
+) -> RenderedEvidenceRecord:
+ rendered = _render_evidence_value(value)
+ rendered_value: Any = rendered
+ if not isinstance(value, str):
+ try:
+ rendered_value = json.loads(rendered)
+ except (TypeError, ValueError):
+ # A truncated or redaction-invalidated structure was not present as a complete
+ # judge-visible record. Keep its rendered text so nested citations fail closed.
+ pass
+ return RenderedEvidenceRecord(value=rendered_value, source=source)
+
_UNAVAILABLE = CompletionVerificationResult(status="unavailable")
_MISSING_VERDICT_EVIDENCE = "judge did not return a verdict for this criterion"
@@ -782,10 +869,87 @@ def _is_structural_no_blocker_marker(value: Any) -> bool:
def _evidence_source_from_ref(
- block_output_sources: Mapping[str, EvidenceSourceKind], evidence_ref: str | None
+ evidence_catalog: Mapping[str, RenderedEvidenceRecord], evidence_ref: str | None
) -> EvidenceSourceKind | None:
label = _evidence_ref_record_label(evidence_ref)
- return block_output_sources.get(label) if label else None
+ record = evidence_catalog.get(label) if label else None
+ return record.source if record else None
+
+
+def _evidence_ref_path(evidence_ref: str | None) -> tuple[str, tuple[str, ...]] | None:
+ label = _evidence_ref_record_label(evidence_ref)
+ if label is None or evidence_ref is None:
+ return None
+ ref = evidence_ref.strip().removeprefix("block_outputs:")
+ _label, separator, raw_path = ref.partition(".")
+ path = tuple(segment.strip() for segment in raw_path.split(".") if segment.strip()) if separator else ()
+ return label, path
+
+
+def _record_path_resolves(value: Any, path: tuple[str, ...]) -> bool:
+ if not path:
+ return True
+ segment, *remaining = path
+ match = re.fullmatch(r"([^\[\]]+)?(?:\[(\d+)\])?", segment)
+ if match is None or (match.group(1) is None and match.group(2) is None):
+ return False
+ normalized_segment, raw_index = match.groups()
+ if normalized_segment is None:
+ if not isinstance(value, list) or raw_index is None:
+ return False
+ index = int(raw_index)
+ return index < len(value) and _record_path_resolves(value[index], tuple(remaining))
+ if isinstance(value, Mapping):
+ if normalized_segment not in value:
+ return False
+ resolved_value = value[normalized_segment]
+ if raw_index is not None:
+ if not isinstance(resolved_value, list):
+ return False
+ index = int(raw_index)
+ if index >= len(resolved_value):
+ return False
+ resolved_value = resolved_value[index]
+ return _record_path_resolves(resolved_value, tuple(remaining))
+ if isinstance(value, list):
+ return any(_record_path_resolves(item, path) for item in value)
+ return False
+
+
+def _satisfying_evidence_is_admissible(
+ criterion: CompletionCriterion,
+ evidence_ref: str | None,
+ evidence_catalog: Mapping[str, RenderedEvidenceRecord],
+ registered_output_evidence_by_request_slot_id: Mapping[str, tuple[RegisteredBlockerEvidence, ...]],
+) -> bool:
+ ref_path = _evidence_ref_path(evidence_ref)
+ if ref_path is None:
+ return False
+ label, path = ref_path
+ record = evidence_catalog.get(label)
+ if record is None or not _record_path_resolves(record.value, path):
+ return False
+ if criterion.requested_output_evidence_source == "independent_run_evidence" and (
+ record.source not in _INDEPENDENT_REQUESTED_OUTPUT_CORROBORATOR_SOURCES
+ ):
+ return False
+ if not is_neutral_reported_boolean_criterion(criterion):
+ return True
+ output_key = criterion.classification_output_key
+ request_slot_id = criterion.request_slot_id
+ if output_key is None or request_slot_id is None:
+ return False
+ associated_records = registered_output_evidence_by_request_slot_id.get(request_slot_id, ())
+ if len(associated_records) != 1:
+ return False
+ associated_record = associated_records[0]
+ return (
+ record.source == "independent_page_evidence"
+ and label != associated_record.registered_output_key
+ and associated_record.registered_output_key in evidence_catalog
+ and associated_record.output_path == f"output.{output_key}"
+ and isinstance(associated_record.value, bool)
+ )
def _coerce_result(
@@ -796,7 +960,9 @@ def _coerce_result(
contingent_on_by_criterion_id: dict[str, str] | None = None,
contingent_antecedent_output_path_by_criterion_id: dict[str, str] | None = None,
structural_unfired_criterion_ids: Iterable[str] = (),
- block_output_sources: Mapping[str, EvidenceSourceKind] | None = None,
+ criteria_by_id: Mapping[str, CompletionCriterion] | None = None,
+ evidence_catalog: Mapping[str, RenderedEvidenceRecord] | None = None,
+ registered_output_evidence_by_request_slot_id: Mapping[str, tuple[RegisteredBlockerEvidence, ...]] | None = None,
) -> CompletionVerificationResult:
if isinstance(raw, bytes):
raw = raw.decode("utf-8", errors="replace")
@@ -828,6 +994,18 @@ def _coerce_result(
else:
state = "unknown"
evidence_ref = _clean_optional_text(item.get("evidence_ref"), max_chars=_EVIDENCE_REF_MAX_CHARS)
+ criterion = (criteria_by_id or {}).get(criterion_id)
+ if state == "satisfied":
+ admission_criterion = criterion or CompletionCriterion(id=criterion_id, outcome="")
+ if not _satisfying_evidence_is_admissible(
+ admission_criterion,
+ evidence_ref,
+ evidence_catalog or {},
+ registered_output_evidence_by_request_slot_id or {},
+ ):
+ state = "unsatisfied"
+ reason_code = "no_evidence"
+ evidence_ref = None
missing_evidence = None
if state != "satisfied":
missing_evidence = _clean_optional_text(
@@ -839,7 +1017,7 @@ def _coerce_result(
reason_code=reason_code,
evidence_ref=evidence_ref,
missing_evidence=missing_evidence,
- evidence_source=_evidence_source_from_ref(block_output_sources or {}, evidence_ref),
+ evidence_source=_evidence_source_from_ref(evidence_catalog or {}, evidence_ref),
)
verdicts = [by_id.get(criterion_id, _missing_verdict(criterion_id)) for criterion_id in criterion_ids]
@@ -1085,14 +1263,20 @@ def grade_present_value_criteria(
continue
# Every named literal must appear in a SINGLE block output: a partial match
# (e.g. a date present while the named total is not) is not the named outcome.
- match_label = next(
- (
- label
- for label, haystack in haystacks
- if all(_present_verbatim(literal, haystack) for literal in literals)
- ),
- None,
- )
+ matching_labels = [
+ label for label, haystack in haystacks if all(_present_verbatim(literal, haystack) for literal in literals)
+ ]
+ if is_neutral_reported_boolean_criterion(criterion):
+ match_label = next(
+ (
+ label
+ for label in matching_labels
+ if snapshot.block_output_sources.get(label) == "independent_page_evidence"
+ ),
+ None,
+ )
+ else:
+ match_label = matching_labels[0] if matching_labels else None
if match_label is not None:
verdicts.append(
CriterionVerdict(
@@ -3002,7 +3186,9 @@ async def evaluate_completion_criteria(
contingent_on_by_criterion_id=contingent_on_by_id,
contingent_antecedent_output_path_by_criterion_id=contingent_path_by_id,
structural_unfired_criterion_ids=structural_unfired_ids,
- block_output_sources=snapshot.block_output_sources,
+ criteria_by_id={criterion.id: criterion for criterion in criteria},
+ evidence_catalog=snapshot.rendered_evidence_catalog(),
+ registered_output_evidence_by_request_slot_id=snapshot.registered_output_evidence_by_request_slot_id,
)
return replace(
result,
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index abc4d83c2..dd44f85e4 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -210,6 +210,7 @@ _COMPLETION_CRITERION_EXPECTED_VALUE_MAX_CHARS = 500
_COMPLETION_CRITERION_CLASSIFICATION_TARGET_MAX_CHARS = 120
_CLASSIFICATION_OUTPUT_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
_CONTINGENT_ANTECEDENT_OUTPUT_PATH_RE = re.compile(r"^output\.[A-Za-z_][A-Za-z0-9_]*$")
+_REQUEST_SLOT_ID_RE = re.compile(r"^[0-9a-f]{64}$")
_REQUESTED_OUTPUT_CRITERION_ID_PREFIX = "__copilot_requested_output__"
_VALIDATION_CLASSIFICATION_BOOLEAN_OUTPUT_TARGETS: dict[str, tuple[str, bool]] = {
"output.login_only": ("login_only", True),
@@ -594,6 +595,24 @@ class RequestPolicy:
prefix = f"mint_degraded_criterion_{index}"
data[f"{prefix}_id"] = criterion.id
data[f"{prefix}_mint_degrade"] = criterion.mint_degrade
+ neutral_reported_booleans = [
+ criterion for criterion in self.completion_criteria if is_neutral_reported_boolean_criterion(criterion)
+ ]
+ data["neutral_reported_boolean_criterion_count"] = len(neutral_reported_booleans)
+ for index, criterion in enumerate(neutral_reported_booleans[:_MAX_TRACE_COMPLETION_CRITERIA]):
+ prefix = f"neutral_reported_boolean_criterion_{index}"
+ data[f"{prefix}_id"] = criterion.id
+ data[f"{prefix}_kind"] = criterion.kind
+ data[f"{prefix}_classification_output_key"] = criterion.classification_output_key
+ data[f"{prefix}_expected_output_shape"] = criterion.expected_output_shape
+ data[f"{prefix}_expected_output_value"] = criterion.expected_output_value
+ data[f"{prefix}_expected_classification"] = criterion.expected_classification
+ data[f"{prefix}_evidence_source"] = criterion.requested_output_evidence_source
+ data[f"{prefix}_request_slot_id"] = criterion.request_slot_id
+ data[f"{prefix}_pinability"] = criterion.pinability
+ data[f"{prefix}_mint_disposition"] = criterion.mint_disposition
+ data[f"{prefix}_requested_output_floor_rekeyed"] = criterion.requested_output_floor_rekeyed
+ data[f"{prefix}_floor_rekeyed_from_path"] = criterion.floor_rekeyed_from_path
return data
def prompt_summary(self) -> str:
@@ -1276,11 +1295,119 @@ def _request_slot_datum_binding_targets(
return tuple(targets)
+def _interrogative_slot_classification_output_key(
+ source_quote: str,
+ *,
+ preserve_path_subject: bool = False,
+) -> str | None:
+ tokens = _word_tokens(source_quote)
+ if not tokens or tokens[0] != "whether":
+ return None
+ tokens = tokens[1:]
+ while tokens and tokens[0] in {"a", "an", "the"}:
+ tokens = tokens[1:]
+ if tokens[:2] == ["path", "is"]:
+ tokens = [tokens[0], *tokens[2:]] if preserve_path_subject else tokens[2:]
+ if not tokens:
+ return None
+ output_key = "_".join(tokens)
+ return output_key if _CLASSIFICATION_OUTPUT_KEY_RE.fullmatch(output_key) is not None else None
+
+
+def _omitted_request_slot_datum_binding_targets(
+ raw_criteria: Any,
+ *,
+ request_slot_request: RequestSlotProducerInputV1,
+ request_slot_contract: RequestSlotContractV1,
+ existing_targets: tuple[RequestSlotDatumBindingTargetV1, ...],
+) -> tuple[RequestSlotDatumBindingTargetV1, ...]:
+ represented_slot_ids = {binding.slot_id for binding in request_slot_contract.datum_bindings}
+ if isinstance(raw_criteria, list):
+ for item in raw_criteria:
+ if not isinstance(item, dict) or not _request_slot_anchor_is_valid(
+ item,
+ request_slot_request=request_slot_request,
+ ):
+ continue
+ slot = _request_slot_for_anchor(
+ _request_slot_anchor(item),
+ request_slot_request=request_slot_request,
+ request_slot_contract=request_slot_contract,
+ )
+ if slot is not None:
+ represented_slot_ids.add(slot.slot_id)
+ represented_output_keys = {target.datum_value for target in existing_targets}
+ next_criterion_index = len(raw_criteria) if isinstance(raw_criteria, list) else 0
+ targets: list[RequestSlotDatumBindingTargetV1] = []
+ for slot in request_slot_contract.slots:
+ if (
+ slot.slot_id in represented_slot_ids
+ or slot.plane.value != "run"
+ or slot.pinability.value != "shapeless_valid"
+ ):
+ continue
+ source_quote = request_slot_source_text(request_slot_request, slot)
+ output_key = _interrogative_slot_classification_output_key(source_quote)
+ if output_key is None or output_key in represented_output_keys:
+ continue
+ outcome = f"The run reports whether {output_key.replace('_', ' ')}."
+ targets.append(
+ RequestSlotDatumBindingTargetV1(
+ criterion_index=next_criterion_index,
+ datum_field="classification_output_key",
+ datum_value=output_key,
+ criterion_outcome_sha256=hashlib.sha256(outcome.encode()).hexdigest(),
+ )
+ )
+ represented_output_keys.add(output_key)
+ next_criterion_index += 1
+ return tuple(targets)
+
+
+def _trusted_omitted_request_slot_datum_bindings(
+ *,
+ request_slot_request: RequestSlotProducerInputV1,
+ request_slot_contract: RequestSlotContractV1,
+ omitted_targets: tuple[RequestSlotDatumBindingTargetV1, ...],
+) -> tuple[TrustedRequestSlotDatumBindingV1, ...]:
+ target_by_key = {(target.criterion_index, target.datum_field): target for target in omitted_targets}
+ trusted: list[TrustedRequestSlotDatumBindingV1] = []
+ for binding in request_slot_contract.datum_bindings:
+ target = target_by_key.get((binding.criterion_index, binding.datum_field))
+ if (
+ target is None
+ or binding.datum_value != target.datum_value
+ or binding.criterion_outcome_sha256 != target.criterion_outcome_sha256
+ ):
+ continue
+ resolved_slot = _request_slot_for_anchor(
+ (binding.source_id, binding.source_quote),
+ request_slot_request=request_slot_request,
+ request_slot_contract=request_slot_contract,
+ )
+ if resolved_slot is None or binding.slot_id != resolved_slot.slot_id:
+ continue
+ trusted.append(
+ TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ source_id=binding.source_id,
+ source_quote=binding.source_quote,
+ slot_id=binding.slot_id,
+ )
+ )
+ return tuple(trusted)
+
+
def _apply_request_slot_datum_bindings(
original: dict[str, Any],
*,
request_slot_request: RequestSlotProducerInputV1,
request_slot_contract: RequestSlotContractV1,
+ datum_binding_targets: tuple[RequestSlotDatumBindingTargetV1, ...] | None = None,
) -> RequestSlotDatumBindingApplicationDecisionV1:
criteria = original.get("completion_criteria")
if not isinstance(criteria, list):
@@ -1288,22 +1415,36 @@ def _apply_request_slot_datum_bindings(
if request_slot_contract.request_digest != request_slot_request_digest(request_slot_request):
return RequestSlotDatumBindingApplicationDecisionV1(version="1", predicate="invalid_contract")
- targets = _request_slot_datum_binding_targets(
+ targets = datum_binding_targets or _request_slot_datum_binding_targets(
criteria,
request_slot_request=request_slot_request,
)
target_by_key = {(target.criterion_index, target.datum_field): target for target in targets}
- binding_by_key = {
+ all_binding_by_key = {
(binding.criterion_index, binding.datum_field): binding for binding in request_slot_contract.datum_bindings
}
- decline_by_key = {
+ all_decline_by_key = {
(decline.criterion_index, decline.datum_field): decline for decline in request_slot_contract.datum_declines
}
+ request_target_by_key = {
+ (target.criterion_index, target.datum_field): target for target in request_slot_request.datum_targets
+ }
+ binding_by_key = {key: binding for key, binding in all_binding_by_key.items() if key in target_by_key}
+ decline_by_key = {key: decline for key, decline in all_decline_by_key.items() if key in target_by_key}
if (
- len(binding_by_key) != len(request_slot_contract.datum_bindings)
- or len(decline_by_key) != len(request_slot_contract.datum_declines)
- or set(binding_by_key) & set(decline_by_key)
+ len(all_binding_by_key) != len(request_slot_contract.datum_bindings)
+ or len(all_decline_by_key) != len(request_slot_contract.datum_declines)
+ or set(all_binding_by_key) & set(all_decline_by_key)
+ or set(all_binding_by_key) | set(all_decline_by_key) != set(request_target_by_key)
or set(binding_by_key) | set(decline_by_key) != set(target_by_key)
+ or any(
+ (request_target := request_target_by_key.get(key)) is None
+ or request_target.criterion_index != target.criterion_index
+ or request_target.datum_field != target.datum_field
+ or request_target.datum_value != target.datum_value
+ or request_target.criterion_outcome_sha256 != target.criterion_outcome_sha256
+ for key, target in target_by_key.items()
+ )
):
return RequestSlotDatumBindingApplicationDecisionV1(version="1", predicate="invalid_contract")
@@ -1599,6 +1740,115 @@ def _request_slot_has_exact_requirement(criterion: CompletionCriterion) -> bool:
return criterion.expected_output_value is not None or criterion.expected_output_shape is not None
+def _top_level_output_key(output_path: str | None) -> str | None:
+ if output_path is None or not output_path.startswith("output."):
+ return None
+ key = output_path.removeprefix("output.")
+ return key if _CLASSIFICATION_OUTPUT_KEY_RE.fullmatch(key) else None
+
+
+def _boolean_output_signal_count(criterion: CompletionCriterion) -> int:
+ return sum(
+ (
+ isinstance(criterion.expected_output_value, bool),
+ criterion.expected_output_shape == "goal_judgment_boolean",
+ isinstance(criterion.expected_classification, bool),
+ )
+ )
+
+
+def _has_boolean_output_signal(criterion: CompletionCriterion) -> bool:
+ return _boolean_output_signal_count(criterion) > 0
+
+
+def _shapeless_boolean_output_binding_key(criterion: CompletionCriterion) -> str | None:
+ if _boolean_output_signal_count(criterion) != 1:
+ return None
+ bindings: list[str] = []
+ if criterion.output_path is not None:
+ output_key = _top_level_output_key(criterion.output_path)
+ if output_key is None:
+ return None
+ bindings.append(output_key)
+ if criterion.classification_output_key is not None:
+ if _CLASSIFICATION_OUTPUT_KEY_RE.fullmatch(criterion.classification_output_key) is None:
+ return None
+ bindings.append(criterion.classification_output_key)
+ return bindings[0] if len(bindings) == 1 else None
+
+
+def is_neutral_reported_boolean_criterion(criterion: CompletionCriterion) -> bool:
+ output_key = criterion.classification_output_key
+ return (
+ criterion.kind == "outcome"
+ and criterion.output_path is None
+ and criterion.expected_output_value is None
+ and criterion.expected_classification is None
+ and criterion.expected_output_shape == "goal_judgment_boolean"
+ and criterion.requested_output_evidence_source == "independent_run_evidence"
+ and output_key is not None
+ and criterion.requested_output_floor_rekeyed is False
+ and criterion.floor_rekeyed_from_path is None
+ and criterion.request_slot_id is not None
+ and _REQUEST_SLOT_ID_RE.fullmatch(criterion.request_slot_id) is not None
+ and criterion.id == criterion.request_slot_id
+ and _CLASSIFICATION_OUTPUT_KEY_RE.fullmatch(output_key) is not None
+ and criterion.pinability == "shapeless_valid"
+ and criterion.mint_disposition == "decidable"
+ and criterion.mint_degrade is None
+ and criterion.judgment_truth_condition is None
+ and criterion.antecedent_family in {"unconditional", "blocker"}
+ )
+
+
+def _is_legacy_floor_marked_neutral_reported_boolean_criterion(criterion: CompletionCriterion) -> bool:
+ if not (
+ criterion.requested_output_floor_rekeyed
+ and criterion.classification_output_key is not None
+ and criterion.floor_rekeyed_from_path == f"output.{criterion.classification_output_key}"
+ ):
+ return False
+ return is_neutral_reported_boolean_criterion(
+ replace(
+ criterion,
+ requested_output_floor_rekeyed=False,
+ floor_rekeyed_from_path=None,
+ )
+ )
+
+
+def normalize_neutral_reported_boolean_criterion(criterion: CompletionCriterion) -> CompletionCriterion:
+ claims_neutral_reported_boolean = criterion.kind == "outcome" and (
+ criterion.classification_output_key is not None
+ or (criterion.pinability == "shapeless_valid" and _has_boolean_output_signal(criterion))
+ )
+ if not claims_neutral_reported_boolean:
+ return criterion
+ if is_neutral_reported_boolean_criterion(criterion):
+ return criterion
+ if _is_legacy_floor_marked_neutral_reported_boolean_criterion(criterion):
+ return replace(
+ criterion,
+ requested_output_floor_rekeyed=False,
+ floor_rekeyed_from_path=None,
+ )
+ return replace(
+ criterion,
+ output_path=None,
+ expected_output_value=None,
+ expected_output_shape=None,
+ requested_output_evidence_source="runtime_output",
+ classification_output_key=None,
+ expected_classification=None,
+ judgment_truth_condition=None,
+ antecedent_family="undecidable",
+ requested_output_floor_rekeyed=False,
+ floor_rekeyed_from_path=None,
+ mint_disposition="degraded",
+ mint_degrade="undecidable_judgment",
+ )
+
+
def _degrade_unbound_request_slot_criterion(criterion: CompletionCriterion) -> CompletionCriterion:
return replace(
criterion,
@@ -1641,13 +1891,55 @@ def _request_slot_for_anchor(
return matches[0] if len(matches) == 1 else None
+def _omitted_request_slot_criterion(
+ *,
+ slot: CanonicalRequestSlotV1,
+ source_quote: str,
+ trusted_datum_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...],
+) -> CompletionCriterion:
+ criterion = CompletionCriterion(id="c0", outcome=source_quote)
+ if len(trusted_datum_bindings) != 1:
+ return criterion
+ binding = trusted_datum_bindings[0]
+ classification_output_key = _coerce_classification_output_key(binding.datum_value)
+ if not (
+ binding.slot_id == slot.slot_id
+ and binding.source_id == slot.source_id
+ and binding.source_quote == source_quote
+ and binding.datum_field == "classification_output_key"
+ and classification_output_key == binding.datum_value
+ ):
+ return criterion
+ return replace(
+ criterion,
+ kind="validation_classification",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ classification_output_key=classification_output_key,
+ mint_disposition="pending",
+ )
+
+
def _bind_criterion_to_request_slot(
criterion: CompletionCriterion,
*,
slot: CanonicalRequestSlotV1,
source_quote: str,
+ allow_canonical_path_fallback: bool = True,
) -> CompletionCriterion:
pinability = cast(Pinability, slot.pinability.value)
+ lexical_interrogative_key = _interrogative_slot_classification_output_key(
+ source_quote,
+ preserve_path_subject=True,
+ )
+ if (
+ pinability == "shapeless_valid"
+ and slot.plane.value == "run"
+ and lexical_interrogative_key is not None
+ and criterion.classification_output_key is not None
+ and lexical_interrogative_key == f"path_{criterion.classification_output_key}"
+ ):
+ criterion = replace(criterion, classification_output_key=lexical_interrogative_key)
has_exact_requirement = _request_slot_has_exact_requirement(criterion)
antecedent_family = cast(AntecedentFamily, slot.antecedent_family.value)
degraded = (
@@ -1655,10 +1947,30 @@ def _bind_criterion_to_request_slot(
or (pinability == "pinned" and not has_exact_requirement)
or antecedent_family == "undecidable"
)
+ neutral_boolean_candidate = (
+ slot.plane.value == "run"
+ and criterion.level == "run"
+ and pinability == "shapeless_valid"
+ and lexical_interrogative_key is not None
+ and not degraded
+ )
+ neutral_boolean_key = _shapeless_boolean_output_binding_key(criterion) if neutral_boolean_candidate else None
+ malformed_shapeless_boolean = (
+ neutral_boolean_candidate and _has_boolean_output_signal(criterion) and neutral_boolean_key is None
+ )
+ assertive_shapeless_boolean = (
+ slot.plane.value == "run"
+ and criterion.level == "run"
+ and pinability == "shapeless_valid"
+ and lexical_interrogative_key is None
+ and _has_boolean_output_signal(criterion)
+ )
+ degraded = degraded or malformed_shapeless_boolean or assertive_shapeless_boolean
rekeyed = degraded or pinability == "shapeless_valid"
expected_output_value = criterion.expected_output_value
expected_output_shape = criterion.expected_output_shape
+ requested_output_evidence_source = criterion.requested_output_evidence_source
expected_classification = criterion.expected_classification
kind = criterion.kind
classification_output_key = criterion.classification_output_key
@@ -1666,17 +1978,35 @@ def _bind_criterion_to_request_slot(
original_output_path = (
criterion.output_path
or (f"output.{classification_output_key}" if classification_output_key is not None else None)
- or slot.canonical_path
+ or (slot.canonical_path if allow_canonical_path_fallback else None)
)
+ if rekeyed and neutral_boolean_key is None and original_output_path is None:
+ degraded = True
+ antecedent_family = "undecidable"
if pinability != "pinned" or degraded:
expected_output_value = None
expected_output_shape = None
expected_classification = None
judgment_truth_condition = None
+ if malformed_shapeless_boolean:
+ requested_output_evidence_source = "runtime_output"
+ classification_output_key = None
+ antecedent_family = "undecidable"
+ if neutral_boolean_key is not None:
+ kind = "outcome"
+ outcome = f"The run reports whether {neutral_boolean_key.replace('_', ' ')}."
+ expected_output_shape = "goal_judgment_boolean"
+ requested_output_evidence_source = "independent_run_evidence"
+ classification_output_key = neutral_boolean_key
+ else:
+ outcome = criterion.outcome or source_quote
if rekeyed and kind == "validation_classification":
kind = "outcome"
classification_output_key = None
output_path = None if rekeyed else criterion.output_path
+ if neutral_boolean_key is not None:
+ original_output_path = f"output.{neutral_boolean_key}"
+ requested_output_floor_rekeyed = rekeyed and neutral_boolean_key is None and original_output_path is not None
pending = pinability == "pinned" and (
isinstance(expected_output_value, bool)
@@ -1702,11 +2032,12 @@ def _bind_criterion_to_request_slot(
return replace(
criterion,
id=slot.slot_id,
- outcome=criterion.outcome or source_quote,
+ outcome=outcome,
level=cast(CriterionLevel, slot.plane.value),
output_path=output_path,
expected_output_value=expected_output_value,
expected_output_shape=expected_output_shape,
+ requested_output_evidence_source=requested_output_evidence_source,
kind=kind,
classification_output_key=classification_output_key,
expected_classification=expected_classification,
@@ -1716,8 +2047,8 @@ def _bind_criterion_to_request_slot(
pinability=pinability,
mint_disposition="degraded" if degraded else ("pending" if pending else "decidable"),
mint_degrade="undecidable_judgment" if degraded else None,
- requested_output_floor_rekeyed=rekeyed,
- floor_rekeyed_from_path=original_output_path if rekeyed else None,
+ requested_output_floor_rekeyed=requested_output_floor_rekeyed,
+ floor_rekeyed_from_path=original_output_path if requested_output_floor_rekeyed else None,
)
@@ -1731,7 +2062,11 @@ def _parse_fresh_request_slot_criteria(
if request_slot_contract.version != "1" or not isinstance(raw, list):
return []
criterion_index_by_item_id = {id(item): index for index, item in enumerate(raw) if isinstance(item, dict)}
- trusted_binding_by_index = {binding.criterion_index: binding for binding in trusted_datum_bindings}
+ trusted_bindings_by_index: dict[int, list[TrustedRequestSlotDatumBindingV1]] = {}
+ trusted_bindings_by_slot_id: dict[str, list[TrustedRequestSlotDatumBindingV1]] = {}
+ for binding in trusted_datum_bindings:
+ trusted_bindings_by_index.setdefault(binding.criterion_index, []).append(binding)
+ trusted_bindings_by_slot_id.setdefault(binding.slot_id, []).append(binding)
bound_by_slot_id: dict[str, CompletionCriterion] = {}
non_slot_criteria: list[CompletionCriterion] = []
for item, criterion in _parse_completion_criterion_entries(raw):
@@ -1747,7 +2082,11 @@ def _parse_fresh_request_slot_criteria(
item,
request_slot_request=request_slot_request,
criterion_index=criterion_index,
- trusted_binding=trusted_binding_by_index.get(criterion_index),
+ trusted_binding=(
+ trusted_bindings_by_index[criterion_index][0]
+ if len(trusted_bindings_by_index.get(criterion_index, ())) == 1
+ else None
+ ),
allow_embedded_binding=False,
)
else None
@@ -1777,10 +2116,16 @@ def _parse_fresh_request_slot_criteria(
if slot.slot_id in bound_by_slot_id:
continue
source_quote = request_slot_source_text(request_slot_request, slot)
+ slot_bindings = tuple(trusted_bindings_by_slot_id.get(slot.slot_id, ()))
bound_by_slot_id[slot.slot_id] = _bind_criterion_to_request_slot(
- CompletionCriterion(id="c0", outcome=source_quote),
+ _omitted_request_slot_criterion(
+ slot=slot,
+ source_quote=source_quote,
+ trusted_datum_bindings=slot_bindings,
+ ),
slot=slot,
source_quote=source_quote,
+ allow_canonical_path_fallback=False,
)
return ([bound_by_slot_id[slot.slot_id] for slot in request_slot_contract.slots] + non_slot_criteria)[
:_MAX_COMPLETION_CRITERIA
@@ -3519,6 +3864,7 @@ async def _classify_request(
item.pop("request_slot_datum_binding", None)
request_slot_contract: RequestSlotContractV1 | None = None
+ request_slot_contract_request = request_slot_request
request_slot_failure_kind: str | None = None
trusted_datum_bindings: tuple[TrustedRequestSlotDatumBindingV1, ...] = ()
datum_binding_targets = _request_slot_datum_binding_targets(
@@ -3550,6 +3896,7 @@ async def _classify_request(
request_slot_result = await produce_request_slots(request=targeted_request, handler=handler)
if request_slot_result.status == "success":
request_slot_contract = request_slot_result.contract
+ request_slot_contract_request = targeted_request
else:
request_slot_failure_kind = (
request_slot_result.failure_kind.value
@@ -3566,6 +3913,7 @@ async def _classify_request(
raw_payload,
request_slot_request=targeted_request,
request_slot_contract=request_slot_contract,
+ datum_binding_targets=datum_binding_targets,
)
if datum_binding_decision.accepted_payload is not None:
raw_payload = datum_binding_decision.accepted_payload
@@ -3639,6 +3987,7 @@ async def _classify_request(
request_slot_result = await produce_request_slots(request=request_slot_request, handler=handler)
if request_slot_result.status == "success":
request_slot_contract = request_slot_result.contract
+ request_slot_contract_request = request_slot_request
else:
request_slot_failure_kind = (
request_slot_result.failure_kind.value if request_slot_result.failure_kind is not None else "unknown"
@@ -3648,9 +3997,66 @@ async def _classify_request(
failure_kind=request_slot_failure_kind,
attempts=request_slot_result.attempts,
)
+ if request_slot_failure_kind is None and request_slot_contract is not None:
+ omitted_targets = _omitted_request_slot_datum_binding_targets(
+ raw_criteria,
+ request_slot_request=request_slot_contract_request,
+ request_slot_contract=request_slot_contract,
+ existing_targets=datum_binding_targets,
+ )
+ if omitted_targets:
+ completed_request_data = request_slot_request.model_dump()
+ completed_request_data["datum_targets"] = tuple(
+ RequestSlotDatumTargetV1(
+ criterion_index=target.criterion_index,
+ datum_field=target.datum_field,
+ datum_value=target.datum_value,
+ criterion_outcome_sha256=target.criterion_outcome_sha256,
+ )
+ for target in (*datum_binding_targets, *omitted_targets)
+ )
+ try:
+ completed_request = RequestSlotProducerInputV1.model_validate(completed_request_data)
+ except ValueError:
+ completed_request = None
+ request_slot_failure_kind = "invalid_output"
+ request_slot_contract = None
+ if completed_request is not None:
+ completed_result = await produce_request_slots(request=completed_request, handler=handler)
+ if completed_result.status == "success" and completed_result.contract is not None:
+ request_slot_contract = completed_result.contract
+ request_slot_contract_request = completed_request
+ if datum_binding_targets:
+ datum_binding_decision = _apply_request_slot_datum_bindings(
+ raw_payload,
+ request_slot_request=completed_request,
+ request_slot_contract=request_slot_contract,
+ datum_binding_targets=datum_binding_targets,
+ )
+ if datum_binding_decision.accepted_payload is None:
+ request_slot_failure_kind = "invalid_anchor_correction"
+ request_slot_contract = None
+ else:
+ raw_payload = datum_binding_decision.accepted_payload
+ raw_criteria = raw_payload.get("completion_criteria")
+ trusted_datum_bindings = datum_binding_decision.trusted_bindings
+ if request_slot_contract is not None:
+ trusted_datum_bindings = (
+ *trusted_datum_bindings,
+ *_trusted_omitted_request_slot_datum_bindings(
+ request_slot_request=completed_request,
+ request_slot_contract=request_slot_contract,
+ omitted_targets=omitted_targets,
+ ),
+ )
+ else:
+ request_slot_failure_kind = (
+ completed_result.failure_kind.value if completed_result.failure_kind is not None else "unknown"
+ )
+ request_slot_contract = None
policy = _classification_from_raw(
raw_payload,
- request_slot_request=request_slot_request,
+ request_slot_request=request_slot_contract_request,
request_slot_contract=request_slot_contract,
request_slot_failure_kind=request_slot_failure_kind,
trusted_datum_bindings=trusted_datum_bindings,
diff --git a/skyvern/forge/sdk/copilot/request_slots.py b/skyvern/forge/sdk/copilot/request_slots.py
index e26a76e42..1f8dc5142 100644
--- a/skyvern/forge/sdk/copilot/request_slots.py
+++ b/skyvern/forge/sdk/copilot/request_slots.py
@@ -33,6 +33,14 @@ _MAX_CLASSIFIER_ATTEMPTS = 4
_SOURCE_ID_PATTERN = re.compile(r"^u(?:0|[1-9][0-9]?)$")
_PATH_SEGMENT_PATTERN = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
_REQUEST_SLOT_PATH_PATTERN = re.compile(r"^request_slot_[0-9a-f]{48}_[0-9]{2}$")
+_DIRECT_INTERROGATIVE_CLAUSE_PATTERN = re.compile(
+ r"\bwhether\b.*?(?=\s+(?:and|or)\s+whether\b|[,;.!?]|$)",
+ re.IGNORECASE,
+)
+_DIRECT_INTERROGATIVE_TRAILING_WRAPPERS = " \t\r\n)]}\"'"
+_DATUM_ASSOCIATION_FILLER_TOKENS = frozenset(
+ {"a", "an", "the", "am", "is", "are", "was", "were", "be", "been", "being"}
+)
class RequestSlotPlane(StrEnum):
@@ -408,6 +416,34 @@ def request_slot_source_text(request: RequestSlotProducerInputV1, slot: Canonica
return source.text[slot.source_start : slot.source_end]
+def _datum_association_tokens(value: str) -> tuple[str, ...]:
+ leaf = value.rsplit(".", 1)[-1]
+ return tuple(
+ token for token in re.findall(r"[a-z0-9]+", leaf.casefold()) if token not in _DATUM_ASSOCIATION_FILLER_TOKENS
+ )
+
+
+def _direct_interrogative_output_anchor(
+ sources: tuple[RequestSlotSourceV1, ...],
+ target: RequestSlotDatumTargetV1,
+) -> tuple[str, str] | None:
+ target_tokens = _datum_association_tokens(target.datum_value)
+ if not target_tokens:
+ return None
+ for source in reversed(sources):
+ matches: list[str] = []
+ for clause_match in _DIRECT_INTERROGATIVE_CLAUSE_PATTERN.finditer(source.text):
+ quote = clause_match.group(0).rstrip(_DIRECT_INTERROGATIVE_TRAILING_WRAPPERS)
+ clause_tokens = _datum_association_tokens(quote[len("whether") :])
+ if clause_tokens == target_tokens:
+ matches.append(quote)
+ if len(matches) > 1:
+ raise ValueError("direct interrogative output phrase must be unique in its request source")
+ if matches:
+ return source.source_id, matches[0]
+ return None
+
+
def canonicalize_request_slots(
*,
request: RequestSlotProducerInputV1,
@@ -481,7 +517,10 @@ def canonicalize_request_slots(
seen_binding_slot_ids: set[str] = set()
for target in request.datum_targets:
resolution = resolution_by_identity[(target.criterion_index, target.datum_field)]
+ direct_output_anchor = _direct_interrogative_output_anchor(sources, target)
if isinstance(resolution, RequestSlotDatumDeclineDeclarationV1):
+ if direct_output_anchor is not None:
+ raise ValueError("a directly named interrogative output datum cannot be declined")
canonical_declines.append(
CanonicalRequestSlotDatumDeclineV1(
criterion_index=target.criterion_index,
@@ -492,6 +531,15 @@ def canonicalize_request_slots(
)
continue
binding = resolution
+ if (
+ direct_output_anchor is not None
+ and (
+ binding.source_id,
+ binding.source_quote,
+ )
+ != direct_output_anchor
+ ):
+ raise ValueError("datum binding must use the exact direct interrogative output anchor")
source = source_by_id.get(binding.source_id)
if source is None:
raise ValueError(f"unknown datum-binding source: {binding.source_id}")
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 027edf7f3..c50d89c45 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
RunEvidenceSnapshot,
_contingent_metadata_for_criteria,
_is_structural_requested_output_abstention,
+ _satisfying_evidence_is_admissible,
carry_criterion_metadata,
carry_degraded_criterion_ids,
carry_floor_rekeyed_criterion_ids,
@@ -69,6 +70,7 @@ from skyvern.forge.sdk.copilot.request_policy import (
RequestedOutputPathMintSource,
_is_judgment_boolean_criterion,
is_fallback_floor_criterion,
+ is_neutral_reported_boolean_criterion,
)
from skyvern.forge.sdk.copilot.runtime import PreRunPageReference, RegisteredArtifactEvidence
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
@@ -314,23 +316,38 @@ def _exact_registered_output_path_value(value: object, output_path: str) -> tupl
return True, current
-def _registered_blocker_evidence_by_request_slot_id(
+def _registered_output_path_for_request_slot_criterion(criterion: CompletionCriterion) -> str | None:
+ if is_neutral_reported_boolean_criterion(criterion) and criterion.classification_output_key is not None:
+ return f"output.{criterion.classification_output_key}"
+ if criterion.antecedent_family == "blocker":
+ return criterion.output_path or criterion.floor_rekeyed_from_path
+ return None
+
+
+def _registered_output_identity_path(criterion: CompletionCriterion) -> str | None:
+ if is_neutral_reported_boolean_criterion(criterion) and criterion.classification_output_key is not None:
+ return f"output.{criterion.classification_output_key}"
+ return criterion.output_path or criterion.floor_rekeyed_from_path
+
+
+def _registered_output_evidence_by_request_slot_id(
copilot_ctx: Any,
run_data: Mapping[str, Any],
) -> dict[str, tuple[RegisteredBlockerEvidence, ...]]:
- """Bind blocker evidence through exact criterion -> artifact owner -> run row identity."""
+ """Bind request-slot evidence through exact criterion -> artifact owner -> run row identity."""
metadata = _accepted_staged_output_contract_metadata(copilot_ctx)
if not isinstance(metadata, Mapping):
return {}
formed_criteria = _formed_completion_criteria(copilot_ctx)
- blocker_criteria = [
+ request_slot_criteria = [
criterion
for criterion in formed_criteria
- if criterion.antecedent_family == "blocker" and criterion.request_slot_id is not None
+ if criterion.request_slot_id is not None
+ and _registered_output_path_for_request_slot_criterion(criterion) is not None
]
criterion_paths: dict[str, list[CompletionCriterion]] = {}
for criterion in formed_criteria:
- output_path = criterion.output_path or criterion.floor_rekeyed_from_path
+ output_path = _registered_output_identity_path(criterion)
if isinstance(output_path, str) and output_path.startswith("output."):
criterion_paths.setdefault(output_path, []).append(criterion)
@@ -347,9 +364,9 @@ def _registered_blocker_evidence_by_request_slot_id(
registered_rows = _registered_output_parameter_payloads(run_data)
candidates_by_slot_id: dict[str, list[RegisteredBlockerEvidence]] = {}
identity_owners: dict[tuple[str, str, str], list[str]] = {}
- for criterion in blocker_criteria:
+ for criterion in request_slot_criteria:
slot_id = criterion.request_slot_id
- output_path = criterion.output_path or criterion.floor_rekeyed_from_path
+ output_path = _registered_output_path_for_request_slot_criterion(criterion)
if slot_id is None or not isinstance(output_path, str) or len(criterion_paths.get(output_path, ())) != 1:
continue
metadata_labels = metadata_labels_by_path.get(output_path.removeprefix("output."), ())
@@ -394,6 +411,22 @@ def _registered_blocker_evidence_by_request_slot_id(
return associations
+def _registered_blocker_evidence_by_request_slot_id(
+ copilot_ctx: Any,
+ run_data: Mapping[str, Any],
+) -> dict[str, tuple[RegisteredBlockerEvidence, ...]]:
+ blocker_slot_ids = {
+ criterion.request_slot_id
+ for criterion in _formed_completion_criteria(copilot_ctx)
+ if criterion.antecedent_family == "blocker" and criterion.request_slot_id is not None
+ }
+ return {
+ slot_id: evidence
+ for slot_id, evidence in _registered_output_evidence_by_request_slot_id(copilot_ctx, run_data).items()
+ if slot_id in blocker_slot_ids
+ }
+
+
def _authored_output_contract_metadata_paths(metadata: object) -> set[str]:
paths: set[str] = set()
if isinstance(metadata, Mapping):
@@ -1205,6 +1238,7 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
registered_blocker_evidence_by_request_slot_id=_registered_blocker_evidence_by_request_slot_id(
copilot_ctx, data
),
+ registered_output_evidence_by_request_slot_id=_registered_output_evidence_by_request_slot_id(copilot_ctx, data),
current_url=_valid_runtime_anchor_url(data.get("current_url")),
page_title=page_title if isinstance(page_title, str) and page_title.strip() else None,
run_terminal_status=run_terminal_status
@@ -1277,13 +1311,33 @@ def _apply_present_value_upgrades(
if not upgrades and not semantic_verdicts:
return run_result
+ criteria_by_id = {criterion.id: criterion for criterion in run_criteria}
+ evidence_catalog = snapshot.rendered_evidence_catalog()
+
+ def _upgrade_is_admissible(upgrade: CriterionVerdict) -> bool:
+ criterion = criteria_by_id.get(upgrade.criterion_id)
+ if criterion is None or not is_neutral_reported_boolean_criterion(criterion):
+ return True
+ return _satisfying_evidence_is_admissible(
+ criterion,
+ upgrade.evidence_ref,
+ evidence_catalog,
+ snapshot.registered_output_evidence_by_request_slot_id,
+ )
+
def _merged(verdict: CriterionVerdict) -> CriterionVerdict:
# Semantic contradictions are stronger than earlier satisfied verdicts:
# a row-mixing/status contradiction means the record evidence is invalid.
if verdict.criterion_id in semantic_verdicts:
return semantic_verdicts[verdict.criterion_id]
- if verdict.criterion_id in upgrades and not verdict.satisfied and verdict.reason_code != "evidence_contradicts":
- return upgrades[verdict.criterion_id]
+ upgrade = upgrades.get(verdict.criterion_id)
+ if (
+ upgrade is not None
+ and _upgrade_is_admissible(upgrade)
+ and not verdict.satisfied
+ and verdict.reason_code != "evidence_contradicts"
+ ):
+ return upgrade
return verdict
verdicts = [_merged(verdict) for verdict in run_result.verdicts]
diff --git a/skyvern/forge/sdk/db/repositories/workflow_parameters.py b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
index e6f7b3df9..5774779cb 100644
--- a/skyvern/forge/sdk/db/repositories/workflow_parameters.py
+++ b/skyvern/forge/sdk/db/repositories/workflow_parameters.py
@@ -122,7 +122,10 @@ def _decode_completion_criteria_set(
not _floor_rekeyed_association_is_coherent(item)
or any(
field in item and item[field] != canonical_value
- for field, canonical_value in criterion_authority_projection(criterion).items()
+ for field, canonical_value in criterion_authority_projection(
+ criterion,
+ stored_item=item,
+ ).items()
)
or ("antecedent_family" in item and item["antecedent_family"] is None)
for item, criterion in zip(inner, decoded)
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 0e407f333..b461e6789 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -20,9 +20,11 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
ReconcileDecision,
StoredCriteriaSet,
StoredCriteriaSnapshot,
+ _criterion_reconcile_key,
build_turn_state,
criteria_from_json,
criteria_to_json,
+ criterion_authority_projection,
note_adjudication_on_turn_state,
plan_persistence,
reconcile_completion_criteria,
@@ -31,6 +33,8 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
CriterionVerdict,
+ RegisteredBlockerEvidence,
+ RenderedEvidenceRecord,
RunEvidenceSnapshot,
_coerce_result,
_render_criteria,
@@ -552,6 +556,141 @@ def test_typed_boolean_validation_classification_round_trip_preserves_shape_and_
assert reloaded[0].requested_output_evidence_source == "independent_run_evidence"
+def _neutral_reported_boolean(
+ *, output_key: str = "public_form_exists", request_slot_id: str = "a" * 64
+) -> CompletionCriterion:
+ return CompletionCriterion(
+ id=request_slot_id,
+ outcome=f"The run reports whether {output_key.replace('_', ' ')}.",
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key=output_key,
+ request_slot_id=request_slot_id,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+
+
+def test_neutral_reported_boolean_round_trip_preserves_coherent_tuple() -> None:
+ criterion = _neutral_reported_boolean()
+
+ raw = criteria_to_json([criterion])
+ reloaded = criteria_from_json(raw)
+
+ assert raw[0]["classification_output_key"] == "public_form_exists"
+ assert raw[0]["expected_output_shape"] == "goal_judgment_boolean"
+ assert raw[0]["requested_output_evidence_source"] == "independent_run_evidence"
+ assert "requested_output_floor_rekeyed" not in raw[0]
+ assert "floor_rekeyed_from_path" not in raw[0]
+ assert reloaded == (criterion,)
+
+
+def test_legacy_floor_marked_neutral_reported_boolean_normalizes_to_current_tuple() -> None:
+ legacy = replace(
+ _neutral_reported_boolean(),
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.public_form_exists",
+ )
+
+ (reloaded,) = criteria_from_json(criteria_to_json([legacy]))
+
+ assert reloaded == _neutral_reported_boolean()
+
+
+def test_malformed_neutral_boolean_is_degraded_before_persistence() -> None:
+ criterion = replace(
+ _neutral_reported_boolean(),
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.login_only",
+ )
+
+ (raw,) = criteria_to_json([criterion])
+
+ assert raw["classification_output_key"] is None
+ assert raw["expected_output_shape"] is None
+ assert raw["requested_output_evidence_source"] == "runtime_output"
+ assert raw.get("requested_output_floor_rekeyed", False) is False
+ assert raw.get("floor_rekeyed_from_path") is None
+ assert raw["antecedent_family"] == "undecidable"
+ assert raw["mint_disposition"] == "degraded"
+ assert raw["mint_degrade"] == "undecidable_judgment"
+
+
+def test_criterion_authority_projection_uses_only_normalized_neutral_boolean_fields() -> None:
+ malformed = replace(
+ _neutral_reported_boolean(),
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.login_only",
+ )
+
+ projection = criterion_authority_projection(malformed)
+
+ assert projection["classification_output_key"] is None
+ assert projection["expected_output_shape"] is None
+ assert projection["requested_output_evidence_source"] == "runtime_output"
+ assert projection["antecedent_family"] == "undecidable"
+ assert projection["requested_output_floor_rekeyed"] is False
+ assert projection["floor_rekeyed_from_path"] is None
+ assert projection["mint_disposition"] == "degraded"
+ assert projection["mint_degrade"] == "undecidable_judgment"
+
+
+ "mutation",
+ [
+ {"request_slot_id": None},
+ {"id": "b" * 64},
+ {"classification_output_key": None},
+ {
+ "requested_output_floor_rekeyed": True,
+ "floor_rekeyed_from_path": "output.login_only",
+ },
+ {"expected_classification": True},
+ {"pinability": "pinned"},
+ {"expected_output_shape": None},
+ ],
+)
+def test_malformed_neutral_reported_boolean_reload_degrades(mutation: dict[str, object]) -> None:
+ raw = criteria_to_json([_neutral_reported_boolean()])[0]
+ raw.update(mutation)
+
+ (criterion,) = criteria_from_json([raw])
+
+ assert criterion.kind == "outcome"
+ assert criterion.classification_output_key is None
+ assert criterion.expected_output_shape is None
+ assert criterion.requested_output_evidence_source == "runtime_output"
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
+def test_neutral_reported_boolean_reconcile_identity_excludes_request_slot_provenance() -> None:
+ criterion = _neutral_reported_boolean()
+ other_key = _neutral_reported_boolean(output_key="login_only")
+ other_slot = _neutral_reported_boolean(request_slot_id="b" * 64)
+
+ assert _criterion_reconcile_key(criterion) != _criterion_reconcile_key(other_key)
+ assert _criterion_reconcile_key(criterion) == _criterion_reconcile_key(other_slot)
+
+
+def test_selection_outcome_does_not_claim_neutral_boolean_shape() -> None:
+ criterion = CompletionCriterion(
+ id="c0",
+ outcome="The run selects the cheapest available plan.",
+ pinability="shapeless_valid",
+ requested_output_evidence_source="independent_run_evidence",
+ requested_output_floor_rekeyed=True,
+ floor_rekeyed_from_path="output.selected_plan",
+ )
+
+ assert criteria_from_json(criteria_to_json([criterion])) == (criterion,)
+
+
def test_untyped_boolean_validation_classification_reload_normalizes_shape_and_evidence() -> None:
"""Rows without typed mint metadata carry no reliable shape/evidence markers."""
legacy = {
@@ -827,14 +966,23 @@ def test_run_plane_all_no_evidence_ignores_definition_verdicts() -> None:
def test_coerce_result_tristate_mapping() -> None:
raw = {
"verdicts": [
- {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "result.ok",
+ },
{"criterion_id": "c1", "satisfied": False, "reason_code": "no_evidence"},
{"criterion_id": "c2", "satisfied": False, "reason_code": "evidence_contradicts"},
{"criterion_id": "c3", "satisfied": False, "reason_code": "unknown"},
{"criterion_id": "c4", "satisfied": True, "reason_code": "no_evidence"},
]
}
- result = _coerce_result(raw, ["c0", "c1", "c2", "c3", "c4", "c5"])
+ result = _coerce_result(
+ raw,
+ ["c0", "c1", "c2", "c3", "c4", "c5"],
+ evidence_catalog={"result": RenderedEvidenceRecord(value={"ok": True})},
+ )
states = {v.criterion_id: v.state for v in result.verdicts}
assert states == {
"c0": "satisfied",
@@ -1331,6 +1479,79 @@ def test_present_value_upgrade_is_upgrade_only() -> None:
assert upgraded.verdicts[0].reason_code == "present_value_verbatim"
+def test_neutral_boolean_present_value_upgrade_rejects_association_absent() -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome='The run reports public_form_exists as "false".',
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "unowned_output": {"public_form_exists": False},
+ "post_run_page_observation": {"forms": [], "public_form_exists": False},
+ },
+ block_output_sources={
+ "unowned_output": "registered_output_parameter",
+ "post_run_page_observation": "independent_page_evidence",
+ },
+ )
+ judge_result = _evaluated(_verdict(criterion.id, "unsatisfied", "no_evidence"))
+
+ upgraded = _apply_present_value_upgrades(judge_result, [criterion], snapshot)
+
+ assert upgraded.verdicts[0].state == "unsatisfied"
+ assert upgraded.verdicts[0].reason_code == "no_evidence"
+
+
+def test_neutral_boolean_present_value_upgrade_accepts_associated_independent_evidence() -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome='The run reports public_form_exists as "false".',
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "reported_summary": {"public_form_exists": False},
+ "post_run_page_observation": {"forms": [], "public_form_exists": False},
+ },
+ block_output_sources={
+ "reported_summary": "registered_output_parameter",
+ "post_run_page_observation": "independent_page_evidence",
+ },
+ registered_output_evidence_by_request_slot_id={
+ criterion.id: (
+ RegisteredBlockerEvidence(
+ block_label="report_public_form",
+ output_path="output.public_form_exists",
+ registered_output_key="reported_summary",
+ registered_output_id="op_reported_summary",
+ value=False,
+ ),
+ )
+ },
+ )
+ judge_result = _evaluated(_verdict(criterion.id, "unsatisfied", "no_evidence"))
+
+ upgraded = _apply_present_value_upgrades(judge_result, [criterion], snapshot)
+
+ assert upgraded.verdicts[0].state == "satisfied"
+ assert upgraded.verdicts[0].reason_code == "present_value_verbatim"
+
+
def test_present_value_credits_unquoted_structured_identifier() -> None:
criteria = [_criterion("c0", "the submitted request returns confirmation number WTR-1842-DEMO")]
snapshot = _snapshot({"submit_request": {"confirmation_number": "WTR-1842-DEMO", "status": "submitted"}})
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 682ddab06..0973c3e64 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -45,6 +45,7 @@ from skyvern.forge.sdk.copilot.completion_verification import (
FloorRekeyedDeliverableCredit,
FloorRekeyedEmissionWithhold,
RegisteredBlockerEvidence,
+ RenderedEvidenceRecord,
RunEvidenceSnapshot,
_coerce_result,
_structured_record_has_identifier,
@@ -99,6 +100,7 @@ from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDE
from skyvern.forge.sdk.copilot.hooks import _tool_completion_satisfies_turn
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import (
+ AntecedentFamily,
CompletionCriterion,
JudgmentTruthCondition,
RequestPolicy,
@@ -1153,8 +1155,18 @@ def test_fired_contingent_criterion_without_blocker_evidence_fails() -> None:
def test_fired_contingent_criterion_with_blocker_evidence_can_satisfy() -> None:
raw = {
"verdicts": [
- {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
- {"criterion_id": "c1", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "blocker.value",
+ },
+ {
+ "criterion_id": "c1",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "result.value",
+ },
]
}
result = _coerce_result(
@@ -1162,6 +1174,10 @@ def test_fired_contingent_criterion_with_blocker_evidence_can_satisfy() -> None:
["c0", "c1"],
contingent_criterion_ids=["c0"],
contingent_antecedent_output_path_by_criterion_id={"c0": "output.blocker"},
+ evidence_catalog={
+ "blocker": RenderedEvidenceRecord(value={"value": "provider requires a phone call"}),
+ "result": RenderedEvidenceRecord(value={"value": "request reported"}),
+ },
)
assert result.is_fully_satisfied() is True
@@ -1473,14 +1489,44 @@ def test_coerce_ignores_unknown_ids_and_dedupes_first_wins() -> None:
{"criterion_id": "ghost", "satisfied": True, "reason_code": "evidence_confirms"},
]
}
- result = _coerce_result(raw, ["c0"])
+ result = _coerce_result(
+ raw,
+ ["c0"],
+ evidence_catalog={"result": RenderedEvidenceRecord(value={"ok": True}, source="runtime_output")},
+ )
assert len(result.verdicts) == 1
- assert result.verdicts[0].satisfied is True
+ assert result.verdicts[0].satisfied is False
+
+
+def test_coerce_rejects_satisfied_verdict_without_rendered_evidence_ref() -> None:
+ result = _coerce_result(
+ {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ }
+ ]
+ },
+ ["c0"],
+ evidence_catalog={"result": RenderedEvidenceRecord(value={"ok": True}, source="runtime_output")},
+ )
+
+ assert result.verdicts[0].state == "unsatisfied"
+ assert result.verdicts[0].reason_code == "no_evidence"
def test_coerce_accepts_bytes_and_rejects_malformed() -> None:
- raw_bytes = b'{"verdicts": [{"criterion_id": "c0", "satisfied": true, "reason_code": "evidence_confirms"}]}'
- assert _coerce_result(raw_bytes, ["c0"]).is_fully_satisfied() is True
+ raw_bytes = b'{"verdicts": [{"criterion_id": "c0", "satisfied": true, "reason_code": "evidence_confirms", "evidence_ref": "result.ok"}]}'
+ assert (
+ _coerce_result(
+ raw_bytes,
+ ["c0"],
+ evidence_catalog={"result": RenderedEvidenceRecord(value={"ok": True}, source="runtime_output")},
+ ).is_fully_satisfied()
+ is True
+ )
assert _coerce_result("not json at all", ["c0"]).status == "unavailable"
assert _coerce_result({"no_verdicts_key": 1}, ["c0"]).status == "unavailable"
@@ -2617,7 +2663,16 @@ async def test_evaluate_uses_completion_judge_timeout(monkeypatch: pytest.Monkey
async def handler(**_: object) -> dict[str, object]:
await asyncio.sleep(0.05)
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
snapshot = RunEvidenceSnapshot(current_url="https://example.com/done")
result = await evaluate_completion_criteria([_criterion("c0", "done page visible")], snapshot, handler)
@@ -2628,7 +2683,16 @@ async def test_evaluate_uses_completion_judge_timeout(monkeypatch: pytest.Monkey
@pytest.mark.asyncio
async def test_evaluate_happy_path_returns_evaluated() -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "confirm.count",
+ }
+ ]
+ }
snapshot = RunEvidenceSnapshot(block_outputs={"confirm": {"count": 1}})
result = await evaluate_completion_criteria([_criterion("c0", "item in cart")], snapshot, handler)
@@ -5290,7 +5354,16 @@ async def test_classifier_fallback_record_is_not_verified_without_judge(monkeypa
@pytest.mark.asyncio
async def test_non_fallback_judge_confirmed_run_still_fires_barrier(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _ctx_with_blocks("extraction")
@@ -5520,8 +5593,18 @@ async def test_validation_classification_incomplete_contract_abstains_without_bl
calls += 1
return {
"verdicts": [
- {"criterion_id": "c_validation", "satisfied": True, "reason_code": "evidence_confirms"},
- {"criterion_id": "c_other", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c_validation",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "classify_path.path_classification",
+ },
+ {
+ "criterion_id": "c_other",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "classify_path.path_classification",
+ },
]
}
@@ -5923,7 +6006,16 @@ def test_artifact_health_blocks_committed_same_run_outcome() -> None:
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_runs_on_canceled_run(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -5962,7 +6054,16 @@ async def test_active_run_terminal_evidence_sample_matches_current_page(
async def handler(**kwargs: object) -> dict:
captured.update(kwargs)
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "page_evidence.result_containers",
+ }
+ ]
+ }
async def fake_fallback_page_info(_ctx: object) -> tuple[str, str]:
return "https://example.com/cart", "Cart"
@@ -6252,7 +6353,16 @@ async def test_page_observation_verification_recognizes_budgeted_outcome(
async def handler(**kwargs: object) -> dict:
seen_prompt["prompt"] = str(kwargs.get("prompt") or "")
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.hasProduct",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -6351,8 +6461,18 @@ async def test_page_observation_validation_classification_cannot_be_judge_approv
handler_calls += 1
return {
"verdicts": [
- {"criterion_id": "c_validation", "satisfied": True, "reason_code": "evidence_confirms"},
- {"criterion_id": "c_page", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c_validation",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.evidence_text",
+ },
+ {
+ "criterion_id": "c_page",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.evidence_text",
+ },
]
}
@@ -6436,8 +6556,18 @@ async def test_page_observation_validation_classification_incomplete_contract_ab
handler_calls += 1
return {
"verdicts": [
- {"criterion_id": "c_validation", "satisfied": True, "reason_code": "evidence_confirms"},
- {"criterion_id": "c_page", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c_validation",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.path_classification",
+ },
+ {
+ "criterion_id": "c_page",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.path_classification",
+ },
]
}
@@ -6595,7 +6725,16 @@ async def test_page_observation_verification_can_upgrade_unsatisfied_verdict(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "current_page_observation.hasProduct",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -6699,7 +6838,16 @@ def test_outcome_evidence_candidate_admits_clean_run_despite_unverified_prefix()
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_runs_on_unverified_prefix(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _ctx_unverified_prefix()
@@ -6766,7 +6914,16 @@ def test_gate_recognizes_clean_run_despite_unverified_prefix() -> None:
@pytest.mark.asyncio
async def test_method_mandated_criteria_excluded_from_verification(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -8461,7 +8618,16 @@ async def test_requested_output_verifier_accepts_p7_with_unfired_blocker_and_fee
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c_submit",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "extract_profile.customer_name",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -9477,7 +9643,16 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyte
async def handler(**kwargs: object) -> dict:
prompt = str(kwargs.get("prompt") or "")
seen_prompts.append(prompt)
- return {"verdicts": [{"criterion_id": "c_cart", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c_cart",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "extract_profile.items",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -9513,7 +9688,16 @@ async def test_present_generic_requested_output_without_expected_value_structura
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c_submit", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c_submit",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "extract_profile.customer_name",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -11432,8 +11616,18 @@ async def test_completion_verification_receives_verified_context_labels(monkeypa
seen_prompt["prompt"] = str(kwargs.get("prompt") or "")
return {
"verdicts": [
- {"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"},
- {"criterion_id": "c1", "satisfied": True, "reason_code": "evidence_confirms"},
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "extract_credential_details.extracted_information.credentials",
+ },
+ {
+ "criterion_id": "c1",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "extract_credential_details.extracted_information.credentials",
+ },
]
}
@@ -11501,7 +11695,16 @@ async def test_completion_verification_receives_verified_context_labels(monkeypa
@pytest.mark.asyncio
async def test_maybe_run_completion_verification_unavailable_on_low_budget(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "confirm",
+ }
+ ]
+ }
_patch_completion_handler(monkeypatch, handler)
ctx = _run_ctx()
@@ -12615,7 +12818,12 @@ def test_coerce_result_stamps_evidence_source_from_snapshot_sources() -> None:
result = _coerce_result(
raw,
["c_corroborator"],
- block_output_sources={"current_page_observation": "independent_page_evidence"},
+ evidence_catalog={
+ "current_page_observation": RenderedEvidenceRecord(
+ value={"selected_highest_priority": True},
+ source="independent_page_evidence",
+ )
+ },
)
assert result.verdicts[0].evidence_source == "independent_page_evidence"
@@ -12712,7 +12920,12 @@ def test_coerce_result_stamps_evidence_source_from_bare_label_ref() -> None:
result = _coerce_result(
raw,
["c_corroborator"],
- block_output_sources={"current_page_observation": "independent_page_evidence"},
+ evidence_catalog={
+ "current_page_observation": RenderedEvidenceRecord(
+ value={"selected_highest_priority": True},
+ source="independent_page_evidence",
+ )
+ },
)
assert result.verdicts[0].evidence_source == "independent_page_evidence"
@@ -12731,7 +12944,12 @@ def test_bare_label_corroborator_certifies_self_emitted_judgment() -> None:
]
},
["c_judgment__requested_output_corroborator"],
- block_output_sources={"current_page_observation": "independent_page_evidence"},
+ evidence_catalog={
+ "current_page_observation": RenderedEvidenceRecord(
+ value={"selected_highest_priority": True},
+ source="independent_page_evidence",
+ )
+ },
).verdicts[0]
run_result = CompletionVerificationResult(
status="evaluated",
@@ -13958,7 +14176,6 @@ def test_run_evidence_snapshot_preserves_registered_null_for_family_routing() ->
],
}
}
-
snapshot = _build_run_evidence_snapshot(ctx, result)
assert snapshot.registered_output_values["submit_commercial_water_request_output"] == {
@@ -13968,6 +14185,581 @@ def test_run_evidence_snapshot_preserves_registered_null_for_family_routing() ->
assert snapshot.block_outputs["submit_commercial_water_request"]["blocker"] == "incidental runtime value"
+ ("classification_output_key", "emitted_output"),
+ [
+ ("public_form_exists", {"public_form_exists": False}),
+ ("login_only", {"login_only": True}),
+ ],
+ ids=["correct_negative", "login_only"],
+)
+async def test_neutral_reported_boolean_accepts_evidence_without_floor_rekey_withhold(
+ classification_output_key: str,
+ emitted_output: dict[str, bool],
+) -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome=f"The run reports whether {classification_output_key.replace('_', ' ')}.",
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key=classification_output_key,
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ workflow_run_id="wr_553389131952493792",
+ block_outputs={
+ "reported_summary": emitted_output,
+ "post_run_page_observation": {
+ "forms": [],
+ "result_containers": [],
+ "visible_text_excerpt": "Sign in or register to continue.",
+ },
+ },
+ block_output_sources={
+ "reported_summary": "registered_output_parameter",
+ "post_run_page_observation": "independent_page_evidence",
+ },
+ registered_output_evidence_by_request_slot_id={
+ criterion.id: (
+ RegisteredBlockerEvidence(
+ block_label="report_outcome",
+ output_path=f"output.{classification_output_key}",
+ registered_output_key="reported_summary",
+ registered_output_id="op_reported_summary",
+ value=emitted_output[classification_output_key],
+ ),
+ )
+ },
+ run_terminal_status="completed",
+ )
+
+ async def verifier(*, prompt: str, **_: object) -> dict[str, object]:
+ assert f"classification_output_key={classification_output_key}" in prompt
+ assert json.dumps(emitted_output, sort_keys=True)[1:-1] in prompt
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "post_run_page_observation.forms",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+ result = carry_floor_rekeyed_criterion_ids(result, [criterion])
+
+ assert result.is_fully_satisfied() is True
+ assert result.floor_rekeyed_criterion_ids == []
+ assert floor_rekeyed_emission_withhold(result) is None
+ assert result.verdicts[0].reason_code == "evidence_confirms"
+ assert result.verdicts[0].evidence_source == "independent_page_evidence"
+ assert all(verdict.reason_code != "evidence_contradicts" for verdict in result.verdicts)
+
+
[email protected]("antecedent_family", ["unconditional", "blocker"])
+async def test_neutral_reported_boolean_rejects_its_associated_output_as_evidence(
+ antecedent_family: AntecedentFamily,
+) -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether public form exists.",
+ antecedent_family=antecedent_family,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={"reported_summary": {"public_form_exists": False}},
+ block_output_sources={"reported_summary": "registered_output_parameter"},
+ registered_output_evidence_by_request_slot_id={
+ criterion.id: (
+ RegisteredBlockerEvidence(
+ block_label="report_public_form",
+ output_path="output.public_form_exists",
+ registered_output_key="reported_summary",
+ registered_output_id="op_reported_summary",
+ value=False,
+ ),
+ )
+ },
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "reported_summary.public_form_exists",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].state == "unsatisfied"
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
+ ("block_outputs", "block_output_sources", "evidence_ref"),
+ [
+ (
+ {},
+ {"post_run_page_observation": "independent_page_evidence"},
+ "post_run_page_observation.forms",
+ ),
+ (
+ {"post_run_page_observation": {"result_containers": []}},
+ {"post_run_page_observation": "independent_page_evidence"},
+ "post_run_page_observation.forms",
+ ),
+ (
+ {"post_run_page_observation": {"forms": []}},
+ {"post_run_page_observation": "runtime_output"},
+ "post_run_page_observation.forms",
+ ),
+ ],
+ ids=["source_map_only", "unresolved_path", "wrong_source"],
+)
+async def test_judge_evidence_provenance_invariant(
+ block_outputs: dict[str, object],
+ block_output_sources: dict[str, EvidenceSourceKind],
+ evidence_ref: str,
+) -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether a public form exists.",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs=block_outputs,
+ block_output_sources=block_output_sources,
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": evidence_ref,
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].state == "unsatisfied"
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
+ "evidence_ref",
+ [
+ "post_run_page_observation.forms[]",
+ "post_run_page_observation.forms[invalid]",
+ "post_run_page_observation.forms[-1]",
+ "post_run_page_observation.forms[1]",
+ "post_run_page_observation.forms[0].inputs[1]",
+ "post_run_page_observation.forms[0].missing",
+ ],
+ ids=[
+ "empty_index",
+ "non_numeric_index",
+ "negative_index",
+ "form_index_out_of_bounds",
+ "nested_index_out_of_bounds",
+ "remaining_path_missing",
+ ],
+)
+async def test_judge_evidence_provenance_rejects_invalid_indexed_path(evidence_ref: str) -> None:
+ criterion = CompletionCriterion(id="c0", outcome="The page contains the requested form.")
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "post_run_page_observation": {"forms": [{"inputs": [{"id": "account"}]}]},
+ },
+ block_output_sources={"post_run_page_observation": "independent_page_evidence"},
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": evidence_ref,
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
+async def test_judge_evidence_provenance_accepts_existing_indexed_element_and_remaining_path() -> None:
+ criterion = CompletionCriterion(id="c0", outcome="The page contains the requested form.")
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "post_run_page_observation": {"forms": [{"inputs": [{"id": "account"}]}]},
+ },
+ block_output_sources={"post_run_page_observation": "independent_page_evidence"},
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "post_run_page_observation.forms[0].inputs[0].id",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is True
+
+
+async def test_judge_evidence_provenance_accepts_label_for_truncated_structured_record() -> None:
+ criterion = CompletionCriterion(id="c0", outcome="The page contains the requested form.")
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "post_run_page_observation": {
+ "forms": [{"inputs": [{"id": "account", "padding": "x" * 3000}]}],
+ },
+ },
+ block_output_sources={"post_run_page_observation": "independent_page_evidence"},
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "post_run_page_observation",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is True
+
+
[email protected]("antecedent_family", ["unconditional", "blocker"])
+ "registered_outputs",
+ [
+ {},
+ {"reported_summary": {"public_form_exists": "false"}},
+ {"reported_summary": {"public_form_exists": False}},
+ {
+ "reported_summary": {"public_form_exists": False},
+ "duplicate_summary": {"public_form_exists": False},
+ },
+ ],
+ ids=["missing", "non_boolean", "unassociated", "ambiguous"],
+)
+async def test_neutral_reported_boolean_requires_one_registered_boolean_output(
+ registered_outputs: dict[str, object],
+ antecedent_family: AntecedentFamily,
+) -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether a public form exists.",
+ antecedent_family=antecedent_family,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ **registered_outputs,
+ "post_run_page_observation": {"forms": []},
+ },
+ block_output_sources={
+ **{label: "registered_output_parameter" for label in registered_outputs},
+ "post_run_page_observation": "independent_page_evidence",
+ },
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "post_run_page_observation.forms",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
+async def test_active_page_evidence_is_admissible_independent_evidence() -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether a public form exists.",
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={"reported_summary": {"public_form_exists": False}},
+ block_output_sources={"reported_summary": "registered_output_parameter"},
+ registered_output_evidence_by_request_slot_id={
+ criterion.id: (
+ RegisteredBlockerEvidence(
+ block_label="report_public_form",
+ output_path="output.public_form_exists",
+ registered_output_key="reported_summary",
+ registered_output_id="op_reported_summary",
+ value=False,
+ ),
+ )
+ },
+ page_evidence={"forms": [{"id": "service-login"}]},
+ )
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "page_evidence.forms",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is True
+ assert result.verdicts[0].evidence_source == "independent_page_evidence"
+
+
+ ("reserved_label", "server_value"),
+ [
+ ("observed_end_state_url", "https://example.com/final"),
+ ("observed_end_state_page_title", "Final page"),
+ ("page_evidence", {"forms": [{"id": "service-login"}]}),
+ ],
+)
+def test_reserved_evidence_labels_fail_closed_on_runtime_collision(
+ reserved_label: str,
+ server_value: object,
+) -> None:
+ snapshot = RunEvidenceSnapshot(
+ current_url=server_value if reserved_label == "observed_end_state_url" else None,
+ page_title=server_value if reserved_label == "observed_end_state_page_title" else None,
+ page_evidence=server_value if reserved_label == "page_evidence" else {},
+ block_outputs={reserved_label: {"runtime": "ambiguous"}},
+ block_output_sources={reserved_label: "runtime_output"},
+ )
+
+ assert reserved_label not in snapshot.rendered_evidence_catalog()
+
+
+async def test_neutral_boolean_rejects_associated_output_beyond_rendered_catalog_cap() -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether a public form exists.",
+ antecedent_family="unconditional",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ padding = {f"padding_{index}": {"value": index} for index in range(19)}
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={
+ "post_run_page_observation": {"forms": [], "public_form_exists": False},
+ **padding,
+ "reported_summary": {"public_form_exists": False},
+ },
+ block_output_sources={
+ "post_run_page_observation": "independent_page_evidence",
+ **{label: "runtime_output" for label in padding},
+ "reported_summary": "registered_output_parameter",
+ },
+ registered_output_evidence_by_request_slot_id={
+ criterion.id: (
+ RegisteredBlockerEvidence(
+ block_label="report_public_form",
+ output_path="output.public_form_exists",
+ registered_output_key="reported_summary",
+ registered_output_id="op_reported_summary",
+ value=False,
+ ),
+ )
+ },
+ )
+ assert "reported_summary" not in snapshot.rendered_evidence_catalog()
+
+ async def verifier(**_: object) -> dict[str, object]:
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "post_run_page_observation.forms",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
[email protected]("antecedent_family", ["unconditional", "blocker"])
+ ("registered_block_label", "association_expected"),
+ [("report_public_form", True), ("unowned_producer", False)],
+ ids=["accepted-owner", "association-absent"],
+)
+def test_snapshot_binds_neutral_boolean_to_exact_artifact_owner_and_request_slot(
+ antecedent_family: AntecedentFamily,
+ registered_block_label: str,
+ association_expected: bool,
+) -> None:
+ criterion = CompletionCriterion(
+ id="a" * 64,
+ outcome="The run reports whether a public form exists.",
+ antecedent_family=antecedent_family,
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ kind="outcome",
+ classification_output_key="public_form_exists",
+ request_slot_id="a" * 64,
+ pinability="shapeless_valid",
+ mint_disposition="decidable",
+ )
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "report_public_form", "unowned_producer")
+ ctx.request_policy = RequestPolicy(completion_contract_status="present", completion_criteria=[criterion])
+ ctx.code_artifact_metadata = {
+ "report_public_form": {
+ "block_label": "report_public_form",
+ "claimed_outcomes": [{"goal_value_paths": ["public_form_exists"]}],
+ "completion_criteria": [{"id": "artifact:public_form_exists"}],
+ }
+ }
+ run_result = {
+ "data": {
+ "workflow_run_id": "wr_neutral_association",
+ "workflow_run_output_parameters": [
+ {
+ "workflow_run_id": "wr_neutral_association",
+ "output_parameter_id": "op_public_form",
+ "output_parameter_key": "reported_summary",
+ "block_label": registered_block_label,
+ "block_type": "code",
+ "value": {"public_form_exists": False},
+ }
+ ],
+ }
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, run_result)
+
+ assert snapshot.registered_output_values["reported_summary"] == {"public_form_exists": False}
+ if not association_expected:
+ assert criterion.id not in snapshot.registered_output_evidence_by_request_slot_id
+ return
+ association = snapshot.registered_output_evidence_by_request_slot_id[criterion.id]
+ assert len(association) == 1
+ assert association[0].block_label == "report_public_form"
+ assert association[0].output_path == "output.public_form_exists"
+ assert association[0].registered_output_key == "reported_summary"
+ assert association[0].value is False
+
+
+async def test_judge_cannot_cite_a_path_truncated_out_of_its_prompt() -> None:
+ criterion = CompletionCriterion(id="c0", outcome="The run reaches the requested state.")
+ snapshot = RunEvidenceSnapshot(
+ block_outputs={"result": {"padding": "x" * 2200, "hidden": {"ok": True}}},
+ block_output_sources={"result": "runtime_output"},
+ )
+
+ async def verifier(*, prompt: str, **_: object) -> dict[str, object]:
+ assert '"hidden"' not in prompt
+ return {
+ "verdicts": [
+ {
+ "criterion_id": criterion.id,
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "result.hidden.ok",
+ }
+ ]
+ }
+
+ result = await evaluate_completion_criteria([criterion], snapshot, verifier)
+
+ assert result.is_fully_satisfied() is False
+ assert result.verdicts[0].reason_code == "no_evidence"
+
+
def test_runtime_output_named_like_registered_output_cannot_fire_blocker_family() -> None:
ctx = _run_ctx()
_set_workflow_labels(ctx, "submit_commercial_water_request")
diff --git a/tests/unit/test_copilot_completion_zero_run_plane.py b/tests/unit/test_copilot_completion_zero_run_plane.py
index d4965220b..288e835dc 100644
--- a/tests/unit/test_copilot_completion_zero_run_plane.py
+++ b/tests/unit/test_copilot_completion_zero_run_plane.py
@@ -112,7 +112,16 @@ async def test_all_method_mandated_floor_fallback_gate_fires(monkeypatch: pytest
@pytest.mark.asyncio
async def test_fallback_with_real_run_plane_criterion_does_not_fire(monkeypatch: pytest.MonkeyPatch) -> None:
async def handler(**_: object) -> dict:
- return {"verdicts": [{"criterion_id": "c0", "satisfied": True, "reason_code": "evidence_confirms"}]}
+ return {
+ "verdicts": [
+ {
+ "criterion_id": "c0",
+ "satisfied": True,
+ "reason_code": "evidence_confirms",
+ "evidence_ref": "observed_end_state_url",
+ }
+ ]
+ }
monkeypatch.setattr(
"skyvern.forge.sdk.copilot.tools.completion._completion_verification_handler",
diff --git a/tests/unit/test_copilot_request_policy_pinability_consumer.py b/tests/unit/test_copilot_request_policy_pinability_consumer.py
index 7dd0ffd71..77e79a165 100644
--- a/tests/unit/test_copilot_request_policy_pinability_consumer.py
+++ b/tests/unit/test_copilot_request_policy_pinability_consumer.py
@@ -17,17 +17,28 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
criteria_to_json,
reconcile_completion_criteria,
)
+from skyvern.forge.sdk.copilot.completion_output_grounding import floor_rekeyed_path_backing
from skyvern.forge.sdk.copilot.completion_verification import RunEvidenceSnapshot, evaluate_completion_criteria
from skyvern.forge.sdk.copilot.request_policy import (
+ CompletionCriterion,
+ CriterionLevel,
+ ExpectedOutputShape,
+ RequestPolicy,
+ TrustedRequestSlotDatumBindingV1,
_accept_request_slot_anchor_correction,
_anchor_correction_rejection_capture,
_apply_request_slot_datum_bindings,
+ _bind_criterion_to_request_slot,
_classification_from_raw,
_classify_request,
+ _interrogative_slot_classification_output_key,
+ _omitted_request_slot_datum_binding_targets,
+ _parse_fresh_request_slot_criteria,
schema_output_path_aliases_from_criteria,
)
from skyvern.forge.sdk.copilot.request_slots import PROMPT_NAME as REQUEST_SLOT_PROMPT_NAME
from skyvern.forge.sdk.copilot.request_slots import (
+ CanonicalRequestSlotV1,
RequestSlotAntecedentFamily,
RequestSlotDatumBindingDeclarationV1,
RequestSlotDatumDeclineDeclarationV1,
@@ -82,6 +93,55 @@ P8_CLEAN_RETRY_CUSTODY_GAP_FIXTURE = (
)
+def test_legacy_interrogative_key_synthesis_still_omits_path_subject() -> None:
+ assert _interrogative_slot_classification_output_key("whether the path is login-only") == "login_only"
+
+
+def test_direct_interrogative_slot_preserves_producer_undecidable_disposition() -> None:
+ source_quote = "whether the path is login-only"
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request=f"Return {source_quote}.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNDECIDABLE,
+ ),
+ ),
+ ),
+ )
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="The run classifies whether the path is login-only.",
+ kind="validation_classification",
+ classification_output_key="login_only",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ slot=contract.slots[0],
+ source_quote=source_quote,
+ )
+
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.classification_output_key is None
+ assert criterion.mint_disposition == "degraded"
+
+
def _recorded_p8_original_payload_bytes() -> bytes:
# The fixture stores backend.log:1316's original_payload_json verbatim; the
# repository text-file newline is not part of the recorded JSON field.
@@ -314,7 +374,10 @@ async def test_independent_request_slot_producer_recovers_primary_classifier_omi
assert len(policy.completion_criteria) == 1
criterion = policy.completion_criteria[0]
assert criterion.outcome == source_quote
- assert criterion.antecedent_family == "blocker"
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
assert criterion.request_slot_id is not None
@@ -472,6 +535,472 @@ def _fresh_payload() -> dict[str, Any]:
}
+def _reported_boolean_contract(
+ pinability: RequestSlotPinability,
+) -> tuple[RequestSlotProducerInputV1, CanonicalRequestSlotV1]:
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return a structured summary showing whether a public form exists.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether a public form exists",
+ plane=RequestSlotPlane.RUN,
+ pinability=pinability,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ ),
+ )
+ return request, contract.slots[0]
+
+
+def test_neutral_reported_boolean_mints_keyed_polarity_neutral_outcome() -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path="output.public_form_exists",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.outcome == "The run reports whether public form exists."
+ assert criterion.kind == "outcome"
+ assert criterion.output_path is None
+ assert criterion.expected_output_value is None
+ assert criterion.expected_classification is None
+ assert criterion.expected_output_shape == "goal_judgment_boolean"
+ assert criterion.requested_output_evidence_source == "independent_run_evidence"
+ assert criterion.classification_output_key == "public_form_exists"
+ assert criterion.request_slot_id == slot.slot_id
+ assert criterion.pinability == "shapeless_valid"
+ assert criterion.mint_disposition == "decidable"
+ assert criterion.mint_degrade is None
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ trace = RequestPolicy(completion_criteria=[criterion]).to_trace_data()
+ assert trace["neutral_reported_boolean_criterion_count"] == 1
+ assert trace["neutral_reported_boolean_criterion_0_id"] == slot.slot_id
+ assert trace["neutral_reported_boolean_criterion_0_classification_output_key"] == "public_form_exists"
+ assert trace["neutral_reported_boolean_criterion_0_request_slot_id"] == slot.slot_id
+ assert trace["neutral_reported_boolean_criterion_0_mint_disposition"] == "decidable"
+ assert trace["neutral_reported_boolean_criterion_0_expected_output_value"] is None
+ assert trace["neutral_reported_boolean_criterion_0_expected_classification"] is None
+ assert trace["neutral_reported_boolean_criterion_0_requested_output_floor_rekeyed"] is False
+ assert trace["neutral_reported_boolean_criterion_0_floor_rekeyed_from_path"] is None
+
+
+def test_assertive_positive_reported_boolean_keeps_expected_polarity() -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.PINNED)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path="output.public_form_exists",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.outcome == "A public form exists."
+ assert criterion.output_path == "output.public_form_exists"
+ assert criterion.expected_output_value is True
+ assert criterion.classification_output_key is None
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ assert criterion.pinability == "pinned"
+ assert criterion.mint_disposition == "pending"
+
+
+def test_assertive_shapeless_boolean_degrades_instead_of_becoming_neutral() -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path="output.public_form_exists",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="a public form exists",
+ )
+
+ assert criterion.classification_output_key is None
+ assert criterion.expected_output_value is None
+ assert criterion.expected_output_shape is None
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
[email protected]("binding_count", [0, 1, 2], ids=["association-absent", "associated", "association-ambiguous"])
+def test_omitted_public_form_row_preserves_neutral_output_binding(binding_count: int) -> None:
+ source_quote = "whether a public form exists"
+ criterion_outcome_sha256 = hashlib.sha256(b"The run classifies whether public form exists.").hexdigest()
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request=f"Return a structured summary showing {source_quote}.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="classification_output_key",
+ datum_value="public_form_exists",
+ criterion_outcome_sha256=criterion_outcome_sha256,
+ ),
+ ),
+ )
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="classification_output_key",
+ declined=False,
+ source_id="u0",
+ source_quote=source_quote,
+ ),
+ ),
+ ),
+ )
+ canonical_binding = contract.datum_bindings[0]
+ binding = TrustedRequestSlotDatumBindingV1(
+ version="1",
+ criterion_index=canonical_binding.criterion_index,
+ datum_field=canonical_binding.datum_field,
+ datum_value=canonical_binding.datum_value,
+ criterion_outcome_sha256=canonical_binding.criterion_outcome_sha256,
+ source_id=canonical_binding.source_id,
+ source_quote=canonical_binding.source_quote,
+ slot_id=canonical_binding.slot_id,
+ )
+
+ criterion = _parse_fresh_request_slot_criteria(
+ [],
+ request_slot_request=request,
+ request_slot_contract=contract,
+ trusted_datum_bindings=(binding,) * binding_count,
+ )[0]
+ backing = floor_rekeyed_path_backing(
+ SimpleNamespace(
+ code_artifact_metadata={
+ "validate_riverbend_public_service_path": {
+ "claimed_outcomes": [{"goal_value_paths": ["public_form_exists"]}],
+ }
+ },
+ workflow_verification_evidence=None,
+ last_workflow_yaml=None,
+ workflow_yaml=None,
+ ),
+ [criterion],
+ {"validate_riverbend_public_service_path": {"public_form_exists": False}},
+ {"validate_riverbend_public_service_path": "registered_output_parameter"},
+ {"validate_riverbend_public_service_path": "code"},
+ )
+
+ if binding_count == 1:
+ assert criterion.outcome == "The run reports whether public form exists."
+ assert criterion.classification_output_key == "public_form_exists"
+ assert criterion.expected_output_shape == "goal_judgment_boolean"
+ assert criterion.requested_output_evidence_source == "independent_run_evidence"
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ assert backing == {}
+ else:
+ assert criterion.classification_output_key is None
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ assert backing == {}
+
+
+def test_anchored_criterion_counts_as_representing_slot_without_datum_binding() -> None:
+ source_quote = "whether a public form exists"
+ request, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ raw = [
+ {
+ "outcome": "The run reports whether public form exists.",
+ "request_slot_source_id": "u0",
+ "request_slot_source_quote": source_quote,
+ }
+ ]
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ ),
+ )
+
+ assert contract.slots[0].slot_id == slot.slot_id
+ assert (
+ _omitted_request_slot_datum_binding_targets(
+ raw,
+ request_slot_request=request,
+ request_slot_contract=contract,
+ existing_targets=(),
+ )
+ == ()
+ )
+
+
+async def test_classify_request_mints_neutral_boolean_when_primary_omits_interrogative_row() -> None:
+ request_text = "Return a structured summary showing whether a public form exists."
+ source_quote = "whether a public form exists"
+ primary = {
+ "testing_intent": "require_test",
+ "credential_input_kind": "none",
+ "requires_user_clarification": False,
+ "completion_criteria": [],
+ }
+ envelope = RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote=source_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ )
+ calls: list[str] = []
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, Any]:
+ calls.append(prompt_name)
+ if prompt_name != REQUEST_SLOT_PROMPT_NAME:
+ return primary
+ marker = "Datum targets from the immutable request-policy payload:\n```"
+ targets = json.loads(prompt.split(marker, 1)[1].split("```", 1)[0])
+ if not targets:
+ return envelope.model_dump(mode="json")
+ return _request_slot_envelope_with_target_bindings(
+ prompt,
+ envelope,
+ anchors_by_index={0: ("u0", source_quote)},
+ )
+
+ policy = await _classify_request(request_text, "", [], "", handler)
+
+ assert calls == [
+ "workflow-copilot-request-policy",
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ REQUEST_SLOT_PROMPT_NAME,
+ ]
+ assert policy.request_slot_failure_kind is None
+ assert len(policy.completion_criteria) == 1
+ criterion = policy.completion_criteria[0]
+ assert criterion.outcome == "The run reports whether public form exists."
+ assert criterion.classification_output_key == "public_form_exists"
+ assert criterion.expected_output_shape == "goal_judgment_boolean"
+ assert criterion.requested_output_evidence_source == "independent_run_evidence"
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+ assert criterion.mint_disposition == "decidable"
+
+
+ ("slot_plane", "criterion_level"),
+ [
+ (RequestSlotPlane.DEFINITION, "run"),
+ (RequestSlotPlane.RUN, "definition"),
+ ],
+ ids=["definition_slot", "definition_criterion"],
+)
+def test_neutral_reported_boolean_requires_run_plane_slot_and_criterion(
+ slot_plane: RequestSlotPlane,
+ criterion_level: CriterionLevel,
+) -> None:
+ request = RequestSlotProducerInputV1(
+ version="1",
+ latest_request="Return a structured summary showing whether a public form exists.",
+ workflow_context="",
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history=(),
+ global_context="",
+ )
+ slot = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id="u0",
+ source_quote="whether a public form exists",
+ plane=slot_plane,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ ),
+ ).slots[0]
+
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ level=criterion_level,
+ output_path="output.public_form_exists",
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.classification_output_key is None
+ assert criterion.expected_output_shape is None
+ assert criterion.requested_output_evidence_source == "independent_run_evidence"
+
+
[email protected]("output_path", [None, "output.review.public_form_exists"])
+def test_malformed_neutral_boolean_without_one_top_level_output_binding_stays_fail_closed(
+ output_path: str | None,
+) -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path=output_path,
+ expected_output_value=True,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.classification_output_key is None
+ assert criterion.expected_output_shape is None
+ assert criterion.expected_output_value is None
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
+ ("output_path", "classification_output_key"),
+ [
+ ("output.public_form_exists", "login_only"),
+ (None, "public.form.exists"),
+ ],
+)
+def test_malformed_neutral_boolean_with_conflicting_or_invalid_bindings_stays_fail_closed(
+ output_path: str | None,
+ classification_output_key: str,
+) -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path=output_path,
+ expected_output_value=True,
+ classification_output_key=classification_output_key,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.classification_output_key is None
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
+ ("expected_output_value", "expected_output_shape", "expected_classification"),
+ [
+ (True, "goal_judgment_boolean", None),
+ (True, None, False),
+ (None, "goal_judgment_boolean", True),
+ ],
+)
+def test_malformed_neutral_boolean_with_multiple_boolean_signals_stays_fail_closed(
+ expected_output_value: bool | None,
+ expected_output_shape: ExpectedOutputShape | None,
+ expected_classification: bool | None,
+) -> None:
+ _request_input, slot = _reported_boolean_contract(RequestSlotPinability.SHAPELESS_VALID)
+ criterion = _bind_criterion_to_request_slot(
+ CompletionCriterion(
+ id="c0",
+ outcome="A public form exists.",
+ output_path="output.public_form_exists",
+ expected_output_value=expected_output_value,
+ expected_output_shape=expected_output_shape,
+ expected_classification=expected_classification,
+ requested_output_evidence_source="independent_run_evidence",
+ mint_disposition="pending",
+ ),
+ slot=slot,
+ source_quote="whether a public form exists",
+ )
+
+ assert criterion.classification_output_key is None
+ assert criterion.expected_output_shape is None
+ assert criterion.expected_output_value is None
+ assert criterion.expected_classification is None
+ assert criterion.antecedent_family == "undecidable"
+ assert criterion.mint_disposition == "degraded"
+ assert criterion.mint_degrade == "undecidable_judgment"
+
+
def test_fresh_contract_consumes_declared_pinability_without_guessed_polarity() -> None:
request = _request()
contract = canonicalize_request_slots(request=request, envelope=_envelope())
diff --git a/tests/unit/test_copilot_request_slots.py b/tests/unit/test_copilot_request_slots.py
index 97aa7b24c..55117ffa8 100644
--- a/tests/unit/test_copilot_request_slots.py
+++ b/tests/unit/test_copilot_request_slots.py
@@ -875,6 +875,334 @@ def test_datum_targets_must_map_injectively_to_independent_slots() -> None:
)
+def test_datum_binding_prefers_exact_direct_interrogative_output_over_broad_task_goal() -> None:
+ request = _input(
+ latest_request=(
+ "Check whether Riverbend Gas has a public path for starting gas service. "
+ "Return a structured summary showing whether a public form exists, "
+ "whether the path is login-only, and the recommended next action."
+ ),
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, "Riverbend Gas")
+ broad_quote = "whether Riverbend Gas has a public path for starting gas service"
+ exact_quote = "whether a public form exists"
+
+ with pytest.raises(ValueError, match="direct interrogative output"):
+ canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote=broad_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNDECIDABLE,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=source_id,
+ source_quote=broad_quote,
+ ),
+ ),
+ ),
+ )
+
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote=exact_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=source_id,
+ source_quote=exact_quote,
+ ),
+ ),
+ ),
+ )
+
+ assert contract.datum_bindings[0].source_quote == exact_quote
+ assert contract.slots[0].antecedent_family == RequestSlotAntecedentFamily.UNCONDITIONAL
+
+
+ ("latest_request", "exact_quote"),
+ [
+ (
+ "Return whether a public form exists and whether the path is login-only.",
+ "whether a public form exists",
+ ),
+ (
+ "Return (whether a public form exists), plus the recommended next action.",
+ "whether a public form exists",
+ ),
+ ],
+)
+def test_direct_interrogative_output_anchor_handles_coordination_and_wrappers(
+ latest_request: str,
+ exact_quote: str,
+) -> None:
+ request = _input(
+ latest_request=latest_request,
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, exact_quote)
+
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote=exact_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=source_id,
+ source_quote=exact_quote,
+ ),
+ ),
+ ),
+ )
+
+ assert contract.datum_bindings[0].source_quote == exact_quote
+
+
+def test_direct_interrogative_output_anchor_prefers_latest_matching_source() -> None:
+ exact_quote = "whether a public form exists"
+ request = _input(
+ latest_prior_user_turn=f"Earlier, return {exact_quote}.",
+ latest_request=f"Now return {exact_quote}.",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ latest_source_id = request_slot_sources(request)[-1].source_id
+
+ contract = canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=latest_source_id,
+ source_quote=exact_quote,
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumBindingDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=False,
+ source_id=latest_source_id,
+ source_quote=exact_quote,
+ ),
+ ),
+ ),
+ )
+
+ assert contract.datum_bindings[0].source_id == latest_source_id
+
+
+def test_duplicate_direct_interrogative_output_in_latest_source_fails_closed() -> None:
+ exact_quote = "whether a public form exists"
+ request = _input(
+ latest_request=f"Return {exact_quote}; confirm {exact_quote}.",
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, exact_quote)
+
+ with pytest.raises(ValueError, match="must be unique"):
+ canonicalize_request_slots(
+ request=request,
+ envelope=RequestSlotEnvelopeV1(
+ version="1",
+ slots=(
+ RequestSlotDeclarationV1(
+ source_id=source_id,
+ source_quote="Return",
+ plane=RequestSlotPlane.RUN,
+ pinability=RequestSlotPinability.SHAPELESS_VALID,
+ antecedent_family=RequestSlotAntecedentFamily.UNCONDITIONAL,
+ ),
+ ),
+ datum_bindings=(
+ RequestSlotDatumDeclineDeclarationV1(
+ criterion_index=0,
+ datum_field="output_path",
+ declined=True,
+ ),
+ ),
+ ),
+ )
+
+
+async def test_producer_retries_broad_binding_until_direct_interrogative_output_is_bound() -> None:
+ request = _input(
+ latest_request=(
+ "Check whether Riverbend Gas has a public path for starting gas service. "
+ "Return a structured summary showing whether a public form exists."
+ ),
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, "Riverbend Gas")
+ broad_quote = "whether Riverbend Gas has a public path for starting gas service"
+ exact_quote = "whether a public form exists"
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal calls
+ calls += 1
+ assert "Never bind the target to an earlier, broader task goal" in prompt
+ quote = broad_quote if calls == 1 else exact_quote
+ antecedent_family = "undecidable" if calls == 1 else "unconditional"
+ return {
+ "version": "1",
+ "slots": [
+ {
+ "source_id": source_id,
+ "source_quote": quote,
+ "plane": "run",
+ "pinability": "shapeless_valid",
+ "antecedent_family": antecedent_family,
+ }
+ ],
+ "datum_bindings": [
+ {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": False,
+ "source_id": source_id,
+ "source_quote": quote,
+ }
+ ],
+ }
+
+ result = await produce_request_slots(request=request, handler=handler, timeout_seconds=1.0)
+
+ assert result.status == "success"
+ assert result.attempts == 3
+ assert result.contract is not None
+ assert result.contract.datum_bindings[0].source_quote == exact_quote
+ assert calls == 3
+
+
+async def test_producer_exhausts_retries_when_direct_interrogative_output_is_always_misbound() -> None:
+ request = _input(
+ latest_request=(
+ "Check whether Riverbend Gas has a public path for starting gas service. "
+ "Return a structured summary showing whether a public form exists."
+ ),
+ datum_targets=(
+ RequestSlotDatumTargetV1(
+ criterion_index=0,
+ datum_field="output_path",
+ datum_value="output.public_form_exists",
+ criterion_outcome_sha256="1" * 64,
+ ),
+ ),
+ )
+ source_id = _source_id(request, "Riverbend Gas")
+ broad_quote = "whether Riverbend Gas has a public path for starting gas service"
+ calls = 0
+
+ async def handler(*, prompt: str, prompt_name: str) -> dict[str, object]:
+ nonlocal calls
+ calls += 1
+ return {
+ "version": "1",
+ "slots": [
+ {
+ "source_id": source_id,
+ "source_quote": broad_quote,
+ "plane": "run",
+ "pinability": "shapeless_valid",
+ "antecedent_family": "undecidable",
+ }
+ ],
+ "datum_bindings": [
+ {
+ "criterion_index": 0,
+ "datum_field": "output_path",
+ "declined": False,
+ "source_id": source_id,
+ "source_quote": broad_quote,
+ }
+ ],
+ }
+
+ result = await produce_request_slots(request=request, handler=handler, timeout_seconds=1.0)
+
+ assert result.status == "failure"
+ assert result.attempts == 4
+ assert result.failure_kind == RequestSlotProducerFailureKind.INVALID_OUTPUT
+ assert calls == 4
+
+
def test_datum_binding_model_accepts_only_identity_and_new_source_evidence() -> None:
binding = RequestSlotDatumBindingDeclarationV1.model_validate(
{
diff --git a/tests/unit/test_download_file_action_handler.py b/tests/unit/test_download_file_action_handler.py
index 6bcf76a91..3b32b60ee 100644
--- a/tests/unit/test_download_file_action_handler.py
+++ b/tests/unit/test_download_file_action_handler.py
@@ -2244,7 +2244,8 @@ async def test_handle_action_download_in_flight_request_does_not_extend_custom_t
)
elapsed = time.monotonic() - started_at
- assert elapsed < 0.1
+ # Keep wall-clock bounds loose to tolerate scheduling jitter on shared CI runners.
+ assert elapsed < 0.5
assert results[-1].download_triggered is False
assert action.download_triggered is False
span_attrs = _download_wait_span_attrs(span_exporter)
diff --git a/tests/unit/test_workflow_copilot_completion_criteria_schema.py b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
index 8a6cd5c83..985ac7d91 100644
--- a/tests/unit/test_workflow_copilot_completion_criteria_schema.py
+++ b/tests/unit/test_workflow_copilot_completion_criteria_schema.py
@@ -10,7 +10,7 @@ import pytest
from sqlalchemy.ext.asyncio import AsyncEngine
from skyvern.forge import app
-from skyvern.forge.sdk.copilot.completion_criteria_store import StoredCriteriaSnapshot
+from skyvern.forge.sdk.copilot.completion_criteria_store import StoredCriteriaSnapshot, criteria_from_json
from skyvern.forge.sdk.db.base_alchemy_db import BaseAlchemyDB
from skyvern.forge.sdk.db.models import WorkflowCopilotCompletionCriteriaSetModel
from skyvern.forge.sdk.db.repositories.workflow_parameters import WorkflowParametersRepository
@@ -198,6 +198,60 @@ async def test_loader_rejects_present_authority_field_that_decode_would_normaliz
assert loaded.reason == "undecodable_v1_criteria"
+async def test_loader_rejects_malformed_neutral_boolean_authority_projection() -> None:
+ request_slot_id = "a" * 64
+ criteria = [
+ {
+ "id": request_slot_id,
+ "outcome": "The run reports whether public form exists.",
+ "antecedent_family": "unconditional",
+ "expected_output_shape": "goal_judgment_boolean",
+ "requested_output_evidence_source": "independent_run_evidence",
+ "kind": "outcome",
+ "classification_output_key": "public_form_exists",
+ "request_slot_id": request_slot_id,
+ "pinability": "shapeless_valid",
+ "mint_disposition": "decidable",
+ "requested_output_floor_rekeyed": True,
+ "floor_rekeyed_from_path": "output.login_only",
+ }
+ ]
+
+ loaded = await _load_latest(_make_row(criteria=criteria))
+
+ assert isinstance(loaded, NonAdoptableCriteriaSet)
+ assert loaded.reason == "undecodable_v1_criteria"
+
+
+async def test_loader_admits_exact_legacy_floor_marked_neutral_boolean_tuple() -> None:
+ request_slot_id = "a" * 64
+ criteria = [
+ {
+ "id": request_slot_id,
+ "outcome": "The run reports whether public form exists.",
+ "antecedent_family": "unconditional",
+ "expected_output_shape": "goal_judgment_boolean",
+ "requested_output_evidence_source": "independent_run_evidence",
+ "kind": "outcome",
+ "classification_output_key": "public_form_exists",
+ "request_slot_id": request_slot_id,
+ "pinability": "shapeless_valid",
+ "mint_disposition": "decidable",
+ "requested_output_floor_rekeyed": True,
+ "floor_rekeyed_from_path": "output.public_form_exists",
+ }
+ ]
+
+ loaded = await _load_latest(_make_row(criteria=criteria))
+
+ assert not isinstance(loaded, NonAdoptableCriteriaSet)
+ (criterion,) = criteria_from_json(loaded.criteria)
+ assert criterion.requested_output_floor_rekeyed is False
+ assert criterion.floor_rekeyed_from_path is None
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"criteria_value",