Skip to content

Automate browser based workflows with AI

SKY-11545: Fail closed requested output grounding (#6994)

Andrew Neilson86ae763d0ee3
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 01ec33343..43abffe8d 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -13,7 +13,7 @@ import yaml
from pydantic import BaseModel, ConfigDict, Field
from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
@@ -47,6 +47,7 @@ BuildTestOutcomeReasonCode = Literal[
"missing_structural_evidence",
"unchanged_after_recorded_outcome",
"metadata_reject",
+ "output_policy_reject",
]
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
@@ -781,6 +782,8 @@ def _missing_requested_output_facts(
for verdict in completion_verification.verdicts:
if verdict.satisfied or not verdict.output_path:
continue
+ if _has_presence_only_output_evidence(verdict):
+ continue
output_path = _bounded_ref(verdict.output_path)
output_root = _output_path_root(output_path)
if not output_root:
@@ -805,6 +808,15 @@ def _missing_requested_output_facts(
return sorted(facts, key=lambda item: str(item.get("output_path") or ""))
+def _has_presence_only_output_evidence(verdict: CriterionVerdict) -> bool:
+ return (
+ verdict.reason_code == "structurally_abstained"
+ and verdict.grounding_mode == "missing"
+ and isinstance(verdict.evidence_ref, str)
+ and bool(verdict.evidence_ref.strip())
+ )
+
+
def _output_path_root(output_path: str) -> str:
return _bounded_ref(output_path.split(".", 1)[0].split("[", 1)[0])
diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.py
index 030a60c3a..a345fc146 100644
--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py
+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py
@@ -55,11 +55,15 @@ def grade_requested_output_criteria(
snapshot: RunEvidenceSnapshot,
) -> list[CriterionVerdict]:
requested_path_aliases = schema_output_path_aliases_from_criteria(criteria)
- authored_paths = _authored_output_contract_paths(copilot_ctx, requested_path_aliases)
+ raw_authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx)
+ authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, requested_path_aliases)
verdicts: list[CriterionVerdict] = []
for criterion in criteria:
path = _normalize_output_path(criterion.output_path)
- if not path or path not in authored_paths:
+ accepted_labels = (
+ {label for label, paths in authored_paths_by_label.items() if path in paths} if path else set()
+ )
+ if not path or not accepted_labels:
verdicts.append(
CriterionVerdict(
criterion_id=criterion.id,
@@ -69,16 +73,24 @@ def grade_requested_output_criteria(
)
)
continue
+ accepted_runtime_outputs = dict(_accepted_runtime_outputs(snapshot.block_outputs, accepted_labels))
+ accepted_authored_paths = set().union(*(authored_paths_by_label[label] for label in accepted_labels))
+ accepted_raw_authored_paths = set().union(
+ *(raw_authored_paths_by_label.get(label, set()) for label in accepted_labels)
+ )
+ projection_roots = _runtime_projection_roots(accepted_authored_paths | accepted_raw_authored_paths)
expected_value = criterion.expected_output_value
expected_shape = criterion.expected_output_shape
grounding_mode = _criterion_grounding_mode(criterion)
trace_fields = _criterion_trace_fields(criterion, grounding_mode)
if expected_value is not None:
- match_state, evidence_ref = _runtime_output_path_match(snapshot.block_outputs, path, expected_value)
+ match_state, evidence_ref = _runtime_output_path_match(
+ accepted_runtime_outputs, path, expected_value, projection_roots
+ )
elif expected_shape is not None:
- match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)
+ match_state, evidence_ref = _runtime_output_path_presence(accepted_runtime_outputs, path, projection_roots)
else:
- match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)
+ match_state, evidence_ref = _runtime_output_path_presence(accepted_runtime_outputs, path, projection_roots)
if match_state == "present" and evidence_ref is not None:
verdicts.append(
CriterionVerdict(
@@ -87,8 +99,8 @@ def grade_requested_output_criteria(
reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,
evidence_ref=evidence_ref,
missing_evidence=(
- "requested-output field is present, but the criterion has no typed expected_output_value "
- "that can prove or refute the value"
+ "requested-output field is present, but the criterion lacks typed expected_output_value "
+ "or expected_output_shape to prove the value"
),
**trace_fields,
)
@@ -178,20 +190,28 @@ def _criterion_trace_fields(criterion: CompletionCriterion, grounding_mode: str)
}
-def _authored_output_contract_paths(copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None) -> set[str]:
+def _authored_output_contract_paths_by_label(
+ copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None
+) -> dict[str, set[str]]:
metadata = copilot_ctx.code_artifact_metadata
- if not isinstance(metadata, Mapping):
- return set()
code_by_label = _code_blocks_by_label(copilot_ctx)
- paths: set[str] = set()
- for label, artifact in metadata.items():
- if not isinstance(label, str) or not isinstance(artifact, Mapping):
- continue
- paths.update(_artifact_contract_paths(artifact))
- code = code_by_label.get(label)
- if code:
- paths.update(_static_return_paths(code))
- return _canonical_output_paths(paths, aliases)
+ paths_by_label: dict[str, set[str]] = {}
+ if isinstance(metadata, Mapping):
+ for label, artifact in metadata.items():
+ if not isinstance(label, str) or not isinstance(artifact, Mapping):
+ continue
+ paths = set(_artifact_contract_paths(artifact))
+ code = code_by_label.get(label)
+ if code:
+ paths.update(_static_return_paths(code))
+ canonical_paths = _canonical_output_paths(paths, aliases)
+ if canonical_paths:
+ paths_by_label[label] = canonical_paths
+ for label, code in code_by_label.items():
+ static_paths = _canonical_output_paths(_static_return_paths(code), aliases)
+ if static_paths:
+ paths_by_label.setdefault(label, set()).update(static_paths)
+ return paths_by_label
def _canonical_output_paths(paths: set[str], aliases: dict[str, str] | None) -> set[str]:
@@ -303,6 +323,21 @@ def _static_return_paths(code: str) -> set[str]:
return {key for key in (_top_level_return_dict_keys(code) or set()) if key}
+def _runtime_projection_roots(authored_paths: set[str]) -> set[str]:
+ return {parts[0] for path in authored_paths if len(parts := _path_parts(path)) > 1}
+
+
+def _accepted_runtime_outputs(block_outputs: Mapping[str, Any], accepted_labels: set[str]) -> list[tuple[str, Any]]:
+ if not accepted_labels:
+ return []
+ accepted_output_keys = {f"{label}_output" for label in accepted_labels}
+ return [
+ (label, payload)
+ for label, payload in block_outputs.items()
+ if label in accepted_labels or label in accepted_output_keys
+ ]
+
+
def _top_level_return_dict_keys(code: str) -> set[str] | None:
wrapped = "async def __copilot_block__():\n" + textwrap.indent(textwrap.dedent(code).strip() or "pass", " ")
try:
@@ -334,14 +369,14 @@ def _dict_keys(node: ast.Dict) -> set[str]:
def _runtime_output_path_match(
- block_outputs: Mapping[str, Any], path: str, expected_value: str | None
+ block_outputs: Mapping[str, Any], path: str, expected_value: str | None, projection_roots: set[str] | None = None
) -> tuple[str, str | None]:
parts = _path_parts(path)
if not parts or expected_value is None:
return "missing", None
contradicted_ref: str | None = None
for label, payload in block_outputs.items():
- values, source_path = _runtime_path_values(payload, parts, path)
+ values, source_path = _runtime_path_values(payload, parts, path, projection_roots or set())
if not values:
continue
evidence_ref = f"block_outputs:{label}.{source_path}"
@@ -353,12 +388,14 @@ def _runtime_output_path_match(
return "missing", None
-def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -> tuple[str, str | None]:
+def _runtime_output_path_presence(
+ block_outputs: Mapping[str, Any], path: str, projection_roots: set[str] | None = None
+) -> tuple[str, str | None]:
parts = _path_parts(path)
if not parts:
return "missing", None
for label, payload in block_outputs.items():
- values, source_path = _runtime_present_path_values(payload, parts, path)
+ values, source_path = _runtime_present_path_values(payload, parts, path, projection_roots or set())
if not values:
continue
evidence_ref = f"block_outputs:{label}.{source_path}"
@@ -366,7 +403,9 @@ def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -
return "missing", None
-def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:
+def _runtime_path_values(
+ payload: Any, parts: list[str], path: str, projection_roots: set[str]
+) -> tuple[list[Any], str]:
canonical_values = _path_values(payload, parts)
if canonical_values:
filtered_canonical_values = [value for value in canonical_values if _is_meaningful_requested_value(value)]
@@ -374,10 +413,22 @@ def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[lis
wrapped_parts = ["output", *parts]
wrapped_path = ".".join(wrapped_parts)
wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]
- return wrapped_values, wrapped_path
-
-
-def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:
+ if wrapped_values:
+ return wrapped_values, wrapped_path
+ for root in sorted(projection_roots):
+ projected_parts = [root, *parts]
+ projected_path = ".".join(projected_parts)
+ projected_values = [
+ value for value in _path_values(payload, projected_parts) if _is_meaningful_requested_value(value)
+ ]
+ if projected_values:
+ return projected_values, projected_path
+ return [], wrapped_path
+
+
+def _runtime_present_path_values(
+ payload: Any, parts: list[str], path: str, projection_roots: set[str]
+) -> tuple[list[Any], str]:
raw_canonical_values = _path_values(payload, parts)
if raw_canonical_values:
canonical_values = [value for value in raw_canonical_values if _is_meaningful_requested_value(value)]
@@ -385,7 +436,17 @@ def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> t
wrapped_parts = ["output", *parts]
wrapped_path = ".".join(wrapped_parts)
wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]
- return wrapped_values, wrapped_path
+ if wrapped_values:
+ return wrapped_values, wrapped_path
+ for root in sorted(projection_roots):
+ projected_parts = [root, *parts]
+ projected_path = ".".join(projected_parts)
+ projected_values = [
+ value for value in _path_values(payload, projected_parts) if _is_meaningful_requested_value(value)
+ ]
+ if projected_values:
+ return projected_values, projected_path
+ return [], wrapped_path
def _path_values(value: Any, parts: list[str]) -> list[Any]:
diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.py
index 672b618d8..f8a90bf92 100644
--- a/skyvern/forge/sdk/copilot/completion_verification.py
+++ b/skyvern/forge/sdk/copilot/completion_verification.py
@@ -155,9 +155,7 @@ class CompletionVerificationResult:
unmet = [
verdict
for verdict in self.verdicts
- if not verdict.satisfied
- and not self.is_structural_contingent_abstention(verdict)
- and not _is_structural_requested_output_abstention(verdict)
+ if not verdict.satisfied and not self.is_structural_contingent_abstention(verdict)
]
missing_evidence: list[str] = []
for verdict in unmet:
@@ -209,12 +207,7 @@ class CompletionVerificationResult:
evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)
if evidence_ref:
data[f"{prefix}_evidence_ref"] = evidence_ref
- detail = (
- None
- if self.is_structural_contingent_abstention(verdict)
- or _is_structural_requested_output_abstention(verdict)
- else verdict_missing_evidence(verdict)
- )
+ detail = None if self.is_structural_contingent_abstention(verdict) else verdict_missing_evidence(verdict)
if detail:
data[f"{prefix}_missing_evidence"] = detail
return data
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 01956dfc1..840502411 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -207,6 +207,16 @@ _OUTPUT_FIELD_CONNECTOR_RE = re.compile(
r"\b(?:with|including|include|includes|containing|contains|fields?|result)\b[:\s]+",
re.I,
)
+_OUTPUT_NEGATED_INTENT_PREFIX_RE = re.compile(
+ r"(?:\b(?:do|does|did|should|must|can)\s+not\s+|\b(?:don't|doesn't|didn't|shouldn't|mustn't|can't|cannot|never|without)\s+)$",
+ re.I,
+)
+_OUTPUT_EXPLICIT_FIELD_KEY_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+_OUTPUT_NAMED_FIELD_RE = re.compile(
+ r"\b(?:(?:requested[-\s]+output|output|return(?:ed)?|final|structured)\s+)?fields?\s+"
+ r"(?:named|called)\s+([A-Za-z_][A-Za-z0-9_]*)\b",
+ re.I,
+)
_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")
_OUTPUT_FIELD_WORDS = frozenset(
"address addresses date dates email emails id identifier identifiers license licenses location locations "
@@ -777,10 +787,12 @@ def normalized_criterion_outcome_key(outcome: str) -> str:
return collapsed.strip().lower().rstrip(".!?;:,").strip()
-def _output_intent_spans(user_message: str) -> list[str]:
+def _output_intent_spans(user_message: str, *, negated: bool = False) -> list[str]:
message = user_message or ""
spans: list[str] = []
for match in _OUTPUT_INTENT_RE.finditer(message):
+ if _output_intent_is_negated(message, match.start()) != negated:
+ continue
tail = message[match.start() :]
span = tail[: _output_span_end_index(tail)]
span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")
@@ -789,6 +801,10 @@ def _output_intent_spans(user_message: str) -> list[str]:
return spans
+def _output_intent_is_negated(message: str, match_start: int) -> bool:
+ return bool(_OUTPUT_NEGATED_INTENT_PREFIX_RE.search(message[max(0, match_start - 32) : match_start]))
+
+
def _output_span_end_index(text: str) -> int:
for index, char in enumerate(text):
if _OUTPUT_SPAN_END_RE.fullmatch(char):
@@ -857,6 +873,9 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | No
if not candidate:
return None
+ named_field = _OUTPUT_NAMED_FIELD_RE.search(candidate)
+ if named_field is not None:
+ return named_field.group(1)
connector_matches = list(_OUTPUT_FIELD_CONNECTOR_RE.finditer(candidate))
if connector_matches:
candidate = candidate[connector_matches[-1].end() :]
@@ -864,6 +883,8 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | No
candidate = " ".join(re.sub(r"[^A-Za-z0-9 _/-]+", " ", candidate).split()).strip(" :-")
if not candidate:
return None
+ if "_" in candidate and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(candidate):
+ return candidate
normalized_aliases = _normalize_requested_output_aliases(aliases)
if " ".join(_word_tokens(candidate)) in normalized_aliases:
return candidate
@@ -897,6 +918,8 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | No
normalized_words = normalized_words[1:]
if not normalized_words:
return None
+ if len(words) == 1 and "_" in words[0] and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(words[0]):
+ return words[0]
if any(word in _OUTPUT_METHOD_WORDS for word in normalized_words):
return None
if all(word in _OUTPUT_GENERIC_WORDS for word in normalized_words):
@@ -932,10 +955,17 @@ def _matched_alias_phrase(candidate: str, alias_key: str) -> str | None:
return match.group(0).strip() if match is not None else None
-def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:
+def _requested_output_fields(
+ user_message: str, aliases: dict[str, str] | None = None, *, negated: bool = False
+) -> list[str]:
fields: list[str] = []
seen: set[str] = set()
- for span in _output_intent_spans(user_message):
+ for span in _output_intent_spans(user_message, negated=negated):
+ if not negated:
+ for match in _OUTPUT_INTENT_RE.finditer(span):
+ if _output_intent_is_negated(span, match.start()):
+ span = span[: match.start()]
+ break
for segment in _OUTPUT_SPLIT_RE.split(span):
field_name = _clean_requested_output_candidate(segment, aliases)
if field_name is None:
@@ -948,6 +978,18 @@ def _requested_output_fields(user_message: str, aliases: dict[str, str] | None =
return fields
+def _requested_output_path_for_detected_field(
+ field_name: str, schema_aliases: dict[str, str], config_aliases: dict[str, str]
+) -> str:
+ if "_" in field_name and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(field_name.strip()):
+ return requested_output_path_for_field(field_name)
+ return (
+ lookup_requested_output_path_alias(field_name, schema_aliases)
+ or lookup_requested_output_path_alias(field_name, config_aliases)
+ or requested_output_path_for_field(field_name)
+ )
+
+
def requested_output_path_for_field(field_name: str, aliases: dict[str, str] | None = None) -> str:
alias_path = lookup_requested_output_path_alias(field_name, aliases)
if alias_path is not None:
@@ -1090,9 +1132,15 @@ def _preserve_after_requested_output_canonicalization(
criterion: CompletionCriterion,
requested_fields: list[str],
requested_output_paths: set[str],
+ forbidden_fields: list[str] | None = None,
+ forbidden_output_paths: set[str] | None = None,
) -> bool:
if criterion.level == "definition" or criterion.method_mandated:
return True
+ if criterion.output_path in (forbidden_output_paths or set()) or _criterion_text_covers_any_requested_output(
+ criterion, forbidden_fields or []
+ ):
+ return False
if criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(
criterion, requested_fields
):
@@ -1115,24 +1163,26 @@ def _apply_requested_output_completion_criteria(
schema_aliases = _normalize_requested_output_aliases(schema_aliases)
detection_aliases = {**config_aliases, **schema_aliases}
requested_fields = _requested_output_fields(user_message, detection_aliases)
- if not requested_fields:
+ forbidden_fields = _requested_output_fields(user_message, detection_aliases, negated=True)
+ if not requested_fields and not forbidden_fields:
return
requested_specs: list[tuple[str, str, str]] = []
requested_output_paths: set[str] = set()
for field_name in requested_fields:
- output_path = (
- lookup_requested_output_path_alias(field_name, schema_aliases)
- or lookup_requested_output_path_alias(field_name, config_aliases)
- or requested_output_path_for_field(field_name)
- )
+ output_path = _requested_output_path_for_detected_field(field_name, schema_aliases, config_aliases)
field_label = _requested_output_field_label(field_name, output_path)
if output_path in requested_output_paths:
continue
requested_output_paths.add(output_path)
requested_specs.append((field_name, output_path, field_label))
- if not requested_output_paths:
+ forbidden_output_paths = {
+ _requested_output_path_for_detected_field(field_name, schema_aliases, config_aliases)
+ for field_name in forbidden_fields
+ }
+
+ if not requested_output_paths and not forbidden_output_paths:
return
value_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)
@@ -1165,7 +1215,13 @@ def _apply_requested_output_completion_criteria(
preserved_criteria = [
criterion
for criterion in policy.completion_criteria
- if _preserve_after_requested_output_canonicalization(criterion, requested_fields, requested_output_paths)
+ if _preserve_after_requested_output_canonicalization(
+ criterion,
+ requested_fields,
+ requested_output_paths,
+ forbidden_fields,
+ forbidden_output_paths,
+ )
]
canonical_requested_criteria = [
CompletionCriterion(
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index f63488829..ddcde437c 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -11,7 +11,7 @@ import tokenize
from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
-from typing import Annotated, Any, Literal, NamedTuple
+from typing import Annotated, Any, Literal, NamedTuple, cast
from urllib.parse import urlsplit
import structlog
@@ -1432,98 +1432,89 @@ def _candidate_missing_required_output_roots(
code_artifact_metadata: object,
*,
required_roots: set[str],
- runtime_produced_roots: set[str] | None = None,
) -> list[str]:
if not required_roots:
return []
- declared_roots = set(_metadata_output_path_roots(code_artifact_metadata))
- produced_roots = _workflow_yaml_produced_output_roots(workflow_yaml)
- covered_roots = (declared_roots & produced_roots) | (runtime_produced_roots or set())
- return sorted(required_roots - covered_roots)
-
-
-def _recorded_runtime_produced_output_roots(ctx: AgentContext, workflow_yaml: str) -> set[str]:
- verified_outputs = getattr(ctx, "verified_block_outputs", None)
- if not isinstance(verified_outputs, Mapping):
- return set()
- code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- roots: set[str] = set()
- for label, output in verified_outputs.items():
- if not isinstance(label, str) or label not in code_block_labels:
+ declared_roots_by_label = {
+ label: set(roots) for label, roots in _metadata_output_path_roots_by_label(code_artifact_metadata).items()
+ }
+ produced_by_label = _workflow_yaml_produced_output_roots_by_label(workflow_yaml)
+ covered_roots: set[str] = set()
+ abstained_declared_roots: set[str] = set()
+ for label, declared_roots in declared_roots_by_label.items():
+ produced = produced_by_label.get(label)
+ if produced is None:
continue
- roots.update(_meaningful_runtime_output_roots(output))
- return roots
-
-
-def _meaningful_runtime_output_roots(value: object, *, prefix: str = "") -> set[str]:
- roots: set[str] = set()
- if isinstance(value, Mapping):
- for raw_key, child in value.items():
- if not isinstance(raw_key, str):
- continue
- key = raw_key.strip()
- if not key or key == "evidence_text" or not _is_structural_runtime_output_key(key):
- continue
- path = f"{prefix}.{key}" if prefix else key
- if _runtime_output_value_is_meaningful(child):
- roots.add(path.split(".", 1)[0])
- roots.update(_meaningful_runtime_output_roots(child, prefix=path))
- return roots
- if isinstance(value, list):
- for item in value:
- roots.update(_meaningful_runtime_output_roots(item, prefix=prefix))
- return roots
+ covered_roots.update(declared_roots & produced.roots)
+ if produced.abstained:
+ abstained_declared_roots.update(declared_roots)
+ missing_roots = required_roots - covered_roots
+ if missing_roots:
+ missing_roots -= abstained_declared_roots
+ return sorted(missing_roots)
-def _runtime_output_value_is_meaningful(value: object) -> bool:
- if value is None:
- return False
- if isinstance(value, str):
- return bool(value.strip())
- if isinstance(value, Mapping):
- return any(_runtime_output_value_is_meaningful(item) for item in value.values())
- if isinstance(value, list):
- return any(_runtime_output_value_is_meaningful(item) for item in value)
- return True
+class _ProducedOutputRoots(NamedTuple):
+ roots: set[str]
+ abstained: bool = False
-def _workflow_yaml_produced_output_roots(workflow_yaml: str) -> set[str]:
- roots: set[str] = set()
- for block in _workflow_yaml_code_blocks_by_label(workflow_yaml).values():
- roots.update(_code_block_produced_output_roots(str(block.get("code") or "")))
- return roots
+def _workflow_yaml_produced_output_roots_by_label(workflow_yaml: str) -> dict[str, _ProducedOutputRoots]:
+ return {
+ label: _code_block_produced_output_roots(str(block.get("code") or ""))
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items()
+ }
-def _code_block_produced_output_roots(code: str) -> set[str]:
+def _code_block_produced_output_roots(code: str) -> _ProducedOutputRoots:
try:
tree = ast.parse(textwrap.dedent(code).strip() or "pass")
except SyntaxError:
- return set()
+ return _ProducedOutputRoots(set(), True)
scope_statements = list(_iter_top_level_scope(tree.body))
assigned_roots = _assigned_top_level_names(tree.body)
dict_assignments: dict[str, set[str]] = {}
+ dynamic_dict_assignment_names: set[str] = set()
helper_return_roots = _helper_function_literal_return_roots(tree.body)
returned_roots: set[str] = set()
+ abstained = False
+ saw_return = False
for node in scope_statements:
if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
- dict_roots = _dict_literal_string_key_roots(node.value)
if isinstance(node.value, ast.Dict):
- dict_assignments[node.targets[0].id] = dict_roots
+ produced = _dict_literal_output_roots(node.value)
+ dict_assignments[node.targets[0].id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(node.targets[0].id)
elif isinstance(node, ast.Assign):
- _apply_literal_dict_key_assignment(dict_assignments, node)
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
elif isinstance(node, ast.AnnAssign):
if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):
- dict_assignments[node.target.id] = _dict_literal_string_key_roots(node.value)
- _apply_literal_dict_key_assignment(dict_assignments, node)
+ produced = _dict_literal_output_roots(node.value)
+ dict_assignments[node.target.id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(node.target.id)
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
elif isinstance(node, ast.AugAssign):
- _apply_literal_dict_key_assignment(dict_assignments, node)
- elif isinstance(node, ast.Return) and node.value is not None:
- returned_roots.update(_return_output_roots(node.value, dict_assignments, helper_return_roots))
- return returned_roots or assigned_roots
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
+ elif isinstance(node, ast.Return):
+ saw_return = True
+ if node.value is not None:
+ returned = _return_output_roots(
+ node.value,
+ dict_assignments,
+ dynamic_dict_assignment_names,
+ helper_return_roots,
+ )
+ returned_roots.update(returned.roots)
+ abstained = abstained or returned.abstained
+ roots = returned_roots if saw_return else assigned_roots
+ return _ProducedOutputRoots(roots, abstained)
def _apply_literal_dict_key_assignment(
dict_assignments: dict[str, set[str]],
+ dynamic_dict_assignment_names: set[str],
node: ast.Assign | ast.AnnAssign | ast.AugAssign,
) -> None:
targets: list[ast.expr] = []
@@ -1533,13 +1524,22 @@ def _apply_literal_dict_key_assignment(
targets = [node.target]
for target in targets:
assignment = _literal_dict_key_assignment(target)
+ target_name = _dict_subscript_target_name(target)
if assignment is None:
+ if target_name in dict_assignments:
+ dynamic_dict_assignment_names.add(target_name)
continue
name, key = assignment
if name in dict_assignments:
dict_assignments[name].add(key)
+def _dict_subscript_target_name(target: ast.expr) -> str | None:
+ if isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name):
+ return target.value.id
+ return None
+
+
def _literal_dict_key_assignment(target: ast.expr) -> tuple[str, str] | None:
if not isinstance(target, ast.Subscript) or not isinstance(target.value, ast.Name):
return None
@@ -1552,52 +1552,103 @@ def _literal_dict_key_assignment(target: ast.expr) -> tuple[str, str] | None:
def _return_output_roots(
node: ast.expr,
dict_assignments: Mapping[str, set[str]],
- helper_return_roots: Mapping[str, set[str]],
-) -> set[str]:
+ dynamic_dict_assignment_names: set[str],
+ helper_return_roots: Mapping[str, _ProducedOutputRoots],
+) -> _ProducedOutputRoots:
if isinstance(node, ast.Await):
- return _return_output_roots(node.value, dict_assignments, helper_return_roots)
+ return _return_output_roots(node.value, dict_assignments, dynamic_dict_assignment_names, helper_return_roots)
if isinstance(node, ast.Name):
- return set(dict_assignments.get(node.id, set()))
+ if node.id in dict_assignments:
+ return _ProducedOutputRoots(
+ set(dict_assignments.get(node.id, set())), node.id in dynamic_dict_assignment_names
+ )
+ return _ProducedOutputRoots(set(), True)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
- return set(helper_return_roots.get(node.func.id, set()))
- return _dict_literal_string_key_roots(node)
-
-
-def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, set[str]]:
- helpers: dict[str, set[str]] = {}
+ return helper_return_roots.get(node.func.id, _ProducedOutputRoots(set(), True))
+ if isinstance(node, ast.Dict):
+ return _dict_literal_output_roots(node)
+ if isinstance(node, ast.List):
+ return _list_literal_output_roots(node)
+ if isinstance(node, (ast.Constant, ast.Tuple, ast.Set)):
+ return _ProducedOutputRoots(set(), False)
+ return _ProducedOutputRoots(set(), True)
+
+
+def _list_literal_output_roots(node: ast.List) -> _ProducedOutputRoots:
+ if not node.elts:
+ return _ProducedOutputRoots(set(), False)
+ if all(isinstance(element, ast.Dict) for element in node.elts):
+ roots: set[str] = set()
+ abstained = False
+ for element in node.elts:
+ produced = _dict_literal_output_roots(cast(ast.Dict, element))
+ roots.update(produced.roots)
+ abstained = abstained or produced.abstained
+ return _ProducedOutputRoots(roots, abstained)
+ if all(isinstance(element, ast.Constant) for element in node.elts):
+ return _ProducedOutputRoots(set(), False)
+ return _ProducedOutputRoots(set(), True)
+
+
+def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, _ProducedOutputRoots]:
+ helpers: dict[str, _ProducedOutputRoots] = {}
for statement in statements:
if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
dict_assignments: dict[str, set[str]] = {}
+ dynamic_dict_assignment_names: set[str] = set()
roots: set[str] = set()
+ abstained = False
+ saw_return = False
for helper_statement in _iter_top_level_scope(statement.body):
if (
isinstance(helper_statement, ast.Assign)
and len(helper_statement.targets) == 1
and isinstance(helper_statement.targets[0], ast.Name)
):
- dict_roots = _dict_literal_string_key_roots(helper_statement.value)
if isinstance(helper_statement.value, ast.Dict):
- dict_assignments[helper_statement.targets[0].id] = dict_roots
+ produced = _dict_literal_output_roots(helper_statement.value)
+ dict_assignments[helper_statement.targets[0].id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(helper_statement.targets[0].id)
elif isinstance(helper_statement, ast.Assign):
- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
elif isinstance(helper_statement, ast.AnnAssign):
if isinstance(helper_statement.target, ast.Name) and isinstance(helper_statement.value, ast.Dict):
- dict_assignments[helper_statement.target.id] = _dict_literal_string_key_roots(
- helper_statement.value
- )
- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ produced = _dict_literal_output_roots(helper_statement.value)
+ dict_assignments[helper_statement.target.id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(helper_statement.target.id)
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
elif isinstance(helper_statement, ast.AugAssign):
- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
- elif isinstance(helper_statement, ast.Return) and helper_statement.value is not None:
- roots.update(_dict_literal_string_key_roots(helper_statement.value))
- if isinstance(helper_statement.value, ast.Name):
- roots.update(dict_assignments.get(helper_statement.value.id, set()))
- if roots:
- helpers[statement.name] = roots
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)
+ elif isinstance(helper_statement, ast.Return):
+ saw_return = True
+ if helper_statement.value is not None:
+ returned = _return_output_roots(
+ helper_statement.value,
+ dict_assignments,
+ dynamic_dict_assignment_names,
+ {},
+ )
+ roots.update(returned.roots)
+ abstained = abstained or returned.abstained
+ if roots or abstained or saw_return:
+ helpers[statement.name] = _ProducedOutputRoots(roots, abstained)
return helpers
+def _dict_literal_output_roots(node: ast.Dict) -> _ProducedOutputRoots:
+ roots: set[str] = set()
+ abstained = False
+ for key in node.keys:
+ if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.strip():
+ roots.add(key.value.strip())
+ else:
+ abstained = True
+ return _ProducedOutputRoots(roots, abstained)
+
+
def _dict_literal_string_key_roots(node: ast.expr) -> set[str]:
if not isinstance(node, ast.Dict):
return set()
@@ -1629,13 +1680,14 @@ def _code_block_has_meaningful_output(code: str, artifact: object) -> bool:
tree = ast.parse(code)
except SyntaxError:
return False
+ helper_return_roots = _helper_function_literal_return_roots(tree.body)
class ReturnVisitor(ast.NodeVisitor):
def __init__(self) -> None:
self.has_meaningful_return = False
def visit_Return(self, node: ast.Return) -> None:
- if _return_value_is_meaningful(node.value):
+ if _return_value_is_meaningful(node.value, helper_return_roots):
self.has_meaningful_return = True
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
@@ -1838,11 +1890,21 @@ def _code_block_has_top_level_return(tree: ast.AST) -> bool:
return any(isinstance(statement, ast.Return) for statement in _iter_top_level_scope(body))
-def _return_value_is_meaningful(node: ast.expr | None) -> bool:
+def _return_value_is_meaningful(
+ node: ast.expr | None,
+ helper_return_roots: Mapping[str, _ProducedOutputRoots],
+) -> bool:
if node is None:
return False
+ if isinstance(node, ast.Await):
+ return _return_value_is_meaningful(node.value, helper_return_roots)
if isinstance(node, ast.Constant) and node.value is None:
return False
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+ produced = helper_return_roots.get(node.func.id)
+ if produced is not None:
+ return bool(produced.roots) or produced.abstained
+ return True
if (
isinstance(node, (ast.Dict, ast.List, ast.Tuple, ast.Set))
and not getattr(node, "elts", None)
@@ -5089,16 +5151,25 @@ async def _update_workflow(
tool_arguments=params,
)
if not output_policy_verdict.allowed:
+ output_policy_trace_data = output_policy_verdict_to_trace_data(
+ output_policy_verdict,
+ surface="tool_body",
+ tool_name="update_workflow",
+ )
+ output_policy_error = format_output_policy_tool_error(output_policy_verdict)
_record_code_only_raw_secret_reject_span(ctx, output_policy_verdict)
LOG.info(
"copilot output policy tool body verdict",
- **output_policy_verdict_to_trace_data(
- output_policy_verdict,
- surface="tool_body",
- tool_name="update_workflow",
- ),
+ **output_policy_trace_data,
)
- return reject(error=format_output_policy_tool_error(output_policy_verdict))
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="output_policy_reject",
+ summary=output_policy_error,
+ structural_payload=output_policy_trace_data,
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(error=output_policy_error)
# Prefer the most-recent in-turn emission so cross-path flows (inline
# REPLACE_WORKFLOW followed by update_workflow) compare against what the
@@ -5183,10 +5254,7 @@ async def _update_workflow(
recorded_missing_output_roots = (
_recorded_outcome_missing_output_roots(ctx) if recorded_output_roots_required else set()
)
- runtime_produced_output_roots = (
- _recorded_runtime_produced_output_roots(ctx, workflow_yaml) if recorded_output_roots_required else set()
- )
- unresolved_recorded_output_roots = recorded_missing_output_roots - runtime_produced_output_roots
+ unresolved_recorded_output_roots = recorded_missing_output_roots
output_empty_labels = (
_output_empty_code_block_labels(workflow_yaml, getattr(ctx, "code_artifact_metadata", None))
@@ -5226,7 +5294,6 @@ async def _update_workflow(
workflow_yaml,
getattr(ctx, "code_artifact_metadata", None),
required_roots=unresolved_recorded_output_roots,
- runtime_produced_roots=runtime_produced_output_roots,
)
if recorded_output_roots_required
else []
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index c53c9930e..b95178b9e 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -369,6 +369,105 @@ def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() ->
assert outcome.is_authoritative is False
+def test_outcome_not_demonstrated_does_not_mark_presence_only_abstention_as_missing_output() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_top_post",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": "extract_top_hn_post",
+ "status": "completed",
+ "extracted_data": {"output": {"top_post": "Claude Sonnet 5"}},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["top_post"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="top_post",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.top_post",
+ grounding_mode="missing",
+ evidence_ref="block_outputs:extract_top_hn_post.output.top_post",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_top_post",
+ ),
+ completion_verification=verification,
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.is_authoritative is True
+ assert outcome.missing_requested_output_facts == []
+
+
+def test_outcome_not_demonstrated_keeps_missing_fact_for_absent_requested_output() -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_no_top_post",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": "extract_top_hn_post",
+ "status": "completed",
+ "extracted_data": {"output": {}},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["top_post"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="top_post",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.top_post",
+ grounding_mode="missing",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_no_top_post",
+ ),
+ completion_verification=verification,
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "criterion_id": "top_post",
+ "output_path": "output.top_post",
+ "output_root": "output",
+ "reason_code": "no_evidence",
+ "value_status": "no_typed_value",
+ "grounding_mode": "missing",
+ }
+ ]
+
+
def test_authoring_repair_context_produces_structural_recorded_outcome() -> None:
context = CodeAuthoringRepairContext(
block_label="search_records",
@@ -556,6 +655,40 @@ def test_metadata_reject_key_uses_typed_fields_not_wording() -> None:
assert changed_missing_field.structural_key != first.structural_key
+def test_output_policy_reject_key_uses_stable_trace_payload() -> None:
+ payload = {
+ "surface": "tool_body",
+ "tool_name": "update_workflow",
+ "allowed": False,
+ "output_kind": "workflow_update_proposal",
+ "reason_codes": ["raw_secret_leak"],
+ }
+ first = recorded_outcome_from_author_time_reject(
+ reason_code="output_policy_reject",
+ structural_payload=payload,
+ observed_evidence_summary="Output policy blocked this Copilot output before persistence.",
+ )
+ same_structure = recorded_outcome_from_author_time_reject(
+ reason_code="output_policy_reject",
+ structural_payload=dict(payload),
+ observed_evidence_summary="Different wording for the same output-policy reject.",
+ )
+ changed_reason = recorded_outcome_from_author_time_reject(
+ reason_code="output_policy_reject",
+ structural_payload={**payload, "reason_codes": ["unapproved_credential_reference"]},
+ )
+ changed_surface = recorded_outcome_from_author_time_reject(
+ reason_code="output_policy_reject",
+ structural_payload={**payload, "surface": "final_response"},
+ )
+
+ assert first.reason_code == "output_policy_reject"
+ assert first.is_authoritative is True
+ assert first.structural_key == same_structure.structural_key
+ assert changed_reason.structural_key != first.structural_key
+ assert changed_surface.structural_key != first.structural_key
+
+
def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:
result = {
"ok": False,
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 02b2eaa60..95f98d213 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -128,6 +128,17 @@ _SAFE_EXTRACTION_CODE_YAML = _yaml(
"""
)
+_RAW_SECRET_OUTPUT_POLICY_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: navigation
+ label: login
+ navigation_goal: Type password: hunter2 into the password field.
+ """
+)
+
def _code_yaml(
code: str,
@@ -1161,61 +1172,972 @@ class TestCodeRepairProgressClassification:
authored_structure_signature=signature,
)
- first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
- first_key = ctx.latest_recorded_build_test_outcome.structural_key
- first_count = ctx.code_authoring_guardrail_reject_count
- second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+ first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ first_count = ctx.code_authoring_guardrail_reject_count
+ second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert first_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
+ assert first_count == 1
+ assert ctx.code_authoring_guardrail_reject_count == 2
+
+ @pytest.mark.asyncio
+ async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
+ changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ authored_structure_signature=signature,
+ )
+
+ result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == changed_yaml
+ assert ctx.has_staged_proposal is True
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
+ ],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: address_city_county_or_zip_code
+ blocks:
+ - block_type: code
+ label: {label}
+ parameter_keys:
+ - address_city_county_or_zip_code
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ await page.locator("#locInput").fill(str(address_city_county_or_zip_code))
+ return {{}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not return any keyed output" in result["error"]
+ assert ctx.has_staged_proposal is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_explicit_empty_return_despite_assigned_goal_root(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ ],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ npi = await page.locator("#npi").inner_text(timeout=5000)
+ return {{}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not return any keyed output" in result["error"]
+ assert ctx.has_staged_proposal is False
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.is_authoritative is True
+ assert outcome.block_labels == [label]
+ assert outcome.structural_failure_identity.startswith("author_time:")
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_known_empty_helper_without_missing_facts(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ ],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ def build():
+ return {{}}
+
+ return build()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not return any keyed output" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.block_labels == [label]
+ assert outcome.missing_requested_output_facts == []
+ assert outcome.structural_failure_identity.startswith("author_time:")
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
+ ],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ npi = "1234567890"
+ locations = [{{"address": "123 Main St"}}]
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ ],
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ address = "North Carolina, USA"
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ assert ctx.has_staged_proposal is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+
+ def test_required_output_roots_ignore_nested_debug_output_root_without_declared_root(self) -> None:
+ label = "extract_provider_profile"
+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["debug_output"]}]}}
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"debug_output": {{"npi": "1234567890"}}}}
+ """
+ )
+
+ assert workflow_update_module._candidate_missing_required_output_roots(
+ candidate_yaml,
+ metadata,
+ required_roots={"npi"},
+ ) == ["npi"]
+
+ def test_required_output_roots_ignore_same_label_undeclared_static_root(self) -> None:
+ label = "extract_provider_profile"
+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["address"]}]}}
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"npi": "1234567890"}}
+ """
+ )
+
+ assert workflow_update_module._candidate_missing_required_output_roots(
+ candidate_yaml,
+ metadata,
+ required_roots={"npi"},
+ ) == ["npi"]
+
+ def test_required_output_roots_ignore_static_root_without_label_metadata(self) -> None:
+ label = "extract_provider_profile"
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"npi": "1234567890"}}
+ """
+ )
+
+ assert workflow_update_module._candidate_missing_required_output_roots(
+ candidate_yaml,
+ {},
+ required_roots={"npi"},
+ ) == ["npi"]
+
+ def test_required_output_roots_accept_same_label_declared_static_root(self) -> None:
+ label = "extract_provider_profile"
+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["npi"]}]}}
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"npi": "1234567890"}}
+ """
+ )
+
+ assert (
+ workflow_update_module._candidate_missing_required_output_roots(
+ candidate_yaml,
+ metadata,
+ required_roots={"npi"},
+ )
+ == []
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_unprovable_dynamic_output_shape(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ result = {{}}
+ key = "npi"
+ value = await page.locator("#npi").inner_text(timeout=5000)
+ result[key] = value
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_literal_list_of_output_dicts(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return [{{"npi": "123"}}]
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_list_output_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ record = {{"npi": "123"}}
+ return [record]
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_list_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ key = "npi"
+ return [{{key: "123"}}]
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_local_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ key = "npi"
+ result = {{key: "123"}}
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ def test_local_dict_dynamic_key_preserves_literal_roots_while_abstaining(self) -> None:
+ produced = workflow_update_module._code_block_produced_output_roots(
+ """
+ key = "npi"
+ result = {"address": "North Carolina", key: "123"}
+ return result
+ """
+ )
+
+ assert produced.roots == {"address"}
+ assert produced.abstained is True
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_dynamic_key_local_dict_to_abstain(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ def build():
+ key = "npi"
+ result = {{key: "123"}}
+ return result
+
+ return build()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert all(
+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"
+ for fact in outcome.missing_requested_output_facts
+ )
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_unrelated_dynamic_output_shape(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ result = {{}}
+ key = "address"
+ result[key] = await page.locator("#address").inner_text(timeout=5000)
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "npi",
+ "output_root": "npi",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_when_unrelated_block_abstains(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ declared_label = "extract_provider_profile"
+ unrelated_label = "extract_unrelated_status"
+ ctx.code_artifact_metadata = {
+ declared_label: {
+ "block_label": declared_label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {declared_label}
+ code: |
+ return {{"address": "North Carolina"}}
+ - block_type: code
+ label: {unrelated_label}
+ code: |
+ result = {{}}
+ key = "status"
+ result[key] = await page.locator("#status").inner_text(timeout=5000)
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "npi",
+ "output_root": "npi",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_keeps_undeclared_missing_roots_with_dynamic_shape(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ {"output_path": "status", "output_root": "status", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ result = {{}}
+ key = "npi"
+ result[key] = await page.locator("#npi").inner_text(timeout=5000)
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert first["ok"] is False
- assert second["ok"] is False
- assert first_key is not None
- assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
- assert first_count == 1
- assert ctx.code_authoring_guardrail_reject_count == 2
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "status",
+ "output_root": "status",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
@pytest.mark.asyncio
- async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(
+ async def test_authoritative_outcome_not_demonstrated_rejects_literal_output_missing_required_root(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
- changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
verdict="repairable_failure",
reason_code="outcome_not_demonstrated",
- structural_failure_identity="completion:typed-outcome",
- authored_structure_signature=signature,
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"address": "North Carolina"}}
+ """
)
- result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
- assert ctx.workflow_yaml == changed_yaml
- assert ctx.has_staged_proposal is True
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.block_labels == [label]
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "npi",
+ "output_root": "npi",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(
+ async def test_authoritative_outcome_not_demonstrated_rejects_empty_return_despite_assigned_root(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi", "locations"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
- ],
- )
ctx.code_artifact_metadata = {
label: {
"block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
@@ -1226,56 +2148,47 @@ class TestCodeRepairProgressClassification:
reason_code="outcome_not_demonstrated",
structural_failure_identity="completion:unsatisfied-output",
authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
)
- partial_yaml = _yaml(
+ candidate_yaml = _yaml(
f"""
title: Provider lookup
workflow_definition:
- parameters:
- - parameter_type: workflow
- workflow_parameter_type: string
- key: address_city_county_or_zip_code
blocks:
- block_type: code
label: {label}
- parameter_keys:
- - address_city_county_or_zip_code
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- await page.locator("#locInput").fill(str(address_city_county_or_zip_code))
+ npi = await page.locator("#npi").inner_text(timeout=5000)
return {{}}
"""
)
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
assert "does not return any keyed output" in result["error"]
- assert ctx.has_staged_proposal is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.block_labels == [label]
+ assert outcome.missing_requested_output_facts == []
+ @pytest.mark.parametrize("return_expression", ["None", "[]", '["not keyed"]'])
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(
- self, monkeypatch: pytest.MonkeyPatch
+ async def test_authoritative_outcome_not_demonstrated_rejects_static_non_keyed_return_with_unrelated_output(
+ self, monkeypatch: pytest.MonkeyPatch, return_expression: str
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
- ctx.completion_verification_result = CompletionVerificationResult(
- status="evaluated",
- criterion_ids=["npi", "locations"],
- verdicts=[
- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
- ],
- )
+ declared_label = "extract_provider_profile"
+ unrelated_label = "extract_unrelated_status"
ctx.code_artifact_metadata = {
- label: {
- "block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ declared_label: {
+ "block_label": declared_label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
@@ -1286,6 +2199,9 @@ class TestCodeRepairProgressClassification:
reason_code="outcome_not_demonstrated",
structural_failure_identity="completion:unsatisfied-output",
authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
+ ],
)
candidate_yaml = _yaml(
f"""
@@ -1293,21 +2209,34 @@ class TestCodeRepairProgressClassification:
workflow_definition:
blocks:
- block_type: code
- label: {label}
+ label: {declared_label}
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- npi = "1234567890"
- locations = [{{"address": "123 Main St"}}]
+ return {return_expression}
+ - block_type: code
+ label: {unrelated_label}
+ code: |
+ return {{"status": "active"}}
"""
)
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "npi",
+ "output_root": "npi",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(
+ async def test_authoritative_outcome_not_demonstrated_rejects_helper_known_empty_output(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -1316,8 +2245,8 @@ class TestCodeRepairProgressClassification:
ctx.code_artifact_metadata = {
label: {
"block_label": label,
- "claimed_outcomes": [{"goal_value_paths": ["address"]}],
- "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],
}
}
ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
@@ -1329,17 +2258,10 @@ class TestCodeRepairProgressClassification:
structural_failure_identity="completion:unsatisfied-output",
authored_structure_signature="authored:previous-failed-candidate",
missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},
],
)
- partial_yaml = _yaml(
+ candidate_yaml = _yaml(
f"""
title: Provider lookup
workflow_definition:
@@ -1347,19 +2269,21 @@ class TestCodeRepairProgressClassification:
- block_type: code
label: {label}
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- address = "North Carolina, USA"
+ def build():
+ return {{}}
+
+ return build()
"""
)
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
- assert "does not cover the missing requested output roots" in result["error"]
- assert ctx.has_staged_proposal is False
- assert ctx.latest_recorded_build_test_outcome is not None
- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert "does not return any keyed output" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == []
@pytest.mark.asyncio
async def test_metadata_reject_persists_missing_roots_for_next_author_context(
@@ -1424,17 +2348,16 @@ class TestCodeRepairProgressClassification:
assert "value_status=no_typed_value" in rendered
@pytest.mark.asyncio
- async def test_authoritative_outcome_not_demonstrated_allows_when_last_run_produced_required_roots(
+ async def test_authoritative_outcome_not_demonstrated_rejects_when_only_last_run_produced_required_roots(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
+ label = "extract_top_hn_post"
ctx.verified_block_outputs[label] = {
- "address": "North Carolina, USA",
- "credentialing_status": "Active",
- "locations": [{"address": "North Carolina, USA"}],
- "statuses": ["Active"],
+ "top_post": "Claude Sonnet 5",
+ "rank": 1,
+ "url": "https://news.ycombinator.com/item?id=1",
}
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
@@ -1444,26 +2367,72 @@ class TestCodeRepairProgressClassification:
structural_failure_identity="completion:unsatisfied-output",
authored_structure_signature="authored:previous-failed-candidate",
missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},
],
)
candidate_yaml = _yaml(
f"""
- title: Provider lookup
+ title: Hacker News lookup
workflow_definition:
blocks:
- block_type: code
label: {label}
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- address = "North Carolina, USA"
+ return {{"rank": 1, "url": "https://news.ycombinator.com/item?id=1"}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not cover the missing requested output roots" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "top_post",
+ "output_root": "top_post",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_static_return_for_required_root(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "extract_top_hn_post"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["top_post"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["top_post"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Hacker News lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"top_post": "Claude Sonnet 5", "rank": 1}}
"""
)
@@ -4631,6 +5600,47 @@ class TestCodeAuthoringGuardrailChurnBackstop:
assert result["ok"] is True
assert ctx.code_authoring_guardrail_reject_count == 0
+ @pytest.mark.asyncio
+ async def test_output_policy_reject_records_author_time_reject_and_guardrail_count(self) -> None:
+ ctx = _code_only_ctx()
+
+ result = await _update_workflow(
+ {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert "raw_secret_leak" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.phase == "author_time_reject"
+ assert outcome.reason_code == "output_policy_reject"
+ assert outcome.is_authoritative is True
+ assert outcome.structural_key is not None
+ assert "Output policy blocked" in outcome.observed_evidence_summary
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert ctx.blocker_signal is None
+
+ @pytest.mark.asyncio
+ async def test_repeated_output_policy_reject_halts_at_guardrail_backstop(self) -> None:
+ ctx = _code_only_ctx()
+
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ result = await _update_workflow(
+ {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert result["ok"] is False
+
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ churn = ctx.blocker_signal
+ assert isinstance(churn, CopilotToolBlockerSignal)
+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ assert ctx.latest_tool_blocker_signal is churn
+ assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED
+
@pytest.mark.asyncio
async def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:
ctx = _code_only_ctx()
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index ebcc32dcf..63d791acf 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -2647,10 +2647,15 @@ def _requested_output_result(output: Any) -> dict:
def _metadata_for_requested_paths(*paths: str) -> dict[str, Any]:
return {
- "extract_profile": {
+ label: {
"claimed_outcomes": [{"goal_value_paths": list(paths)}],
"terminal_verifier_expectations": [{"goal_value_paths": list(paths)}],
}
+ for label in (
+ "extract_profile",
+ "utility_citrus_turn_on",
+ "utility_peach_gas_quickconnect",
+ )
}
@@ -4603,7 +4608,335 @@ async def test_requested_output_path_exact_runtime_field_with_expected_value_sat
assert "1234567890" not in repr(trace)
-def test_requested_output_path_without_expected_value_structurally_abstains_for_present_values() -> None:
+async def test_requested_output_typed_block_fields_do_not_admit_undeclared_returned_field(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("typed requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address", "credentialing_status", "locations")
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ ),
+ CompletionCriterion(
+ id="c_credentialing_status",
+ outcome="The returned profile includes credentialing status.",
+ output_path="output.credentialing_status",
+ expected_output_value="Active",
+ ),
+ CompletionCriterion(
+ id="c_locations",
+ outcome="The returned profile includes location address.",
+ output_path="output.locations.address",
+ expected_output_value="101 Example Ave",
+ ),
+ CompletionCriterion(
+ id="c_statuses",
+ outcome="The returned profile includes statuses.",
+ output_path="output.statuses",
+ expected_output_value="Active",
+ ),
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ _requested_output_result(
+ {
+ "address": "101 Example Ave",
+ "credentialing_status": "Active",
+ "locations": [{"address": "101 Example Ave"}],
+ "statuses": ["Active"],
+ }
+ ),
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is False
+ assert {verdict.criterion_id for verdict in verification.verdicts if verdict.satisfied} == {
+ "c_address",
+ "c_credentialing_status",
+ "c_locations",
+ }
+ statuses = next(verdict for verdict in verification.verdicts if verdict.criterion_id == "c_statuses")
+ assert statuses.state == "unsatisfied"
+ assert statuses.reason_code == "unproducible"
+ assert statuses.evidence_ref is None
+ assert {
+ verdict.evidence_ref for verdict in verification.verdicts if verdict.criterion_id in {"c_address", "c_statuses"}
+ } == {"block_outputs:extract_profile.address", None}
+
+
+def test_requested_output_nested_declared_output_root_maps_to_requested_output_path() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("profile.address")
+ criteria = [
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"profile": {"address": "101 Example Ave"}}}),
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_ref == "block_outputs:extract_profile.profile.address"
+
+
+ ("accepted_payload", "expected_reason", "expected_ref"),
+ [
+ ({"address": "Wrong Example Ave"}, "evidence_contradicts", "block_outputs:extract_profile.address"),
+ ({"status": "active"}, "missing_exact_field", None),
+ ],
+)
+def test_requested_output_ignores_unrelated_block_outputs_when_matching_values(
+ accepted_payload: dict[str, str],
+ expected_reason: str,
+ expected_ref: str | None,
+) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")
+ criteria = [
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ "extract_profile": accepted_payload,
+ "unrelated_lookup": {"address": "101 Example Ave"},
+ }
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == expected_reason
+ assert verdicts[0].evidence_ref == expected_ref
+
+
+def test_requested_output_evidence_text_with_expected_value_does_not_satisfy() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")
+ criteria = [
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"evidence_text": "Address: 101 Example Ave"}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+
+
+def test_requested_output_roots_do_not_satisfy_from_executed_block_output_without_static_metadata_roots() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ criteria = [
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The returned profile includes NPI.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ ),
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ ),
+ CompletionCriterion(
+ id="c_credentialing_status",
+ outcome="The returned profile includes credentialing status.",
+ output_path="output.credentialing_status",
+ expected_output_value="Active",
+ ),
+ CompletionCriterion(
+ id="c_locations",
+ outcome="The returned profile includes locations.",
+ output_path="output.locations.address",
+ expected_output_value="101 Example Ave",
+ ),
+ CompletionCriterion(
+ id="c_statuses",
+ outcome="The returned profile includes statuses.",
+ output_path="output.statuses",
+ expected_output_value="Active",
+ ),
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ executed_block_labels=["extract_profile"],
+ block_outputs={
+ "extract_profile": {
+ "npi": "1234567890",
+ "address": "101 Example Ave",
+ "credentialing_status": "Active",
+ "locations": [{"address": "101 Example Ave"}],
+ "statuses": ["Active"],
+ }
+ },
+ ),
+ )
+
+ assert {verdict.criterion_id for verdict in verdicts} == {
+ "c_npi",
+ "c_address",
+ "c_credentialing_status",
+ "c_locations",
+ "c_statuses",
+ }
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"unproducible"}
+ assert all(verdict.evidence_ref is None for verdict in verdicts)
+
+
+def test_requested_output_roots_without_expected_values_abstain_from_runtime_presence() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address", "statuses")
+ criteria = [
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ ),
+ CompletionCriterion(
+ id="c_statuses",
+ outcome="The returned profile includes statuses.",
+ output_path="output.statuses",
+ ),
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ executed_block_labels=["extract_profile"],
+ block_outputs={"extract_profile": {"address": "101 Example Ave", "statuses": ["Active"]}},
+ ),
+ )
+
+ assert {verdict.criterion_id for verdict in verdicts} == {"c_address", "c_statuses"}
+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}
+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}
+ assert {verdict.evidence_ref for verdict in verdicts} == {
+ "block_outputs:extract_profile.address",
+ "block_outputs:extract_profile.statuses",
+ }
+
+
+def test_requested_output_executed_root_admission_ignores_evidence_text_only_values() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ criteria = [
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The returned profile includes NPI.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ executed_block_labels=["extract_profile"],
+ block_outputs={"extract_profile": {"evidence_text": "The provider NPI is 1234567890."}},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "unproducible"
+
+
+def test_requested_output_executed_root_admission_ignores_unexecuted_block_outputs() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ criteria = [
+ CompletionCriterion(
+ id="c_address",
+ outcome="The returned profile includes address.",
+ output_path="output.address",
+ expected_output_value="101 Example Ave",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ executed_block_labels=["extract_profile"],
+ block_outputs={
+ "extract_profile": {"status": "Active"},
+ "unrelated_lookup": {"address": "101 Example Ave"},
+ },
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "unproducible"
+
+
+def test_requested_output_missing_metadata_and_runtime_field_fails_closed() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")
+ criteria = [
+ CompletionCriterion(
+ id="c_statuses",
+ outcome="The returned profile includes statuses.",
+ output_path="output.statuses",
+ expected_output_value="Active",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"address": "101 Example Ave"}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "unproducible"
+ assert verdicts[0].missing_evidence == "accepted code artifact metadata does not declare output.statuses"
+
+
+def test_requested_output_path_without_expected_value_abstains_for_present_typed_values() -> None:
ctx = _run_ctx()
ctx.code_artifact_metadata = _metadata_for_requested_paths(
"request_id",
@@ -4724,7 +5057,11 @@ def test_requested_output_shape_only_generated_fields_structurally_abstain_witho
assert trace["verdict_0_has_exact_value"] is False
assert "WTR-1842-DEMO" not in repr(trace)
assert trace["fully_satisfied"] is False
- assert trace["unmet_criterion_ids"] == []
+ assert trace["unmet_criterion_ids"] == [
+ "c_confirmation_number",
+ "c_account_number",
+ "c_selected_start_date",
+ ]
def test_requested_output_shape_abstains_on_p8_scrambled_values_and_ignores_evidence_text_status() -> None:
@@ -5750,6 +6087,165 @@ async def test_requested_output_path_can_use_static_return_key_contract(
assert verification.is_fully_satisfied() is True
+async def test_requested_output_path_can_use_static_return_contract_without_matching_metadata(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fail_handler(**_: object) -> object:
+ raise AssertionError("static requested-output evidence must bypass the judge")
+
+ _patch_completion_handler(monkeypatch, fail_handler)
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "story")
+ ctx.code_artifact_metadata = {
+ "stale_extract": _metadata_for_requested_paths("top_post_identified")["extract_profile"]
+ }
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: story
+ code: |
+ return {"top_post_identified": True}
+ """
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ CompletionCriterion(
+ id="c_top_post",
+ outcome="The top Hacker News post is identified.",
+ output_path="output.top_post_identified",
+ expected_output_value="true",
+ )
+ ]
+ )
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_hn",
+ "overall_status": "completed",
+ "executed_block_labels": ["story"],
+ "blocks": [
+ {
+ "label": "story",
+ "block_type": "CODE",
+ "status": "completed",
+ "extracted_data": {"top_post_identified": True},
+ }
+ ],
+ },
+ },
+ time.monotonic(),
+ )
+
+ assert verification is not None
+ assert verification.is_fully_satisfied() is True
+ assert verification.verdicts[0].evidence_ref == "block_outputs:story.top_post_identified"
+
+
+def test_requested_output_path_dynamic_return_without_metadata_does_not_admit_runtime_output() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: story
+ code: |
+ return await extract_top_post()
+ """
+ )
+ criteria = [
+ CompletionCriterion(
+ id="c_top_post",
+ outcome="The top Hacker News post is identified.",
+ output_path="output.top_post_identified",
+ expected_output_value="true",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"story": {"top_post_identified": True}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "unproducible"
+ assert verdicts[0].evidence_ref is None
+
+
+def test_requested_output_static_contract_scopes_runtime_evidence_to_same_label() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: story
+ code: |
+ return {"top_post_identified": False}
+ """
+ )
+ criteria = [
+ CompletionCriterion(
+ id="c_top_post",
+ outcome="The top Hacker News post is identified.",
+ output_path="output.top_post_identified",
+ expected_output_value="true",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"unrelated_lookup": {"top_post_identified": True}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_ref is None
+
+
+def test_requested_output_runtime_debug_output_does_not_create_projection_contract() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = {}
+ ctx.last_workflow_yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_profile
+ code: |
+ return {"npi": await page.locator("#npi").inner_text()}
+ """
+ )
+ criteria = [
+ CompletionCriterion(
+ id="c_npi",
+ outcome="The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"debug_output": {"npi": "1234567890"}}}),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_ref is None
+
+
@pytest.mark.asyncio
async def test_requested_output_path_can_use_static_list_return_key_contract(
monkeypatch: pytest.MonkeyPatch,
@@ -5832,7 +6328,7 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyte
@pytest.mark.asyncio
-async def test_present_generic_requested_output_without_expected_value_abstains_without_vetoing_run_success(
+async def test_present_generic_requested_output_without_expected_value_structurally_abstains(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def handler(**_: object) -> dict:
@@ -5865,14 +6361,21 @@ async def test_present_generic_requested_output_without_expected_value_abstains_
)
assert verification is not None
- assert verification.is_fully_satisfied() is True
verdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}
assert verdicts["c_customer_name"].reason_code == "structurally_abstained"
+ assert verdicts["c_customer_name"].satisfied is False
assert verdicts["c_submit"].reason_code == "evidence_confirms"
+ assert verification.is_fully_satisfied() is True
+ trace = verification.to_trace_data()
+ assert trace["unmet_criterion_ids"] == ["c_customer_name"]
+ assert trace["verdict_0_missing_evidence"] == (
+ "requested-output field is present, but the criterion lacks typed expected_output_value or "
+ "expected_output_shape to prove the value"
+ )
@pytest.mark.asyncio
-async def test_all_abstained_requested_outputs_do_not_satisfy_completion(
+async def test_present_requested_output_without_expected_value_cannot_satisfy_completion(
monkeypatch: pytest.MonkeyPatch,
) -> None:
async def fail_handler(**_: object) -> object:
@@ -6064,7 +6567,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
output = {"status": "DONE", "blocker": False}
else:
- output = {"status": "DONE", "blocker": False, "npi": "1234567890"}
+ output = {"status": "DONE", "blocker": False}
ctx.request_policy = RequestPolicy(
completion_criteria=[
_criterion(
@@ -6104,7 +6607,7 @@ async def test_fired_contingent_requested_output_miss_still_vetoes(
ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")
output = {"status": "DONE", "blocker": True}
else:
- output = {"status": "DONE", "blocker": True, "npi": "1234567890"}
+ output = {"status": "DONE", "blocker": True}
ctx.request_policy = RequestPolicy(
completion_criteria=[
_criterion(
diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.py
index d1bef12f1..d8548e427 100644
--- a/tests/unit/test_copilot_request_policy_completion_criteria.py
+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py
@@ -319,6 +319,54 @@ async def test_requested_output_criteria_preserve_stable_paths_without_derived_c
assert [criterion.expected_output_shape for criterion in canonical_subsets[0]] == [None, None]
+async def test_resale_docs_requested_output_uses_explicit_document_name_and_blocks_terminal_record_fields() -> None:
+ policy = await _policy_for_message(
+ (
+ "I need a reusable workflow that retrieves the resale demand document name from the mock ResaleDocs Hub "
+ "order-status page. The confirmation number should be a reusable input; for this eval run use "
+ "DEMO-RESALE-1842. The workflow should look up the order, open the order-level View / Download document "
+ "list, choose the highest-priority HOA demand or resale statement document row, and output one "
+ "requested-output field named document_name containing the selected row's visible document name. Do not "
+ "include an exact expected value for document_name in the workflow output contract, and do not output "
+ "confirmation_number, order_id, request_id, submission_id, or a terminal-record identifier."
+ ),
+ [
+ {
+ "outcome": "The returned record includes visible document name.",
+ "output_path": "output.visible_document_name",
+ },
+ {
+ "outcome": "The returned record includes terminal record identifier.",
+ "output_path": "output.terminal_record_identifier",
+ },
+ {
+ "outcome": "The returned record includes confirmation number.",
+ "output_path": "output.confirmation_number",
+ },
+ {
+ "outcome": "The returned record includes order id.",
+ "output_path": "output.order_id",
+ },
+ ],
+ )
+
+ assert [criterion.output_path for criterion in policy.completion_criteria] == ["output.document_name"]
+ assert policy.completion_criteria[0].expected_output_value is None
+ assert policy.completion_criteria[0].expected_output_shape is None
+
+
+async def test_named_customer_prose_does_not_become_requested_output_field_name() -> None:
+ policy = await _policy_for_message(
+ "Return a final record with customer named Acme and status.",
+ [],
+ )
+
+ assert not _criteria_for_path(policy, "output.acme")
+ assert _criteria_for_path(policy, "output.status")
+
+
@pytest.mark.asyncio
async def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:
policy = await _policy_for_message(