Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-11545: Fail closed requested output grounding (#6994)
Andrew Neilson86ae763d0ee3
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex 01ec33343..43abffe8d 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -13,7 +13,7 @@ import yamlfrom pydantic import BaseModel, ConfigDict, Fieldfrom skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE-from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdictfrom skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContextfrom skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence@@ -47,6 +47,7 @@ BuildTestOutcomeReasonCode = Literal["missing_structural_evidence","unchanged_after_recorded_outcome","metadata_reject",+ "output_policy_reject",]_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"@@ -781,6 +782,8 @@ def _missing_requested_output_facts(for verdict in completion_verification.verdicts:if verdict.satisfied or not verdict.output_path:continue+ if _has_presence_only_output_evidence(verdict):+ continueoutput_path = _bounded_ref(verdict.output_path)output_root = _output_path_root(output_path)if not output_root:@@ -805,6 +808,15 @@ def _missing_requested_output_facts(return sorted(facts, key=lambda item: str(item.get("output_path") or ""))+def _has_presence_only_output_evidence(verdict: CriterionVerdict) -> bool:+ return (+ verdict.reason_code == "structurally_abstained"+ and verdict.grounding_mode == "missing"+ and isinstance(verdict.evidence_ref, str)+ and bool(verdict.evidence_ref.strip())+ )++def _output_path_root(output_path: str) -> str:return _bounded_ref(output_path.split(".", 1)[0].split("[", 1)[0])diff --git a/skyvern/forge/sdk/copilot/completion_output_grounding.py b/skyvern/forge/sdk/copilot/completion_output_grounding.pyindex 030a60c3a..a345fc146 100644--- a/skyvern/forge/sdk/copilot/completion_output_grounding.py+++ b/skyvern/forge/sdk/copilot/completion_output_grounding.py@@ -55,11 +55,15 @@ def grade_requested_output_criteria(snapshot: RunEvidenceSnapshot,) -> list[CriterionVerdict]:requested_path_aliases = schema_output_path_aliases_from_criteria(criteria)- authored_paths = _authored_output_contract_paths(copilot_ctx, requested_path_aliases)+ raw_authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx)+ authored_paths_by_label = _authored_output_contract_paths_by_label(copilot_ctx, requested_path_aliases)verdicts: list[CriterionVerdict] = []for criterion in criteria:path = _normalize_output_path(criterion.output_path)- if not path or path not in authored_paths:+ accepted_labels = (+ {label for label, paths in authored_paths_by_label.items() if path in paths} if path else set()+ )+ if not path or not accepted_labels:verdicts.append(CriterionVerdict(criterion_id=criterion.id,@@ -69,16 +73,24 @@ def grade_requested_output_criteria())continue+ accepted_runtime_outputs = dict(_accepted_runtime_outputs(snapshot.block_outputs, accepted_labels))+ accepted_authored_paths = set().union(*(authored_paths_by_label[label] for label in accepted_labels))+ accepted_raw_authored_paths = set().union(+ *(raw_authored_paths_by_label.get(label, set()) for label in accepted_labels)+ )+ projection_roots = _runtime_projection_roots(accepted_authored_paths | accepted_raw_authored_paths)expected_value = criterion.expected_output_valueexpected_shape = criterion.expected_output_shapegrounding_mode = _criterion_grounding_mode(criterion)trace_fields = _criterion_trace_fields(criterion, grounding_mode)if expected_value is not None:- match_state, evidence_ref = _runtime_output_path_match(snapshot.block_outputs, path, expected_value)+ match_state, evidence_ref = _runtime_output_path_match(+ accepted_runtime_outputs, path, expected_value, projection_roots+ )elif expected_shape is not None:- match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)+ match_state, evidence_ref = _runtime_output_path_presence(accepted_runtime_outputs, path, projection_roots)else:- match_state, evidence_ref = _runtime_output_path_presence(snapshot.block_outputs, path)+ match_state, evidence_ref = _runtime_output_path_presence(accepted_runtime_outputs, path, projection_roots)if match_state == "present" and evidence_ref is not None:verdicts.append(CriterionVerdict(@@ -87,8 +99,8 @@ def grade_requested_output_criteria(reason_code=_STRUCTURAL_ABSTENTION_REASON_CODE,evidence_ref=evidence_ref,missing_evidence=(- "requested-output field is present, but the criterion has no typed expected_output_value "- "that can prove or refute the value"+ "requested-output field is present, but the criterion lacks typed expected_output_value "+ "or expected_output_shape to prove the value"),**trace_fields,)@@ -178,20 +190,28 @@ def _criterion_trace_fields(criterion: CompletionCriterion, grounding_mode: str)}-def _authored_output_contract_paths(copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None) -> set[str]:+def _authored_output_contract_paths_by_label(+ copilot_ctx: _GroundingCtx, aliases: dict[str, str] | None = None+) -> dict[str, set[str]]:metadata = copilot_ctx.code_artifact_metadata- if not isinstance(metadata, Mapping):- return set()code_by_label = _code_blocks_by_label(copilot_ctx)- paths: set[str] = set()- for label, artifact in metadata.items():- if not isinstance(label, str) or not isinstance(artifact, Mapping):- continue- paths.update(_artifact_contract_paths(artifact))- code = code_by_label.get(label)- if code:- paths.update(_static_return_paths(code))- return _canonical_output_paths(paths, aliases)+ paths_by_label: dict[str, set[str]] = {}+ if isinstance(metadata, Mapping):+ for label, artifact in metadata.items():+ if not isinstance(label, str) or not isinstance(artifact, Mapping):+ continue+ paths = set(_artifact_contract_paths(artifact))+ code = code_by_label.get(label)+ if code:+ paths.update(_static_return_paths(code))+ canonical_paths = _canonical_output_paths(paths, aliases)+ if canonical_paths:+ paths_by_label[label] = canonical_paths+ for label, code in code_by_label.items():+ static_paths = _canonical_output_paths(_static_return_paths(code), aliases)+ if static_paths:+ paths_by_label.setdefault(label, set()).update(static_paths)+ return paths_by_labeldef _canonical_output_paths(paths: set[str], aliases: dict[str, str] | None) -> set[str]:@@ -303,6 +323,21 @@ def _static_return_paths(code: str) -> set[str]:return {key for key in (_top_level_return_dict_keys(code) or set()) if key}+def _runtime_projection_roots(authored_paths: set[str]) -> set[str]:+ return {parts[0] for path in authored_paths if len(parts := _path_parts(path)) > 1}+++def _accepted_runtime_outputs(block_outputs: Mapping[str, Any], accepted_labels: set[str]) -> list[tuple[str, Any]]:+ if not accepted_labels:+ return []+ accepted_output_keys = {f"{label}_output" for label in accepted_labels}+ return [+ (label, payload)+ for label, payload in block_outputs.items()+ if label in accepted_labels or label in accepted_output_keys+ ]++def _top_level_return_dict_keys(code: str) -> set[str] | None:wrapped = "async def __copilot_block__():\n" + textwrap.indent(textwrap.dedent(code).strip() or "pass", " ")try:@@ -334,14 +369,14 @@ def _dict_keys(node: ast.Dict) -> set[str]:def _runtime_output_path_match(- block_outputs: Mapping[str, Any], path: str, expected_value: str | None+ block_outputs: Mapping[str, Any], path: str, expected_value: str | None, projection_roots: set[str] | None = None) -> tuple[str, str | None]:parts = _path_parts(path)if not parts or expected_value is None:return "missing", Nonecontradicted_ref: str | None = Nonefor label, payload in block_outputs.items():- values, source_path = _runtime_path_values(payload, parts, path)+ values, source_path = _runtime_path_values(payload, parts, path, projection_roots or set())if not values:continueevidence_ref = f"block_outputs:{label}.{source_path}"@@ -353,12 +388,14 @@ def _runtime_output_path_match(return "missing", None-def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -> tuple[str, str | None]:+def _runtime_output_path_presence(+ block_outputs: Mapping[str, Any], path: str, projection_roots: set[str] | None = None+) -> tuple[str, str | None]:parts = _path_parts(path)if not parts:return "missing", Nonefor label, payload in block_outputs.items():- values, source_path = _runtime_present_path_values(payload, parts, path)+ values, source_path = _runtime_present_path_values(payload, parts, path, projection_roots or set())if not values:continueevidence_ref = f"block_outputs:{label}.{source_path}"@@ -366,7 +403,9 @@ def _runtime_output_path_presence(block_outputs: Mapping[str, Any], path: str) -return "missing", None-def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:+def _runtime_path_values(+ payload: Any, parts: list[str], path: str, projection_roots: set[str]+) -> tuple[list[Any], str]:canonical_values = _path_values(payload, parts)if canonical_values:filtered_canonical_values = [value for value in canonical_values if _is_meaningful_requested_value(value)]@@ -374,10 +413,22 @@ def _runtime_path_values(payload: Any, parts: list[str], path: str) -> tuple[liswrapped_parts = ["output", *parts]wrapped_path = ".".join(wrapped_parts)wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]- return wrapped_values, wrapped_path---def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> tuple[list[Any], str]:+ if wrapped_values:+ return wrapped_values, wrapped_path+ for root in sorted(projection_roots):+ projected_parts = [root, *parts]+ projected_path = ".".join(projected_parts)+ projected_values = [+ value for value in _path_values(payload, projected_parts) if _is_meaningful_requested_value(value)+ ]+ if projected_values:+ return projected_values, projected_path+ return [], wrapped_path+++def _runtime_present_path_values(+ payload: Any, parts: list[str], path: str, projection_roots: set[str]+) -> tuple[list[Any], str]:raw_canonical_values = _path_values(payload, parts)if raw_canonical_values:canonical_values = [value for value in raw_canonical_values if _is_meaningful_requested_value(value)]@@ -385,7 +436,17 @@ def _runtime_present_path_values(payload: Any, parts: list[str], path: str) -> twrapped_parts = ["output", *parts]wrapped_path = ".".join(wrapped_parts)wrapped_values = [value for value in _path_values(payload, wrapped_parts) if _is_meaningful_requested_value(value)]- return wrapped_values, wrapped_path+ if wrapped_values:+ return wrapped_values, wrapped_path+ for root in sorted(projection_roots):+ projected_parts = [root, *parts]+ projected_path = ".".join(projected_parts)+ projected_values = [+ value for value in _path_values(payload, projected_parts) if _is_meaningful_requested_value(value)+ ]+ if projected_values:+ return projected_values, projected_path+ return [], wrapped_pathdef _path_values(value: Any, parts: list[str]) -> list[Any]:diff --git a/skyvern/forge/sdk/copilot/completion_verification.py b/skyvern/forge/sdk/copilot/completion_verification.pyindex 672b618d8..f8a90bf92 100644--- a/skyvern/forge/sdk/copilot/completion_verification.py+++ b/skyvern/forge/sdk/copilot/completion_verification.py@@ -155,9 +155,7 @@ class CompletionVerificationResult:unmet = [verdictfor verdict in self.verdicts- if not verdict.satisfied- and not self.is_structural_contingent_abstention(verdict)- and not _is_structural_requested_output_abstention(verdict)+ if not verdict.satisfied and not self.is_structural_contingent_abstention(verdict)]missing_evidence: list[str] = []for verdict in unmet:@@ -209,12 +207,7 @@ class CompletionVerificationResult:evidence_ref = _clean_optional_text(verdict.evidence_ref, max_chars=_EVIDENCE_REF_MAX_CHARS)if evidence_ref:data[f"{prefix}_evidence_ref"] = evidence_ref- detail = (- None- if self.is_structural_contingent_abstention(verdict)- or _is_structural_requested_output_abstention(verdict)- else verdict_missing_evidence(verdict)- )+ detail = None if self.is_structural_contingent_abstention(verdict) else verdict_missing_evidence(verdict)if detail:data[f"{prefix}_missing_evidence"] = detailreturn datadiff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 01956dfc1..840502411 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -207,6 +207,16 @@ _OUTPUT_FIELD_CONNECTOR_RE = re.compile(r"\b(?:with|including|include|includes|containing|contains|fields?|result)\b[:\s]+",re.I,)+_OUTPUT_NEGATED_INTENT_PREFIX_RE = re.compile(+ r"(?:\b(?:do|does|did|should|must|can)\s+not\s+|\b(?:don't|doesn't|didn't|shouldn't|mustn't|can't|cannot|never|without)\s+)$",+ re.I,+)+_OUTPUT_EXPLICIT_FIELD_KEY_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")+_OUTPUT_NAMED_FIELD_RE = re.compile(+ r"\b(?:(?:requested[-\s]+output|output|return(?:ed)?|final|structured)\s+)?fields?\s+"+ r"(?:named|called)\s+([A-Za-z_][A-Za-z0-9_]*)\b",+ re.I,+)_OUTPUT_ACRONYM_RE = re.compile(r"\b[A-Z][A-Z0-9]{1,10}\b")_OUTPUT_FIELD_WORDS = frozenset("address addresses date dates email emails id identifier identifiers license licenses location locations "@@ -777,10 +787,12 @@ def normalized_criterion_outcome_key(outcome: str) -> str:return collapsed.strip().lower().rstrip(".!?;:,").strip()-def _output_intent_spans(user_message: str) -> list[str]:+def _output_intent_spans(user_message: str, *, negated: bool = False) -> list[str]:message = user_message or ""spans: list[str] = []for match in _OUTPUT_INTENT_RE.finditer(message):+ if _output_intent_is_negated(message, match.start()) != negated:+ continuetail = message[match.start() :]span = tail[: _output_span_end_index(tail)]span = _OUTPUT_METHOD_TAIL_RE.sub("", span).strip(" :,-")@@ -789,6 +801,10 @@ def _output_intent_spans(user_message: str) -> list[str]:return spans+def _output_intent_is_negated(message: str, match_start: int) -> bool:+ return bool(_OUTPUT_NEGATED_INTENT_PREFIX_RE.search(message[max(0, match_start - 32) : match_start]))++def _output_span_end_index(text: str) -> int:for index, char in enumerate(text):if _OUTPUT_SPAN_END_RE.fullmatch(char):@@ -857,6 +873,9 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | Noif not candidate:return None+ named_field = _OUTPUT_NAMED_FIELD_RE.search(candidate)+ if named_field is not None:+ return named_field.group(1)connector_matches = list(_OUTPUT_FIELD_CONNECTOR_RE.finditer(candidate))if connector_matches:candidate = candidate[connector_matches[-1].end() :]@@ -864,6 +883,8 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | Nocandidate = " ".join(re.sub(r"[^A-Za-z0-9 _/-]+", " ", candidate).split()).strip(" :-")if not candidate:return None+ if "_" in candidate and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(candidate):+ return candidatenormalized_aliases = _normalize_requested_output_aliases(aliases)if " ".join(_word_tokens(candidate)) in normalized_aliases:return candidate@@ -897,6 +918,8 @@ def _clean_requested_output_candidate(segment: str, aliases: dict[str, str] | Nonormalized_words = normalized_words[1:]if not normalized_words:return None+ if len(words) == 1 and "_" in words[0] and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(words[0]):+ return words[0]if any(word in _OUTPUT_METHOD_WORDS for word in normalized_words):return Noneif all(word in _OUTPUT_GENERIC_WORDS for word in normalized_words):@@ -932,10 +955,17 @@ def _matched_alias_phrase(candidate: str, alias_key: str) -> str | None:return match.group(0).strip() if match is not None else None-def _requested_output_fields(user_message: str, aliases: dict[str, str] | None = None) -> list[str]:+def _requested_output_fields(+ user_message: str, aliases: dict[str, str] | None = None, *, negated: bool = False+) -> list[str]:fields: list[str] = []seen: set[str] = set()- for span in _output_intent_spans(user_message):+ for span in _output_intent_spans(user_message, negated=negated):+ if not negated:+ for match in _OUTPUT_INTENT_RE.finditer(span):+ if _output_intent_is_negated(span, match.start()):+ span = span[: match.start()]+ breakfor segment in _OUTPUT_SPLIT_RE.split(span):field_name = _clean_requested_output_candidate(segment, aliases)if field_name is None:@@ -948,6 +978,18 @@ def _requested_output_fields(user_message: str, aliases: dict[str, str] | None =return fields+def _requested_output_path_for_detected_field(+ field_name: str, schema_aliases: dict[str, str], config_aliases: dict[str, str]+) -> str:+ if "_" in field_name and _OUTPUT_EXPLICIT_FIELD_KEY_RE.fullmatch(field_name.strip()):+ return requested_output_path_for_field(field_name)+ return (+ lookup_requested_output_path_alias(field_name, schema_aliases)+ or lookup_requested_output_path_alias(field_name, config_aliases)+ or requested_output_path_for_field(field_name)+ )++def requested_output_path_for_field(field_name: str, aliases: dict[str, str] | None = None) -> str:alias_path = lookup_requested_output_path_alias(field_name, aliases)if alias_path is not None:@@ -1090,9 +1132,15 @@ def _preserve_after_requested_output_canonicalization(criterion: CompletionCriterion,requested_fields: list[str],requested_output_paths: set[str],+ forbidden_fields: list[str] | None = None,+ forbidden_output_paths: set[str] | None = None,) -> bool:if criterion.level == "definition" or criterion.method_mandated:return True+ if criterion.output_path in (forbidden_output_paths or set()) or _criterion_text_covers_any_requested_output(+ criterion, forbidden_fields or []+ ):+ return Falseif criterion.output_path in requested_output_paths or _criterion_text_covers_any_requested_output(criterion, requested_fields):@@ -1115,24 +1163,26 @@ def _apply_requested_output_completion_criteria(schema_aliases = _normalize_requested_output_aliases(schema_aliases)detection_aliases = {**config_aliases, **schema_aliases}requested_fields = _requested_output_fields(user_message, detection_aliases)- if not requested_fields:+ forbidden_fields = _requested_output_fields(user_message, detection_aliases, negated=True)+ if not requested_fields and not forbidden_fields:returnrequested_specs: list[tuple[str, str, str]] = []requested_output_paths: set[str] = set()for field_name in requested_fields:- output_path = (- lookup_requested_output_path_alias(field_name, schema_aliases)- or lookup_requested_output_path_alias(field_name, config_aliases)- or requested_output_path_for_field(field_name)- )+ output_path = _requested_output_path_for_detected_field(field_name, schema_aliases, config_aliases)field_label = _requested_output_field_label(field_name, output_path)if output_path in requested_output_paths:continuerequested_output_paths.add(output_path)requested_specs.append((field_name, output_path, field_label))- if not requested_output_paths:+ forbidden_output_paths = {+ _requested_output_path_for_detected_field(field_name, schema_aliases, config_aliases)+ for field_name in forbidden_fields+ }++ if not requested_output_paths and not forbidden_output_paths:returnvalue_by_output_path = _requested_output_expected_values_from_criteria(policy.completion_criteria, requested_specs)@@ -1165,7 +1215,13 @@ def _apply_requested_output_completion_criteria(preserved_criteria = [criterionfor criterion in policy.completion_criteria- if _preserve_after_requested_output_canonicalization(criterion, requested_fields, requested_output_paths)+ if _preserve_after_requested_output_canonicalization(+ criterion,+ requested_fields,+ requested_output_paths,+ forbidden_fields,+ forbidden_output_paths,+ )]canonical_requested_criteria = [CompletionCriterion(diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex f63488829..ddcde437c 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -11,7 +11,7 @@ import tokenizefrom collections.abc import Callable, Iterable, Iterator, Mappingfrom dataclasses import dataclassfrom dataclasses import field as dataclass_field-from typing import Annotated, Any, Literal, NamedTuple+from typing import Annotated, Any, Literal, NamedTuple, castfrom urllib.parse import urlsplitimport structlog@@ -1432,98 +1432,89 @@ def _candidate_missing_required_output_roots(code_artifact_metadata: object,*,required_roots: set[str],- runtime_produced_roots: set[str] | None = None,) -> list[str]:if not required_roots:return []- declared_roots = set(_metadata_output_path_roots(code_artifact_metadata))- produced_roots = _workflow_yaml_produced_output_roots(workflow_yaml)- covered_roots = (declared_roots & produced_roots) | (runtime_produced_roots or set())- return sorted(required_roots - covered_roots)---def _recorded_runtime_produced_output_roots(ctx: AgentContext, workflow_yaml: str) -> set[str]:- verified_outputs = getattr(ctx, "verified_block_outputs", None)- if not isinstance(verified_outputs, Mapping):- return set()- code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))- roots: set[str] = set()- for label, output in verified_outputs.items():- if not isinstance(label, str) or label not in code_block_labels:+ declared_roots_by_label = {+ label: set(roots) for label, roots in _metadata_output_path_roots_by_label(code_artifact_metadata).items()+ }+ produced_by_label = _workflow_yaml_produced_output_roots_by_label(workflow_yaml)+ covered_roots: set[str] = set()+ abstained_declared_roots: set[str] = set()+ for label, declared_roots in declared_roots_by_label.items():+ produced = produced_by_label.get(label)+ if produced is None:continue- roots.update(_meaningful_runtime_output_roots(output))- return roots---def _meaningful_runtime_output_roots(value: object, *, prefix: str = "") -> set[str]:- roots: set[str] = set()- if isinstance(value, Mapping):- for raw_key, child in value.items():- if not isinstance(raw_key, str):- continue- key = raw_key.strip()- if not key or key == "evidence_text" or not _is_structural_runtime_output_key(key):- continue- path = f"{prefix}.{key}" if prefix else key- if _runtime_output_value_is_meaningful(child):- roots.add(path.split(".", 1)[0])- roots.update(_meaningful_runtime_output_roots(child, prefix=path))- return roots- if isinstance(value, list):- for item in value:- roots.update(_meaningful_runtime_output_roots(item, prefix=prefix))- return roots+ covered_roots.update(declared_roots & produced.roots)+ if produced.abstained:+ abstained_declared_roots.update(declared_roots)+ missing_roots = required_roots - covered_roots+ if missing_roots:+ missing_roots -= abstained_declared_roots+ return sorted(missing_roots)-def _runtime_output_value_is_meaningful(value: object) -> bool:- if value is None:- return False- if isinstance(value, str):- return bool(value.strip())- if isinstance(value, Mapping):- return any(_runtime_output_value_is_meaningful(item) for item in value.values())- if isinstance(value, list):- return any(_runtime_output_value_is_meaningful(item) for item in value)- return True+class _ProducedOutputRoots(NamedTuple):+ roots: set[str]+ abstained: bool = False-def _workflow_yaml_produced_output_roots(workflow_yaml: str) -> set[str]:- roots: set[str] = set()- for block in _workflow_yaml_code_blocks_by_label(workflow_yaml).values():- roots.update(_code_block_produced_output_roots(str(block.get("code") or "")))- return roots+def _workflow_yaml_produced_output_roots_by_label(workflow_yaml: str) -> dict[str, _ProducedOutputRoots]:+ return {+ label: _code_block_produced_output_roots(str(block.get("code") or ""))+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items()+ }-def _code_block_produced_output_roots(code: str) -> set[str]:+def _code_block_produced_output_roots(code: str) -> _ProducedOutputRoots:try:tree = ast.parse(textwrap.dedent(code).strip() or "pass")except SyntaxError:- return set()+ return _ProducedOutputRoots(set(), True)scope_statements = list(_iter_top_level_scope(tree.body))assigned_roots = _assigned_top_level_names(tree.body)dict_assignments: dict[str, set[str]] = {}+ dynamic_dict_assignment_names: set[str] = set()helper_return_roots = _helper_function_literal_return_roots(tree.body)returned_roots: set[str] = set()+ abstained = False+ saw_return = Falsefor node in scope_statements:if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):- dict_roots = _dict_literal_string_key_roots(node.value)if isinstance(node.value, ast.Dict):- dict_assignments[node.targets[0].id] = dict_roots+ produced = _dict_literal_output_roots(node.value)+ dict_assignments[node.targets[0].id] = produced.roots+ if produced.abstained:+ dynamic_dict_assignment_names.add(node.targets[0].id)elif isinstance(node, ast.Assign):- _apply_literal_dict_key_assignment(dict_assignments, node)+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)elif isinstance(node, ast.AnnAssign):if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):- dict_assignments[node.target.id] = _dict_literal_string_key_roots(node.value)- _apply_literal_dict_key_assignment(dict_assignments, node)+ produced = _dict_literal_output_roots(node.value)+ dict_assignments[node.target.id] = produced.roots+ if produced.abstained:+ dynamic_dict_assignment_names.add(node.target.id)+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)elif isinstance(node, ast.AugAssign):- _apply_literal_dict_key_assignment(dict_assignments, node)- elif isinstance(node, ast.Return) and node.value is not None:- returned_roots.update(_return_output_roots(node.value, dict_assignments, helper_return_roots))- return returned_roots or assigned_roots+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)+ elif isinstance(node, ast.Return):+ saw_return = True+ if node.value is not None:+ returned = _return_output_roots(+ node.value,+ dict_assignments,+ dynamic_dict_assignment_names,+ helper_return_roots,+ )+ returned_roots.update(returned.roots)+ abstained = abstained or returned.abstained+ roots = returned_roots if saw_return else assigned_roots+ return _ProducedOutputRoots(roots, abstained)def _apply_literal_dict_key_assignment(dict_assignments: dict[str, set[str]],+ dynamic_dict_assignment_names: set[str],node: ast.Assign | ast.AnnAssign | ast.AugAssign,) -> None:targets: list[ast.expr] = []@@ -1533,13 +1524,22 @@ def _apply_literal_dict_key_assignment(targets = [node.target]for target in targets:assignment = _literal_dict_key_assignment(target)+ target_name = _dict_subscript_target_name(target)if assignment is None:+ if target_name in dict_assignments:+ dynamic_dict_assignment_names.add(target_name)continuename, key = assignmentif name in dict_assignments:dict_assignments[name].add(key)+def _dict_subscript_target_name(target: ast.expr) -> str | None:+ if isinstance(target, ast.Subscript) and isinstance(target.value, ast.Name):+ return target.value.id+ return None++def _literal_dict_key_assignment(target: ast.expr) -> tuple[str, str] | None:if not isinstance(target, ast.Subscript) or not isinstance(target.value, ast.Name):return None@@ -1552,52 +1552,103 @@ def _literal_dict_key_assignment(target: ast.expr) -> tuple[str, str] | None:def _return_output_roots(node: ast.expr,dict_assignments: Mapping[str, set[str]],- helper_return_roots: Mapping[str, set[str]],-) -> set[str]:+ dynamic_dict_assignment_names: set[str],+ helper_return_roots: Mapping[str, _ProducedOutputRoots],+) -> _ProducedOutputRoots:if isinstance(node, ast.Await):- return _return_output_roots(node.value, dict_assignments, helper_return_roots)+ return _return_output_roots(node.value, dict_assignments, dynamic_dict_assignment_names, helper_return_roots)if isinstance(node, ast.Name):- return set(dict_assignments.get(node.id, set()))+ if node.id in dict_assignments:+ return _ProducedOutputRoots(+ set(dict_assignments.get(node.id, set())), node.id in dynamic_dict_assignment_names+ )+ return _ProducedOutputRoots(set(), True)if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):- return set(helper_return_roots.get(node.func.id, set()))- return _dict_literal_string_key_roots(node)---def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, set[str]]:- helpers: dict[str, set[str]] = {}+ return helper_return_roots.get(node.func.id, _ProducedOutputRoots(set(), True))+ if isinstance(node, ast.Dict):+ return _dict_literal_output_roots(node)+ if isinstance(node, ast.List):+ return _list_literal_output_roots(node)+ if isinstance(node, (ast.Constant, ast.Tuple, ast.Set)):+ return _ProducedOutputRoots(set(), False)+ return _ProducedOutputRoots(set(), True)+++def _list_literal_output_roots(node: ast.List) -> _ProducedOutputRoots:+ if not node.elts:+ return _ProducedOutputRoots(set(), False)+ if all(isinstance(element, ast.Dict) for element in node.elts):+ roots: set[str] = set()+ abstained = False+ for element in node.elts:+ produced = _dict_literal_output_roots(cast(ast.Dict, element))+ roots.update(produced.roots)+ abstained = abstained or produced.abstained+ return _ProducedOutputRoots(roots, abstained)+ if all(isinstance(element, ast.Constant) for element in node.elts):+ return _ProducedOutputRoots(set(), False)+ return _ProducedOutputRoots(set(), True)+++def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, _ProducedOutputRoots]:+ helpers: dict[str, _ProducedOutputRoots] = {}for statement in statements:if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):continuedict_assignments: dict[str, set[str]] = {}+ dynamic_dict_assignment_names: set[str] = set()roots: set[str] = set()+ abstained = False+ saw_return = Falsefor helper_statement in _iter_top_level_scope(statement.body):if (isinstance(helper_statement, ast.Assign)and len(helper_statement.targets) == 1and isinstance(helper_statement.targets[0], ast.Name)):- dict_roots = _dict_literal_string_key_roots(helper_statement.value)if isinstance(helper_statement.value, ast.Dict):- dict_assignments[helper_statement.targets[0].id] = dict_roots+ produced = _dict_literal_output_roots(helper_statement.value)+ dict_assignments[helper_statement.targets[0].id] = produced.roots+ if produced.abstained:+ dynamic_dict_assignment_names.add(helper_statement.targets[0].id)elif isinstance(helper_statement, ast.Assign):- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)elif isinstance(helper_statement, ast.AnnAssign):if isinstance(helper_statement.target, ast.Name) and isinstance(helper_statement.value, ast.Dict):- dict_assignments[helper_statement.target.id] = _dict_literal_string_key_roots(- helper_statement.value- )- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)+ produced = _dict_literal_output_roots(helper_statement.value)+ dict_assignments[helper_statement.target.id] = produced.roots+ if produced.abstained:+ dynamic_dict_assignment_names.add(helper_statement.target.id)+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)elif isinstance(helper_statement, ast.AugAssign):- _apply_literal_dict_key_assignment(dict_assignments, helper_statement)- elif isinstance(helper_statement, ast.Return) and helper_statement.value is not None:- roots.update(_dict_literal_string_key_roots(helper_statement.value))- if isinstance(helper_statement.value, ast.Name):- roots.update(dict_assignments.get(helper_statement.value.id, set()))- if roots:- helpers[statement.name] = roots+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, helper_statement)+ elif isinstance(helper_statement, ast.Return):+ saw_return = True+ if helper_statement.value is not None:+ returned = _return_output_roots(+ helper_statement.value,+ dict_assignments,+ dynamic_dict_assignment_names,+ {},+ )+ roots.update(returned.roots)+ abstained = abstained or returned.abstained+ if roots or abstained or saw_return:+ helpers[statement.name] = _ProducedOutputRoots(roots, abstained)return helpers+def _dict_literal_output_roots(node: ast.Dict) -> _ProducedOutputRoots:+ roots: set[str] = set()+ abstained = False+ for key in node.keys:+ if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.strip():+ roots.add(key.value.strip())+ else:+ abstained = True+ return _ProducedOutputRoots(roots, abstained)++def _dict_literal_string_key_roots(node: ast.expr) -> set[str]:if not isinstance(node, ast.Dict):return set()@@ -1629,13 +1680,14 @@ def _code_block_has_meaningful_output(code: str, artifact: object) -> bool:tree = ast.parse(code)except SyntaxError:return False+ helper_return_roots = _helper_function_literal_return_roots(tree.body)class ReturnVisitor(ast.NodeVisitor):def __init__(self) -> None:self.has_meaningful_return = Falsedef visit_Return(self, node: ast.Return) -> None:- if _return_value_is_meaningful(node.value):+ if _return_value_is_meaningful(node.value, helper_return_roots):self.has_meaningful_return = Truedef visit_FunctionDef(self, node: ast.FunctionDef) -> None:@@ -1838,11 +1890,21 @@ def _code_block_has_top_level_return(tree: ast.AST) -> bool:return any(isinstance(statement, ast.Return) for statement in _iter_top_level_scope(body))-def _return_value_is_meaningful(node: ast.expr | None) -> bool:+def _return_value_is_meaningful(+ node: ast.expr | None,+ helper_return_roots: Mapping[str, _ProducedOutputRoots],+) -> bool:if node is None:return False+ if isinstance(node, ast.Await):+ return _return_value_is_meaningful(node.value, helper_return_roots)if isinstance(node, ast.Constant) and node.value is None:return False+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):+ produced = helper_return_roots.get(node.func.id)+ if produced is not None:+ return bool(produced.roots) or produced.abstained+ return Trueif (isinstance(node, (ast.Dict, ast.List, ast.Tuple, ast.Set))and not getattr(node, "elts", None)@@ -5089,16 +5151,25 @@ async def _update_workflow(tool_arguments=params,)if not output_policy_verdict.allowed:+ output_policy_trace_data = output_policy_verdict_to_trace_data(+ output_policy_verdict,+ surface="tool_body",+ tool_name="update_workflow",+ )+ output_policy_error = format_output_policy_tool_error(output_policy_verdict)_record_code_only_raw_secret_reject_span(ctx, output_policy_verdict)LOG.info("copilot output policy tool body verdict",- **output_policy_verdict_to_trace_data(- output_policy_verdict,- surface="tool_body",- tool_name="update_workflow",- ),+ **output_policy_trace_data,)- return reject(error=format_output_policy_tool_error(output_policy_verdict))+ _record_author_time_reject_outcome(+ ctx,+ reason_code="output_policy_reject",+ summary=output_policy_error,+ structural_payload=output_policy_trace_data,+ )+ _record_code_authoring_guardrail_reject(ctx)+ return reject(error=output_policy_error)# Prefer the most-recent in-turn emission so cross-path flows (inline# REPLACE_WORKFLOW followed by update_workflow) compare against what the@@ -5183,10 +5254,7 @@ async def _update_workflow(recorded_missing_output_roots = (_recorded_outcome_missing_output_roots(ctx) if recorded_output_roots_required else set())- runtime_produced_output_roots = (- _recorded_runtime_produced_output_roots(ctx, workflow_yaml) if recorded_output_roots_required else set()- )- unresolved_recorded_output_roots = recorded_missing_output_roots - runtime_produced_output_roots+ unresolved_recorded_output_roots = recorded_missing_output_rootsoutput_empty_labels = (_output_empty_code_block_labels(workflow_yaml, getattr(ctx, "code_artifact_metadata", None))@@ -5226,7 +5294,6 @@ async def _update_workflow(workflow_yaml,getattr(ctx, "code_artifact_metadata", None),required_roots=unresolved_recorded_output_roots,- runtime_produced_roots=runtime_produced_output_roots,)if recorded_output_roots_requiredelse []diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.pyindex c53c9930e..b95178b9e 100644--- a/tests/unit/test_copilot_build_test_outcome.py+++ b/tests/unit/test_copilot_build_test_outcome.py@@ -369,6 +369,105 @@ def test_not_evaluated_recorded_outcome_is_not_authoritative_repair_failure() ->assert outcome.is_authoritative is False+def test_outcome_not_demonstrated_does_not_mark_presence_only_abstention_as_missing_output() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_top_post",+ "overall_status": "completed",+ "blocks": [+ {+ "label": "extract_top_hn_post",+ "status": "completed",+ "extracted_data": {"output": {"top_post": "Claude Sonnet 5"}},+ }+ ],+ },+ }+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["top_post"],+ verdicts=[+ CriterionVerdict(+ criterion_id="top_post",+ state="unsatisfied",+ reason_code="structurally_abstained",+ output_path="output.top_post",+ grounding_mode="missing",+ evidence_ref="block_outputs:extract_top_hn_post.output.top_post",+ )+ ],+ )++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ workflow_run_id="wr_top_post",+ ),+ completion_verification=verification,+ )++ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert outcome.is_authoritative is True+ assert outcome.missing_requested_output_facts == []+++def test_outcome_not_demonstrated_keeps_missing_fact_for_absent_requested_output() -> None:+ result = {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_no_top_post",+ "overall_status": "completed",+ "blocks": [+ {+ "label": "extract_top_hn_post",+ "status": "completed",+ "extracted_data": {"output": {}},+ }+ ],+ },+ }+ verification = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["top_post"],+ verdicts=[+ CriterionVerdict(+ criterion_id="top_post",+ state="unsatisfied",+ reason_code="no_evidence",+ output_path="output.top_post",+ grounding_mode="missing",+ )+ ],+ )++ outcome = recorded_outcome_from_run_blocks_result(+ result,+ recorded_run_outcome=RecordedRunOutcome(+ verdict="not_demonstrated",+ reason_code="outcome_not_demonstrated",+ workflow_run_id="wr_no_top_post",+ ),+ completion_verification=verification,+ )++ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert outcome.missing_requested_output_facts == [+ {+ "criterion_id": "top_post",+ "output_path": "output.top_post",+ "output_root": "output",+ "reason_code": "no_evidence",+ "value_status": "no_typed_value",+ "grounding_mode": "missing",+ }+ ]++def test_authoring_repair_context_produces_structural_recorded_outcome() -> None:context = CodeAuthoringRepairContext(block_label="search_records",@@ -556,6 +655,40 @@ def test_metadata_reject_key_uses_typed_fields_not_wording() -> None:assert changed_missing_field.structural_key != first.structural_key+def test_output_policy_reject_key_uses_stable_trace_payload() -> None:+ payload = {+ "surface": "tool_body",+ "tool_name": "update_workflow",+ "allowed": False,+ "output_kind": "workflow_update_proposal",+ "reason_codes": ["raw_secret_leak"],+ }+ first = recorded_outcome_from_author_time_reject(+ reason_code="output_policy_reject",+ structural_payload=payload,+ observed_evidence_summary="Output policy blocked this Copilot output before persistence.",+ )+ same_structure = recorded_outcome_from_author_time_reject(+ reason_code="output_policy_reject",+ structural_payload=dict(payload),+ observed_evidence_summary="Different wording for the same output-policy reject.",+ )+ changed_reason = recorded_outcome_from_author_time_reject(+ reason_code="output_policy_reject",+ structural_payload={**payload, "reason_codes": ["unapproved_credential_reference"]},+ )+ changed_surface = recorded_outcome_from_author_time_reject(+ reason_code="output_policy_reject",+ structural_payload={**payload, "surface": "final_response"},+ )++ assert first.reason_code == "output_policy_reject"+ assert first.is_authoritative is True+ assert first.structural_key == same_structure.structural_key+ assert changed_reason.structural_key != first.structural_key+ assert changed_surface.structural_key != first.structural_key++def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:result = {"ok": False,diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 02b2eaa60..95f98d213 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -128,6 +128,17 @@ _SAFE_EXTRACTION_CODE_YAML = _yaml(""")+_RAW_SECRET_OUTPUT_POLICY_YAML = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: navigation+ label: login+ navigation_goal: Type password: hunter2 into the password field.+ """+)+def _code_yaml(code: str,@@ -1161,61 +1172,972 @@ class TestCodeRepairProgressClassification:authored_structure_signature=signature,)- first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)- first_key = ctx.latest_recorded_build_test_outcome.structural_key- first_count = ctx.code_authoring_guardrail_reject_count- second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)+ first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)+ first_key = ctx.latest_recorded_build_test_outcome.structural_key+ first_count = ctx.code_authoring_guardrail_reject_count+ second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)++ assert first["ok"] is False+ assert second["ok"] is False+ assert first_key is not None+ assert ctx.latest_recorded_build_test_outcome.structural_key == first_key+ assert first_count == 1+ assert ctx.code_authoring_guardrail_reject_count == 2++ @pytest.mark.asyncio+ async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)+ changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:typed-outcome",+ authored_structure_signature=signature,+ )++ result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == changed_yaml+ assert ctx.has_staged_proposal is True++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["npi", "locations"],+ verdicts=[+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),+ ],+ )+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ )+ partial_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ parameters:+ - parameter_type: workflow+ workflow_parameter_type: string+ key: address_city_county_or_zip_code+ blocks:+ - block_type: code+ label: {label}+ parameter_keys:+ - address_city_county_or_zip_code+ code: |+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)+ await page.locator("#locInput").fill(str(address_city_county_or_zip_code))+ return {{}}+ """+ )++ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not return any keyed output" in result["error"]+ assert ctx.has_staged_proposal is False+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_explicit_empty_return_despite_assigned_goal_root(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["npi"],+ verdicts=[+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),+ ],+ )+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ npi = await page.locator("#npi").inner_text(timeout=5000)+ return {{}}+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not return any keyed output" in result["error"]+ assert ctx.has_staged_proposal is False+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.is_authoritative is True+ assert outcome.block_labels == [label]+ assert outcome.structural_failure_identity.startswith("author_time:")++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_known_empty_helper_without_missing_facts(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["npi"],+ verdicts=[+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),+ ],+ )+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ def build():+ return {{}}++ return build()+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not return any keyed output" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.block_labels == [label]+ assert outcome.missing_requested_output_facts == []+ assert outcome.structural_failure_identity.startswith("author_time:")++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.completion_verification_result = CompletionVerificationResult(+ status="evaluated",+ criterion_ids=["npi", "locations"],+ verdicts=[+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),+ ],+ )+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)+ npi = "1234567890"+ locations = [{{"address": "123 Main St"}}]+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["address"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},+ {+ "output_path": "credentialing_status",+ "output_root": "credentialing_status",+ "value_status": "no_typed_value",+ },+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},+ ],+ )+ partial_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)+ address = "North Carolina, USA"+ """+ )++ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ assert ctx.has_staged_proposal is False+ assert ctx.latest_recorded_build_test_outcome is not None+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True++ def test_required_output_roots_ignore_nested_debug_output_root_without_declared_root(self) -> None:+ label = "extract_provider_profile"+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["debug_output"]}]}}+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"debug_output": {{"npi": "1234567890"}}}}+ """+ )++ assert workflow_update_module._candidate_missing_required_output_roots(+ candidate_yaml,+ metadata,+ required_roots={"npi"},+ ) == ["npi"]++ def test_required_output_roots_ignore_same_label_undeclared_static_root(self) -> None:+ label = "extract_provider_profile"+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["address"]}]}}+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"npi": "1234567890"}}+ """+ )++ assert workflow_update_module._candidate_missing_required_output_roots(+ candidate_yaml,+ metadata,+ required_roots={"npi"},+ ) == ["npi"]++ def test_required_output_roots_ignore_static_root_without_label_metadata(self) -> None:+ label = "extract_provider_profile"+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"npi": "1234567890"}}+ """+ )++ assert workflow_update_module._candidate_missing_required_output_roots(+ candidate_yaml,+ {},+ required_roots={"npi"},+ ) == ["npi"]++ def test_required_output_roots_accept_same_label_declared_static_root(self) -> None:+ label = "extract_provider_profile"+ metadata = {label: {"block_label": label, "claimed_outcomes": [{"goal_value_paths": ["npi"]}]}}+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"npi": "1234567890"}}+ """+ )++ assert (+ workflow_update_module._candidate_missing_required_output_roots(+ candidate_yaml,+ metadata,+ required_roots={"npi"},+ )+ == []+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_unprovable_dynamic_output_shape(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ result = {{}}+ key = "npi"+ value = await page.locator("#npi").inner_text(timeout=5000)+ result[key] = value+ return result+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_literal_list_of_output_dicts(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return [{{"npi": "123"}}]+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_list_output_to_abstain(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ record = {{"npi": "123"}}+ return [record]+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_list_dict_to_abstain(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ key = "npi"+ return [{{key: "123"}}]+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_dynamic_key_local_dict_to_abstain(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ key = "npi"+ result = {{key: "123"}}+ return result+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ def test_local_dict_dynamic_key_preserves_literal_roots_while_abstaining(self) -> None:+ produced = workflow_update_module._code_block_produced_output_roots(+ """+ key = "npi"+ result = {"address": "North Carolina", key: "123"}+ return result+ """+ )++ assert produced.roots == {"address"}+ assert produced.abstained is True++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_helper_dynamic_key_local_dict_to_abstain(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ def build():+ key = "npi"+ result = {{key: "123"}}+ return result++ return build()+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is True+ assert ctx.workflow_yaml == candidate_yaml+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "outcome_not_demonstrated"+ assert all(+ fact.get("reason_code") != "recorded_outcome_missing_output_coverage"+ for fact in outcome.missing_requested_output_facts+ )++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_unrelated_dynamic_output_shape(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["address"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ result = {{}}+ key = "address"+ result[key] = await page.locator("#address").inner_text(timeout=5000)+ return result+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "npi",+ "output_root": "npi",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_rejects_when_unrelated_block_abstains(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ declared_label = "extract_provider_profile"+ unrelated_label = "extract_unrelated_status"+ ctx.code_artifact_metadata = {+ declared_label: {+ "block_label": declared_label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {declared_label}+ code: |+ return {{"address": "North Carolina"}}+ - block_type: code+ label: {unrelated_label}+ code: |+ result = {{}}+ key = "status"+ result[key] = await page.locator("#status").inner_text(timeout=5000)+ return result+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "npi",+ "output_root": "npi",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_keeps_undeclared_missing_roots_with_dynamic_shape(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ {"output_path": "status", "output_root": "status", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ result = {{}}+ key = "npi"+ result[key] = await page.locator("#npi").inner_text(timeout=5000)+ return result+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)- assert first["ok"] is False- assert second["ok"] is False- assert first_key is not None- assert ctx.latest_recorded_build_test_outcome.structural_key == first_key- assert first_count == 1- assert ctx.code_authoring_guardrail_reject_count == 2+ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "status",+ "output_root": "status",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]@pytest.mark.asyncio- async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(+ async def test_authoritative_outcome_not_demonstrated_rejects_literal_output_missing_required_root(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)ctx = _code_only_ctx()- signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)- changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')+ label = "lookup_provider_and_extract_credentials"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(phase="persisted_block_run",attempted_tool="update_and_run_blocks",verdict="repairable_failure",reason_code="outcome_not_demonstrated",- structural_failure_identity="completion:typed-outcome",- authored_structure_signature=signature,+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Provider lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"address": "North Carolina"}}+ """)- result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)- assert result["ok"] is True- assert ctx.workflow_yaml == changed_yaml- assert ctx.has_staged_proposal is True+ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.block_labels == [label]+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "npi",+ "output_root": "npi",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]@pytest.mark.asyncio- async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(+ async def test_authoritative_outcome_not_demonstrated_rejects_empty_return_despite_assigned_root(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)ctx = _code_only_ctx()label = "lookup_provider_and_extract_credentials"- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["npi", "locations"],- verdicts=[- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),- ],- )ctx.code_artifact_metadata = {label: {"block_label": label,- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],}}ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)@@ -1226,56 +2148,47 @@ class TestCodeRepairProgressClassification:reason_code="outcome_not_demonstrated",structural_failure_identity="completion:unsatisfied-output",authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],)- partial_yaml = _yaml(+ candidate_yaml = _yaml(f"""title: Provider lookupworkflow_definition:- parameters:- - parameter_type: workflow- workflow_parameter_type: string- key: address_city_county_or_zip_codeblocks:- block_type: codelabel: {label}- parameter_keys:- - address_city_county_or_zip_codecode: |- await page.locator("#locInput").wait_for(state="visible", timeout=15000)- await page.locator("#locInput").fill(str(address_city_county_or_zip_code))+ npi = await page.locator("#npi").inner_text(timeout=5000)return {{}}""")- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)assert result["ok"] is Falseassert "does not return any keyed output" in result["error"]- assert ctx.has_staged_proposal is False- assert ctx.latest_recorded_build_test_outcome is not None- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.block_labels == [label]+ assert outcome.missing_requested_output_facts == []+ @pytest.mark.parametrize("return_expression", ["None", "[]", '["not keyed"]'])@pytest.mark.asyncio- async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(- self, monkeypatch: pytest.MonkeyPatch+ async def test_authoritative_outcome_not_demonstrated_rejects_static_non_keyed_return_with_unrelated_output(+ self, monkeypatch: pytest.MonkeyPatch, return_expression: str) -> None:_stub_successful_update(monkeypatch)ctx = _code_only_ctx()- label = "lookup_provider_and_extract_credentials"- ctx.completion_verification_result = CompletionVerificationResult(- status="evaluated",- criterion_ids=["npi", "locations"],- verdicts=[- CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),- CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),- ],- )+ declared_label = "extract_provider_profile"+ unrelated_label = "extract_unrelated_status"ctx.code_artifact_metadata = {- label: {- "block_label": label,- "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],- "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],+ declared_label: {+ "block_label": declared_label,+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],}}ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)@@ -1286,6 +2199,9 @@ class TestCodeRepairProgressClassification:reason_code="outcome_not_demonstrated",structural_failure_identity="completion:unsatisfied-output",authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},+ ],)candidate_yaml = _yaml(f"""@@ -1293,21 +2209,34 @@ class TestCodeRepairProgressClassification:workflow_definition:blocks:- block_type: code- label: {label}+ label: {declared_label}code: |- await page.locator("#locInput").wait_for(state="visible", timeout=15000)- npi = "1234567890"- locations = [{{"address": "123 Main St"}}]+ return {return_expression}+ - block_type: code+ label: {unrelated_label}+ code: |+ return {{"status": "active"}}""")result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)- assert result["ok"] is True- assert ctx.workflow_yaml == candidate_yaml+ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "npi",+ "output_root": "npi",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]@pytest.mark.asyncio- async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(+ async def test_authoritative_outcome_not_demonstrated_rejects_helper_known_empty_output(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)@@ -1316,8 +2245,8 @@ class TestCodeRepairProgressClassification:ctx.code_artifact_metadata = {label: {"block_label": label,- "claimed_outcomes": [{"goal_value_paths": ["address"]}],- "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],+ "claimed_outcomes": [{"goal_value_paths": ["npi"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi"]}],}}ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)@@ -1329,17 +2258,10 @@ class TestCodeRepairProgressClassification:structural_failure_identity="completion:unsatisfied-output",authored_structure_signature="authored:previous-failed-candidate",missing_requested_output_facts=[- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},- {- "output_path": "credentialing_status",- "output_root": "credentialing_status",- "value_status": "no_typed_value",- },- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},+ {"output_path": "npi", "output_root": "npi", "value_status": "no_typed_value"},],)- partial_yaml = _yaml(+ candidate_yaml = _yaml(f"""title: Provider lookupworkflow_definition:@@ -1347,19 +2269,21 @@ class TestCodeRepairProgressClassification:- block_type: codelabel: {label}code: |- await page.locator("#locInput").wait_for(state="visible", timeout=15000)- address = "North Carolina, USA"+ def build():+ return {{}}++ return build()""")- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)assert result["ok"] is False- assert "does not cover the missing requested output roots" in result["error"]- assert ctx.has_staged_proposal is False- assert ctx.latest_recorded_build_test_outcome is not None- assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"- assert ctx.latest_recorded_build_test_outcome.is_authoritative is True+ assert "does not return any keyed output" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == []@pytest.mark.asyncioasync def test_metadata_reject_persists_missing_roots_for_next_author_context(@@ -1424,17 +2348,16 @@ class TestCodeRepairProgressClassification:assert "value_status=no_typed_value" in rendered@pytest.mark.asyncio- async def test_authoritative_outcome_not_demonstrated_allows_when_last_run_produced_required_roots(+ async def test_authoritative_outcome_not_demonstrated_rejects_when_only_last_run_produced_required_roots(self, monkeypatch: pytest.MonkeyPatch) -> None:_stub_successful_update(monkeypatch)ctx = _code_only_ctx()- label = "lookup_provider_and_extract_credentials"+ label = "extract_top_hn_post"ctx.verified_block_outputs[label] = {- "address": "North Carolina, USA",- "credentialing_status": "Active",- "locations": [{"address": "North Carolina, USA"}],- "statuses": ["Active"],+ "top_post": "Claude Sonnet 5",+ "rank": 1,+ "url": "https://news.ycombinator.com/item?id=1",}ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(phase="persisted_block_run",@@ -1444,26 +2367,72 @@ class TestCodeRepairProgressClassification:structural_failure_identity="completion:unsatisfied-output",authored_structure_signature="authored:previous-failed-candidate",missing_requested_output_facts=[- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},- {- "output_path": "credentialing_status",- "output_root": "credentialing_status",- "value_status": "no_typed_value",- },- {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},- {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},+ {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},],)candidate_yaml = _yaml(f"""- title: Provider lookup+ title: Hacker News lookupworkflow_definition:blocks:- block_type: codelabel: {label}code: |- await page.locator("#locInput").wait_for(state="visible", timeout=15000)- address = "North Carolina, USA"+ return {{"rank": 1, "url": "https://news.ycombinator.com/item?id=1"}}+ """+ )++ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)++ assert result["ok"] is False+ assert "does not cover the missing requested output roots" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.reason_code == "metadata_reject"+ assert outcome.missing_requested_output_facts == [+ {+ "output_path": "top_post",+ "output_root": "top_post",+ "reason_code": "recorded_outcome_missing_output_coverage",+ "value_status": "no_typed_value",+ }+ ]++ @pytest.mark.asyncio+ async def test_authoritative_outcome_not_demonstrated_allows_static_return_for_required_root(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ label = "extract_top_hn_post"+ ctx.code_artifact_metadata = {+ label: {+ "block_label": label,+ "claimed_outcomes": [{"goal_value_paths": ["top_post"]}],+ "terminal_verifier_expectations": [{"goal_value_paths": ["top_post"]}],+ }+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(+ phase="persisted_block_run",+ attempted_tool="update_and_run_blocks",+ verdict="repairable_failure",+ reason_code="outcome_not_demonstrated",+ structural_failure_identity="completion:unsatisfied-output",+ authored_structure_signature="authored:previous-failed-candidate",+ missing_requested_output_facts=[+ {"output_path": "top_post", "output_root": "top_post", "value_status": "presence_only_evidence"},+ ],+ )+ candidate_yaml = _yaml(+ f"""+ title: Hacker News lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ return {{"top_post": "Claude Sonnet 5", "rank": 1}}""")@@ -4631,6 +5600,47 @@ class TestCodeAuthoringGuardrailChurnBackstop:assert result["ok"] is Trueassert ctx.code_authoring_guardrail_reject_count == 0+ @pytest.mark.asyncio+ async def test_output_policy_reject_records_author_time_reject_and_guardrail_count(self) -> None:+ ctx = _code_only_ctx()++ result = await _update_workflow(+ {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},+ ctx,+ allow_missing_credentials=True,+ )++ assert result["ok"] is False+ assert "raw_secret_leak" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert outcome is not None+ assert outcome.phase == "author_time_reject"+ assert outcome.reason_code == "output_policy_reject"+ assert outcome.is_authoritative is True+ assert outcome.structural_key is not None+ assert "Output policy blocked" in outcome.observed_evidence_summary+ assert ctx.code_authoring_guardrail_reject_count == 1+ assert ctx.blocker_signal is None++ @pytest.mark.asyncio+ async def test_repeated_output_policy_reject_halts_at_guardrail_backstop(self) -> None:+ ctx = _code_only_ctx()++ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):+ result = await _update_workflow(+ {"workflow_yaml": _RAW_SECRET_OUTPUT_POLICY_YAML},+ ctx,+ allow_missing_credentials=True,+ )+ assert result["ok"] is False++ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS+ churn = ctx.blocker_signal+ assert isinstance(churn, CopilotToolBlockerSignal)+ assert churn.internal_reason_code == "code_authoring_guardrail_churn"+ assert ctx.latest_tool_blocker_signal is churn+ assert _kind_for_blocker_signal(churn) is TurnHaltKind.LOOP_DETECTED+@pytest.mark.asyncioasync def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:ctx = _code_only_ctx()diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.pyindex ebcc32dcf..63d791acf 100644--- a/tests/unit/test_copilot_completion_verification.py+++ b/tests/unit/test_copilot_completion_verification.py@@ -2647,10 +2647,15 @@ def _requested_output_result(output: Any) -> dict:def _metadata_for_requested_paths(*paths: str) -> dict[str, Any]:return {- "extract_profile": {+ label: {"claimed_outcomes": [{"goal_value_paths": list(paths)}],"terminal_verifier_expectations": [{"goal_value_paths": list(paths)}],}+ for label in (+ "extract_profile",+ "utility_citrus_turn_on",+ "utility_peach_gas_quickconnect",+ )}@@ -4603,7 +4608,335 @@ async def test_requested_output_path_exact_runtime_field_with_expected_value_satassert "1234567890" not in repr(trace)-def test_requested_output_path_without_expected_value_structurally_abstains_for_present_values() -> None:+async def test_requested_output_typed_block_fields_do_not_admit_undeclared_returned_field(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("typed requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "extract_profile")+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address", "credentialing_status", "locations")+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ ),+ CompletionCriterion(+ id="c_credentialing_status",+ outcome="The returned profile includes credentialing status.",+ output_path="output.credentialing_status",+ expected_output_value="Active",+ ),+ CompletionCriterion(+ id="c_locations",+ outcome="The returned profile includes location address.",+ output_path="output.locations.address",+ expected_output_value="101 Example Ave",+ ),+ CompletionCriterion(+ id="c_statuses",+ outcome="The returned profile includes statuses.",+ output_path="output.statuses",+ expected_output_value="Active",+ ),+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ _requested_output_result(+ {+ "address": "101 Example Ave",+ "credentialing_status": "Active",+ "locations": [{"address": "101 Example Ave"}],+ "statuses": ["Active"],+ }+ ),+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is False+ assert {verdict.criterion_id for verdict in verification.verdicts if verdict.satisfied} == {+ "c_address",+ "c_credentialing_status",+ "c_locations",+ }+ statuses = next(verdict for verdict in verification.verdicts if verdict.criterion_id == "c_statuses")+ assert statuses.state == "unsatisfied"+ assert statuses.reason_code == "unproducible"+ assert statuses.evidence_ref is None+ assert {+ verdict.evidence_ref for verdict in verification.verdicts if verdict.criterion_id in {"c_address", "c_statuses"}+ } == {"block_outputs:extract_profile.address", None}+++def test_requested_output_nested_declared_output_root_maps_to_requested_output_path() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("profile.address")+ criteria = [+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"profile": {"address": "101 Example Ave"}}}),+ )++ assert verdicts[0].state == "satisfied"+ assert verdicts[0].reason_code == "evidence_confirms"+ assert verdicts[0].evidence_ref == "block_outputs:extract_profile.profile.address"+++ ("accepted_payload", "expected_reason", "expected_ref"),+ [+ ({"address": "Wrong Example Ave"}, "evidence_contradicts", "block_outputs:extract_profile.address"),+ ({"status": "active"}, "missing_exact_field", None),+ ],+)+def test_requested_output_ignores_unrelated_block_outputs_when_matching_values(+ accepted_payload: dict[str, str],+ expected_reason: str,+ expected_ref: str | None,+) -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")+ criteria = [+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ block_outputs={+ "extract_profile": accepted_payload,+ "unrelated_lookup": {"address": "101 Example Ave"},+ }+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == expected_reason+ assert verdicts[0].evidence_ref == expected_ref+++def test_requested_output_evidence_text_with_expected_value_does_not_satisfy() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")+ criteria = [+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"evidence_text": "Address: 101 Example Ave"}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "missing_exact_field"+++def test_requested_output_roots_do_not_satisfy_from_executed_block_output_without_static_metadata_roots() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ criteria = [+ CompletionCriterion(+ id="c_npi",+ outcome="The returned profile includes NPI.",+ output_path="output.npi",+ expected_output_value="1234567890",+ ),+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ ),+ CompletionCriterion(+ id="c_credentialing_status",+ outcome="The returned profile includes credentialing status.",+ output_path="output.credentialing_status",+ expected_output_value="Active",+ ),+ CompletionCriterion(+ id="c_locations",+ outcome="The returned profile includes locations.",+ output_path="output.locations.address",+ expected_output_value="101 Example Ave",+ ),+ CompletionCriterion(+ id="c_statuses",+ outcome="The returned profile includes statuses.",+ output_path="output.statuses",+ expected_output_value="Active",+ ),+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ executed_block_labels=["extract_profile"],+ block_outputs={+ "extract_profile": {+ "npi": "1234567890",+ "address": "101 Example Ave",+ "credentialing_status": "Active",+ "locations": [{"address": "101 Example Ave"}],+ "statuses": ["Active"],+ }+ },+ ),+ )++ assert {verdict.criterion_id for verdict in verdicts} == {+ "c_npi",+ "c_address",+ "c_credentialing_status",+ "c_locations",+ "c_statuses",+ }+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"unproducible"}+ assert all(verdict.evidence_ref is None for verdict in verdicts)+++def test_requested_output_roots_without_expected_values_abstain_from_runtime_presence() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address", "statuses")+ criteria = [+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ ),+ CompletionCriterion(+ id="c_statuses",+ outcome="The returned profile includes statuses.",+ output_path="output.statuses",+ ),+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ executed_block_labels=["extract_profile"],+ block_outputs={"extract_profile": {"address": "101 Example Ave", "statuses": ["Active"]}},+ ),+ )++ assert {verdict.criterion_id for verdict in verdicts} == {"c_address", "c_statuses"}+ assert {verdict.state for verdict in verdicts} == {"unsatisfied"}+ assert {verdict.reason_code for verdict in verdicts} == {"structurally_abstained"}+ assert {verdict.evidence_ref for verdict in verdicts} == {+ "block_outputs:extract_profile.address",+ "block_outputs:extract_profile.statuses",+ }+++def test_requested_output_executed_root_admission_ignores_evidence_text_only_values() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ criteria = [+ CompletionCriterion(+ id="c_npi",+ outcome="The returned profile includes NPI.",+ output_path="output.npi",+ expected_output_value="1234567890",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ executed_block_labels=["extract_profile"],+ block_outputs={"extract_profile": {"evidence_text": "The provider NPI is 1234567890."}},+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "unproducible"+++def test_requested_output_executed_root_admission_ignores_unexecuted_block_outputs() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ criteria = [+ CompletionCriterion(+ id="c_address",+ outcome="The returned profile includes address.",+ output_path="output.address",+ expected_output_value="101 Example Ave",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(+ executed_block_labels=["extract_profile"],+ block_outputs={+ "extract_profile": {"status": "Active"},+ "unrelated_lookup": {"address": "101 Example Ave"},+ },+ ),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "unproducible"+++def test_requested_output_missing_metadata_and_runtime_field_fails_closed() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = _metadata_for_requested_paths("address")+ criteria = [+ CompletionCriterion(+ id="c_statuses",+ outcome="The returned profile includes statuses.",+ output_path="output.statuses",+ expected_output_value="Active",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"address": "101 Example Ave"}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "unproducible"+ assert verdicts[0].missing_evidence == "accepted code artifact metadata does not declare output.statuses"+++def test_requested_output_path_without_expected_value_abstains_for_present_typed_values() -> None:ctx = _run_ctx()ctx.code_artifact_metadata = _metadata_for_requested_paths("request_id",@@ -4724,7 +5057,11 @@ def test_requested_output_shape_only_generated_fields_structurally_abstain_withoassert trace["verdict_0_has_exact_value"] is Falseassert "WTR-1842-DEMO" not in repr(trace)assert trace["fully_satisfied"] is False- assert trace["unmet_criterion_ids"] == []+ assert trace["unmet_criterion_ids"] == [+ "c_confirmation_number",+ "c_account_number",+ "c_selected_start_date",+ ]def test_requested_output_shape_abstains_on_p8_scrambled_values_and_ignores_evidence_text_status() -> None:@@ -5750,6 +6087,165 @@ async def test_requested_output_path_can_use_static_return_key_contract(assert verification.is_fully_satisfied() is True+async def test_requested_output_path_can_use_static_return_contract_without_matching_metadata(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def fail_handler(**_: object) -> object:+ raise AssertionError("static requested-output evidence must bypass the judge")++ _patch_completion_handler(monkeypatch, fail_handler)+ ctx = _run_ctx()+ _set_workflow_labels(ctx, "story")+ ctx.code_artifact_metadata = {+ "stale_extract": _metadata_for_requested_paths("top_post_identified")["extract_profile"]+ }+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: story+ code: |+ return {"top_post_identified": True}+ """+ )+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ CompletionCriterion(+ id="c_top_post",+ outcome="The top Hacker News post is identified.",+ output_path="output.top_post_identified",+ expected_output_value="true",+ )+ ]+ )++ verification = await _maybe_run_completion_verification(+ ctx,+ {+ "ok": True,+ "data": {+ "workflow_run_id": "wr_hn",+ "overall_status": "completed",+ "executed_block_labels": ["story"],+ "blocks": [+ {+ "label": "story",+ "block_type": "CODE",+ "status": "completed",+ "extracted_data": {"top_post_identified": True},+ }+ ],+ },+ },+ time.monotonic(),+ )++ assert verification is not None+ assert verification.is_fully_satisfied() is True+ assert verification.verdicts[0].evidence_ref == "block_outputs:story.top_post_identified"+++def test_requested_output_path_dynamic_return_without_metadata_does_not_admit_runtime_output() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: story+ code: |+ return await extract_top_post()+ """+ )+ criteria = [+ CompletionCriterion(+ id="c_top_post",+ outcome="The top Hacker News post is identified.",+ output_path="output.top_post_identified",+ expected_output_value="true",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"story": {"top_post_identified": True}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "unproducible"+ assert verdicts[0].evidence_ref is None+++def test_requested_output_static_contract_scopes_runtime_evidence_to_same_label() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: story+ code: |+ return {"top_post_identified": False}+ """+ )+ criteria = [+ CompletionCriterion(+ id="c_top_post",+ outcome="The top Hacker News post is identified.",+ output_path="output.top_post_identified",+ expected_output_value="true",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"unrelated_lookup": {"top_post_identified": True}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "missing_exact_field"+ assert verdicts[0].evidence_ref is None+++def test_requested_output_runtime_debug_output_does_not_create_projection_contract() -> None:+ ctx = _run_ctx()+ ctx.code_artifact_metadata = {}+ ctx.last_workflow_yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: extract_profile+ code: |+ return {"npi": await page.locator("#npi").inner_text()}+ """+ )+ criteria = [+ CompletionCriterion(+ id="c_npi",+ outcome="The NPI is returned.",+ output_path="output.npi",+ expected_output_value="1234567890",+ )+ ]++ verdicts = grade_requested_output_criteria(+ ctx,+ criteria,+ RunEvidenceSnapshot(block_outputs={"extract_profile": {"debug_output": {"npi": "1234567890"}}}),+ )++ assert verdicts[0].state == "unsatisfied"+ assert verdicts[0].reason_code == "missing_exact_field"+ assert verdicts[0].evidence_ref is None++@pytest.mark.asyncioasync def test_requested_output_path_can_use_static_list_return_key_contract(monkeypatch: pytest.MonkeyPatch,@@ -5832,7 +6328,7 @@ async def test_requested_output_criteria_are_not_sent_to_judge(monkeypatch: pyte@pytest.mark.asyncio-async def test_present_generic_requested_output_without_expected_value_abstains_without_vetoing_run_success(+async def test_present_generic_requested_output_without_expected_value_structurally_abstains(monkeypatch: pytest.MonkeyPatch,) -> None:async def handler(**_: object) -> dict:@@ -5865,14 +6361,21 @@ async def test_present_generic_requested_output_without_expected_value_abstains_)assert verification is not None- assert verification.is_fully_satisfied() is Trueverdicts = {verdict.criterion_id: verdict for verdict in verification.verdicts}assert verdicts["c_customer_name"].reason_code == "structurally_abstained"+ assert verdicts["c_customer_name"].satisfied is Falseassert verdicts["c_submit"].reason_code == "evidence_confirms"+ assert verification.is_fully_satisfied() is True+ trace = verification.to_trace_data()+ assert trace["unmet_criterion_ids"] == ["c_customer_name"]+ assert trace["verdict_0_missing_evidence"] == (+ "requested-output field is present, but the criterion lacks typed expected_output_value or "+ "expected_output_shape to prove the value"+ )@pytest.mark.asyncio-async def test_all_abstained_requested_outputs_do_not_satisfy_completion(+async def test_present_requested_output_without_expected_value_cannot_satisfy_completion(monkeypatch: pytest.MonkeyPatch,) -> None:async def fail_handler(**_: object) -> object:@@ -6064,7 +6567,7 @@ async def test_unfired_contingent_requested_output_miss_does_not_veto_satisfied_ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")output = {"status": "DONE", "blocker": False}else:- output = {"status": "DONE", "blocker": False, "npi": "1234567890"}+ output = {"status": "DONE", "blocker": False}ctx.request_policy = RequestPolicy(completion_criteria=[_criterion(@@ -6104,7 +6607,7 @@ async def test_fired_contingent_requested_output_miss_still_vetoes(ctx.code_artifact_metadata = _metadata_for_requested_paths("npi")output = {"status": "DONE", "blocker": True}else:- output = {"status": "DONE", "blocker": True, "npi": "1234567890"}+ output = {"status": "DONE", "blocker": True}ctx.request_policy = RequestPolicy(completion_criteria=[_criterion(diff --git a/tests/unit/test_copilot_request_policy_completion_criteria.py b/tests/unit/test_copilot_request_policy_completion_criteria.pyindex d1bef12f1..d8548e427 100644--- a/tests/unit/test_copilot_request_policy_completion_criteria.py+++ b/tests/unit/test_copilot_request_policy_completion_criteria.py@@ -319,6 +319,54 @@ async def test_requested_output_criteria_preserve_stable_paths_without_derived_cassert [criterion.expected_output_shape for criterion in canonical_subsets[0]] == [None, None]+async def test_resale_docs_requested_output_uses_explicit_document_name_and_blocks_terminal_record_fields() -> None:+ policy = await _policy_for_message(+ (+ "I need a reusable workflow that retrieves the resale demand document name from the mock ResaleDocs Hub "+ "order-status page. The confirmation number should be a reusable input; for this eval run use "+ "DEMO-RESALE-1842. The workflow should look up the order, open the order-level View / Download document "+ "list, choose the highest-priority HOA demand or resale statement document row, and output one "+ "requested-output field named document_name containing the selected row's visible document name. Do not "+ "include an exact expected value for document_name in the workflow output contract, and do not output "+ "confirmation_number, order_id, request_id, submission_id, or a terminal-record identifier."+ ),+ [+ {+ "outcome": "The returned record includes visible document name.",+ "output_path": "output.visible_document_name",+ },+ {+ "outcome": "The returned record includes terminal record identifier.",+ "output_path": "output.terminal_record_identifier",+ },+ {+ "outcome": "The returned record includes confirmation number.",+ "output_path": "output.confirmation_number",+ },+ {+ "outcome": "The returned record includes order id.",+ "output_path": "output.order_id",+ },+ ],+ )++ assert [criterion.output_path for criterion in policy.completion_criteria] == ["output.document_name"]+ assert policy.completion_criteria[0].expected_output_value is None+ assert policy.completion_criteria[0].expected_output_shape is None+++async def test_named_customer_prose_does_not_become_requested_output_field_name() -> None:+ policy = await _policy_for_message(+ "Return a final record with customer named Acme and status.",+ [],+ )++ assert not _criteria_for_path(policy, "output.acme")+ assert _criteria_for_path(policy, "output.status")++@pytest.mark.asyncioasync def test_requested_output_criteria_do_not_infer_shapes_from_generated_provider_field_names() -> None:policy = await _policy_for_message(