Skip to content

Automate browser based workflows with AI

SKY-11545: Fail closed author-time output gates (#6941)

Andrew Neilson9266ba3539be
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 7beb642bc..34b463103 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -12,7 +12,9 @@ import structlog
import yaml
from pydantic import BaseModel, ConfigDict, Field
+from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.request_policy import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
@@ -174,6 +176,74 @@ def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:
return None
+def recorded_outcome_from_authoring_repair_context(
+ repair_context: CodeAuthoringRepairContext,
+) -> RecordedBuildTestOutcome:
+ reason_code = _authoring_reason_code(repair_context.reason_code)
+ identity_payload = {
+ "reason_code": repair_context.reason_code,
+ "unresolved_names": sorted(repair_context.unresolved_names),
+ "parameter_keys": sorted(repair_context.parameter_keys),
+ "available_parameter_keys": sorted(repair_context.available_parameter_keys),
+ "binding_candidates": sorted(repair_context.binding_candidates),
+ "selector": _bounded_ref(repair_context.selector),
+ "refiner_selector": _bounded_ref(repair_context.refiner_selector),
+ "runtime_failure_class": _bounded_ref(repair_context.runtime_failure_class),
+ "failed_block_status": _bounded_ref(repair_context.failed_block_status),
+ }
+ page_refs = _page_refs_from_authoring_context(repair_context)
+ return RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool="update_workflow",
+ attempted_block_label=repair_context.block_label,
+ verdict="authoring_rejected",
+ reason_code=reason_code,
+ block_labels=[repair_context.block_label],
+ workflow_run_id=repair_context.workflow_run_id,
+ structural_failure_identity="authoring:" + _stable_hash(identity_payload),
+ page_evidence_refs=page_refs,
+ observed_evidence_summary=_bounded_text(repair_context.runtime_failure_reason or repair_context.reason_code),
+ key_provenance={
+ "structural_failure_identity": "CodeAuthoringRepairContext structural fields",
+ "page_evidence_refs": "CodeAuthoringRepairContext bounded page fields",
+ },
+ )
+
+
+def recorded_outcome_from_author_time_reject(
+ *,
+ reason_code: BuildTestOutcomeReasonCode,
+ attempted_tool: str = "update_workflow",
+ attempted_block_label: str = "",
+ block_labels: Sequence[str] = (),
+ structural_failure_identity: str = "",
+ structural_payload: Mapping[str, object] | None = None,
+ authored_structure_signature: str | None = None,
+ observed_evidence_summary: str = "",
+ page_evidence_refs: Sequence[str] = (),
+ missing_requested_output_facts: Sequence[Mapping[str, object]] = (),
+) -> RecordedBuildTestOutcome:
+ if structural_payload is not None:
+ structural_failure_identity = "author_time:" + _stable_hash(structural_payload)
+ return RecordedBuildTestOutcome(
+ phase="author_time_reject",
+ attempted_tool=attempted_tool,
+ attempted_block_label=attempted_block_label,
+ verdict="authoring_rejected",
+ reason_code=reason_code,
+ block_labels=_clean_list(block_labels),
+ structural_failure_identity=structural_failure_identity,
+ authored_structure_signature=authored_structure_signature,
+ missing_requested_output_facts=[dict(fact) for fact in missing_requested_output_facts],
+ observed_evidence_summary=_bounded_text(observed_evidence_summary),
+ page_evidence_refs=_clean_list(page_evidence_refs),
+ key_provenance={
+ "structural_failure_identity": "author-time validator structural reason",
+ "page_evidence_refs": "author-time validator structural refs",
+ },
+ )
+
+
def recorded_outcome_from_loaded_result_evidence(
evidence: LoadedResultCompositionEvidence,
) -> RecordedBuildTestOutcome:
@@ -410,6 +480,19 @@ def _code_block_signature_payloads(value: object) -> list[dict[str, object]]:
return sorted(payloads, key=lambda item: str(item.get("label")))
+def _authoring_reason_code(value: str) -> BuildTestOutcomeReasonCode:
+ if value == SANDBOX_UNRESOLVED_NAME_REASON_CODE:
+ return "sandbox_unresolved_name"
+ if value == "synthesized_parameter_binding_ambiguous":
+ return "synthesized_parameter_binding_ambiguous"
+ if value == "runtime_block_failure":
+ return "runtime_block_failure"
+ if value == "select_option_interaction_mismatch":
+ # Select-option mismatches are author-time policy rejects, not a separate outcome class.
+ return "code_safety_reject"
+ return "code_safety_reject"
+
+
def _normalized_code_text(code: str) -> str:
return "\n".join(line.rstrip() for line in textwrap.dedent(code).strip().splitlines())
@@ -560,6 +643,19 @@ def _origin_ref(value: object) -> str:
return f"origin:{parsed.scheme}://{parsed.netloc}"
+def _page_refs_from_authoring_context(repair_context: CodeAuthoringRepairContext) -> list[str]:
+ refs: list[str] = []
+ if repair_context.current_origin:
+ refs.append(f"origin:{_bounded_ref(repair_context.current_origin)}")
+ for summary in repair_context.page_form_summaries[:3]:
+ refs.append(f"form:{_bounded_ref(summary)}")
+ for summary in repair_context.page_result_summaries[:3]:
+ refs.append(f"result:{_bounded_ref(summary)}")
+ for summary in repair_context.page_action_summaries[:3]:
+ refs.append(f"action:{_bounded_ref(summary)}")
+ return refs
+
+
def _page_evidence_refs(page_evidence: Mapping[str, object] | None) -> list[str]:
if page_evidence is None:
return []
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index 4dbcaf0ea..641988fcd 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -12,7 +12,7 @@ from dataclasses import dataclass
from functools import cache
from pathlib import Path
from types import SimpleNamespace
-from typing import Iterable
+from typing import Iterable, Mapping
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError, author_time_code_security_errors
from skyvern.forge.sdk.workflow.models.block import CodeBlock
@@ -50,6 +50,22 @@ _BROAD_TABLE_RECORD_KEYS = frozenset(("items", "locations", "records", "rows"))
_BROAD_TABLE_SCAN_SELECTORS = frozenset({"article", "section", ".card", "li"})
_BROAD_TABLE_SELECTOR_METHODS = frozenset(("locator", "query_selector", "query_selector_all"))
_LONE_LIST_ITEM_SELECTOR_EXEMPTION = frozenset({"li"})
+_GET_BY_TEXT_NARROWING_ATTRIBUTES = frozenset({"first", "last"})
+_GET_BY_TEXT_NARROWING_METHODS = frozenset({"filter", "first", "last", "nth"})
+_TABLE_WAIT_NARROWING_METHODS = frozenset(
+ {
+ "filter",
+ "get_by_alt_text",
+ "get_by_label",
+ "get_by_placeholder",
+ "get_by_role",
+ "get_by_test_id",
+ "get_by_text",
+ "get_by_title",
+ "locator",
+ "nth",
+ }
+)
_TABLE_ROW_TAG_SELECTOR_RE = re.compile(r"(?<![a-z0-9_-])tr(?![a-z0-9_-])")
_TABLE_ROW_ROLE_SELECTOR_RE = re.compile(r"\[role\s*=\s*(['\"]?)row\1\]")
@@ -380,13 +396,116 @@ def _author_time_ast_diagnostics(tree: ast.AST) -> list[CodeBlockPreflightDiagno
broad_table_scan = _broad_table_record_scan_diagnostic(tree)
if broad_table_scan is not None:
diagnostics.append(broad_table_scan)
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call):
+ diagnostics.extend(_alias_wait_diagnostics(tree))
+ return diagnostics
+
+
+def _alias_wait_diagnostics(tree: ast.AST) -> list[CodeBlockPreflightDiagnostic]:
+ diagnostics: list[CodeBlockPreflightDiagnostic] = []
+ statements = [node for node in ast.iter_child_nodes(tree) if isinstance(node, ast.stmt)]
+ _alias_wait_block_diagnostics(statements, {}, {}, diagnostics)
+ return diagnostics
+
+
+def _alias_wait_block_diagnostics(
+ statements: list[ast.stmt],
+ text_aliases: dict[str, bool],
+ table_aliases: dict[str, bool],
+ diagnostics: list[CodeBlockPreflightDiagnostic],
+) -> None:
+ for statement in statements:
+ _alias_wait_statement_diagnostics(statement, text_aliases, table_aliases, diagnostics)
+
+
+def _alias_wait_statement_diagnostics(
+ node: ast.stmt,
+ text_aliases: dict[str, bool],
+ table_aliases: dict[str, bool],
+ diagnostics: list[CodeBlockPreflightDiagnostic],
+) -> None:
+ if isinstance(node, (ast.Assign, ast.AnnAssign)):
+ assigned_value, _targets = _assignment_value_and_targets(node)
+ if assigned_value is not None:
+ _alias_wait_expr_diagnostics(assigned_value, text_aliases, table_aliases, diagnostics)
+ _update_global_get_by_text_aliases(node, text_aliases)
+ _update_global_table_locator_aliases(node, table_aliases)
+ return
+
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+ for decorator in node.decorator_list:
+ _alias_wait_expr_diagnostics(decorator, text_aliases, table_aliases, diagnostics)
+ _alias_wait_block_diagnostics(list(node.body), dict(text_aliases), dict(table_aliases), diagnostics)
+ return
+
+ for child in ast.iter_child_nodes(node):
+ if isinstance(child, ast.expr):
+ _alias_wait_expr_diagnostics(child, text_aliases, table_aliases, diagnostics)
+ for child_statements in _alias_wait_child_statement_blocks(node):
+ _alias_wait_block_diagnostics(child_statements, dict(text_aliases), dict(table_aliases), diagnostics)
+
+
+def _alias_wait_child_statement_blocks(node: ast.stmt) -> list[list[ast.stmt]]:
+ blocks: list[list[ast.stmt]] = []
+ for _field_name, value in ast.iter_fields(node):
+ if isinstance(value, list):
+ if all(isinstance(item, ast.stmt) for item in value):
+ blocks.append(value)
+ for item in value:
+ if isinstance(item, ast.ExceptHandler):
+ blocks.append(item.body)
+ return blocks
+
+
+def _alias_wait_expr_diagnostics(
+ node: ast.expr,
+ text_aliases: Mapping[str, bool],
+ table_aliases: Mapping[str, bool],
+ diagnostics: list[CodeBlockPreflightDiagnostic],
+) -> None:
+ for child in ast.walk(node):
+ if not isinstance(child, ast.Call):
continue
- body_text_wait_diagnostic = _broad_body_text_wait_for_function_diagnostic(node)
+ body_text_wait_diagnostic = _broad_body_text_wait_for_function_diagnostic(child)
if body_text_wait_diagnostic is not None:
diagnostics.append(body_text_wait_diagnostic)
- return diagnostics
+ global_text_wait_diagnostic = _global_get_by_text_wait_for_diagnostic(child, text_aliases)
+ if global_text_wait_diagnostic is not None:
+ diagnostics.append(global_text_wait_diagnostic)
+ global_table_wait_diagnostic = _global_table_wait_for_diagnostic(child, table_aliases)
+ if global_table_wait_diagnostic is not None:
+ diagnostics.append(global_table_wait_diagnostic)
+
+
+def _update_global_get_by_text_aliases(node: ast.Assign | ast.AnnAssign, aliases: dict[str, bool]) -> None:
+ assigned_value, targets = _assignment_value_and_targets(node)
+ if assigned_value is None:
+ return
+ is_global_text_locator, has_narrowing = _global_get_by_text_locator_chain(assigned_value, aliases)
+ for target in targets:
+ if isinstance(target, ast.Name):
+ if is_global_text_locator:
+ aliases[target.id] = has_narrowing
+ else:
+ aliases.pop(target.id, None)
+
+
+def _update_global_table_locator_aliases(node: ast.Assign | ast.AnnAssign, aliases: dict[str, bool]) -> None:
+ assigned_value, targets = _assignment_value_and_targets(node)
+ if assigned_value is None:
+ return
+ is_global_table_locator, has_narrowing = _global_table_locator_chain(assigned_value, aliases)
+ for target in targets:
+ if isinstance(target, ast.Name):
+ if is_global_table_locator:
+ aliases[target.id] = has_narrowing
+ else:
+ aliases.pop(target.id, None)
+
+
+def _assignment_value_and_targets(node: ast.Assign | ast.AnnAssign) -> tuple[ast.expr | None, list[ast.expr]]:
+ if isinstance(node, ast.Assign):
+ return node.value, list(node.targets)
+ return node.value, [node.target]
class _SandboxNameAnalyzer:
@@ -767,6 +886,107 @@ def _page_evaluate_diagnostic(node: ast.Call) -> CodeBlockPreflightDiagnostic |
)
+def _global_get_by_text_wait_for_diagnostic(
+ node: ast.Call,
+ aliases: Mapping[str, bool],
+) -> CodeBlockPreflightDiagnostic | None:
+ func = node.func
+ if not isinstance(func, ast.Attribute) or func.attr != "wait_for":
+ return None
+ is_global_text_locator, has_narrowing = _global_get_by_text_locator_chain(func.value, aliases)
+ if not is_global_text_locator or has_narrowing:
+ return None
+ return CodeBlockPreflightDiagnostic(
+ code="GLOBAL_GET_BY_TEXT_WAIT_FOR",
+ message=(
+ "Code block waits on global `page.get_by_text(...).wait_for(...)`, which can collide with "
+ "multiple matching text nodes under Playwright strict mode. Scope the text lookup "
+ "through a locator/container or narrow it with `first`, `nth`, or `filter` before waiting."
+ ),
+ )
+
+
+def _global_get_by_text_locator_chain(node: ast.expr, aliases: Mapping[str, bool]) -> tuple[bool, bool]:
+ if _is_global_page_get_by_text_call(node):
+ return True, False
+ if isinstance(node, ast.Name) and node.id in aliases:
+ return True, aliases[node.id]
+ if isinstance(node, ast.Attribute):
+ is_global_text_locator, has_narrowing = _global_get_by_text_locator_chain(node.value, aliases)
+ if is_global_text_locator and node.attr in _GET_BY_TEXT_NARROWING_ATTRIBUTES:
+ return True, True
+ return is_global_text_locator, has_narrowing
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ is_global_text_locator, has_narrowing = _global_get_by_text_locator_chain(node.func.value, aliases)
+ if is_global_text_locator and node.func.attr in _GET_BY_TEXT_NARROWING_METHODS:
+ return True, True
+ return is_global_text_locator, has_narrowing
+ return False, False
+
+
+def _global_table_wait_for_diagnostic(
+ node: ast.Call,
+ aliases: Mapping[str, bool],
+) -> CodeBlockPreflightDiagnostic | None:
+ func = node.func
+ if not isinstance(func, ast.Attribute) or func.attr != "wait_for":
+ return None
+ is_global_table_locator, has_narrowing = _global_table_locator_chain(func.value, aliases)
+ if not is_global_table_locator or has_narrowing:
+ return None
+ return CodeBlockPreflightDiagnostic(
+ code="BROAD_GLOBAL_TABLE_WAIT_FOR",
+ message=(
+ "Code block waits on broad `page.locator('table').wait_for(...)`. Pages can contain hidden layout "
+ "tables while the intended content region is ready. Wait on a scoped container or a row-level/narrowed "
+ "table locator before extracting output."
+ ),
+ )
+
+
+def _global_table_locator_chain(node: ast.expr, aliases: Mapping[str, bool]) -> tuple[bool, bool]:
+ if _is_global_page_table_locator_call(node):
+ return True, False
+ if isinstance(node, ast.Name) and node.id in aliases:
+ return True, aliases[node.id]
+ if isinstance(node, ast.Attribute):
+ return _global_table_locator_chain(node.value, aliases)
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ is_global_table_locator, has_narrowing = _global_table_locator_chain(node.func.value, aliases)
+ if is_global_table_locator and node.func.attr in _TABLE_WAIT_NARROWING_METHODS:
+ return True, True
+ return is_global_table_locator, has_narrowing
+ return False, False
+
+
+def _is_global_page_table_locator_call(node: ast.expr) -> bool:
+ if not (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "locator"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "page"
+ and node.args
+ ):
+ return False
+ selector = node.args[0]
+ return (
+ isinstance(selector, ast.Constant)
+ and isinstance(selector.value, str)
+ and selector.value.strip().casefold() == "table"
+ )
+
+
+def _is_global_page_get_by_text_call(node: ast.expr) -> bool:
+ return (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "get_by_text"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "page"
+ )
+
+
def _broad_body_text_wait_for_function_diagnostic(node: ast.Call) -> CodeBlockPreflightDiagnostic | None:
func = node.func
if (
@@ -794,8 +1014,8 @@ def _broad_body_text_wait_for_function_diagnostic(node: ast.Call) -> CodeBlockPr
code="BROAD_DOCUMENT_BODY_TEXT_WAIT",
message=(
"Code block waits for broad `document.body` text with `page.wait_for_function`. "
- "Loaded result/detail pages can be visible while body-level polling still times out. "
- "Wait on a localized result/detail locator or visible field text, then extract and return "
+ "Target content can be visible while body-level polling still times out. "
+ "Wait on a localized container or visible field text, then extract and return "
"a keyed record from that region."
),
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 75974fecc..ab38380ca 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -8,7 +8,7 @@ import keyword
import re
import textwrap
import tokenize
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from dataclasses import field as dataclass_field
from typing import Annotated, Any, Literal, NamedTuple
@@ -27,6 +27,15 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
clear_terminal_evidence_on_workflow_edit,
stash_blocker_signal,
)
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ BuildTestOutcomeReasonCode,
+ RecordedBuildTestOutcome,
+ authored_structure_signature_from_workflow,
+ latest_recorded_build_test_outcome_repeated,
+ record_build_test_outcome,
+ recorded_outcome_from_author_time_reject,
+ recorded_outcome_from_authoring_repair_context,
+)
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
author_time_code_block_diagnostics,
@@ -1135,6 +1144,7 @@ def _repair_context_log_values(values: list[str], *, max_items: int = 20) -> lis
def _set_code_authoring_repair_context(ctx: AgentContext, repair_context: CodeAuthoringRepairContext | None) -> None:
ctx.last_code_authoring_repair_context = repair_context
if repair_context is not None:
+ record_build_test_outcome(ctx, recorded_outcome_from_authoring_repair_context(repair_context))
LOG.info(
"copilot code authoring repair context stored",
reason_code=repair_context.reason_code,
@@ -1158,6 +1168,10 @@ _CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_p
def _record_code_authoring_guardrail_reject(ctx: AgentContext, *, defer_churn_stop: bool = False) -> None:
+ # Callers record the current build-test outcome first so repeat detection compares that key to history.
+ repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
+ if repeated_outcome is False:
+ ctx.code_authoring_guardrail_reject_count = 0
ctx.code_authoring_guardrail_reject_count += 1
ctx.last_code_authoring_reject_was_credential_priority = defer_churn_stop
LOG.info(
@@ -1181,6 +1195,727 @@ def _record_code_authoring_guardrail_reject(ctx: AgentContext, *, defer_churn_st
ctx.blocker_signal = signal
+def _record_author_time_reject_outcome(
+ ctx: AgentContext,
+ *,
+ reason_code: BuildTestOutcomeReasonCode,
+ summary: str,
+ structural_payload: Mapping[str, object] | None = None,
+ authored_structure_signature: str | None = None,
+ block_labels: list[str] | None = None,
+ missing_requested_output_facts: list[dict[str, object]] | None = None,
+) -> None:
+ record_build_test_outcome(
+ ctx,
+ recorded_outcome_from_author_time_reject(
+ reason_code=reason_code,
+ block_labels=block_labels or [],
+ structural_payload=structural_payload,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=summary,
+ missing_requested_output_facts=missing_requested_output_facts or [],
+ ),
+ )
+
+
+def _code_safety_reject_payload(errors: list[str | CodeBlockSecurityError]) -> Mapping[str, object] | None:
+ entries: list[dict[str, object]] = []
+ for error in errors:
+ if isinstance(error, CodeBlockSecurityError):
+ entries.append(
+ {
+ "block_label": error.block_label,
+ "reason_code": error.reason_code,
+ "surface": error.surface,
+ }
+ )
+ if not entries:
+ return None
+ return {"code_safety_errors": entries}
+
+
+def _credential_scout_reject_payload(workflow_yaml: str) -> Mapping[str, object] | None:
+ entries: list[dict[str, object]] = []
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ accesses = [
+ {
+ "parameter_key": access.parameter_key,
+ "field": access.field,
+ "requires_live_scout": access.requires_live_scout,
+ }
+ for access in _credential_field_accesses(code)
+ ]
+ if accesses or _CODE_SUBMIT_ACTION_RE.search(code):
+ entries.append(
+ {
+ "block_label": label,
+ "credential_field_accesses": accesses,
+ "requires_submit": bool(_CODE_SUBMIT_ACTION_RE.search(code)),
+ }
+ )
+ if not entries:
+ return None
+ return {"credential_scout_requirements": entries}
+
+
+def _code_artifact_metadata_reject_payload(
+ *,
+ workflow_yaml: str,
+ raw_metadata: object,
+ offending_labels: list[str],
+ violation_categories: list[str],
+ missing_labels: list[str] | None = None,
+) -> Mapping[str, object] | None:
+ labels = sorted(dict.fromkeys([*offending_labels, *(missing_labels or [])]))
+ if not labels:
+ labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ payload = {
+ "reason_code": "metadata_reject",
+ "offending_labels": labels,
+ "required_fields": [
+ "declared_goal",
+ *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS,
+ "evidence_refs_or_observation_refs",
+ ],
+ "missing_fields_by_label": _metadata_missing_required_fields_by_label(
+ raw_metadata,
+ labels=labels,
+ missing_labels=missing_labels or [],
+ ),
+ "output_path_roots": _metadata_output_path_roots(raw_metadata),
+ "output_path_roots_by_label": _metadata_output_path_roots_by_label(raw_metadata),
+ "code_block_output_status": _metadata_reject_code_block_output_status(
+ workflow_yaml,
+ raw_metadata=raw_metadata,
+ labels=labels,
+ ),
+ "violation_categories": sorted(dict.fromkeys(violation_categories)),
+ }
+ return payload
+
+
+def _metadata_missing_required_fields_by_label(
+ raw_metadata: object,
+ *,
+ labels: list[str],
+ missing_labels: list[str],
+) -> dict[str, list[str]]:
+ items_by_label: dict[str, Mapping[str, Any]] = {}
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is None:
+ continue
+ label = str(item.get("block_label") or "").strip()
+ if label and label not in items_by_label:
+ items_by_label[label] = item
+
+ missing_label_set = set(missing_labels)
+ missing_by_label: dict[str, list[str]] = {}
+ for label in labels:
+ item = items_by_label.get(label)
+ if item is None or label in missing_label_set:
+ missing_by_label[label] = [
+ "declared_goal",
+ *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS,
+ "evidence_refs_or_observation_refs",
+ ]
+ continue
+ missing_fields: list[str] = []
+ if not str(item.get("declared_goal") or "").strip():
+ missing_fields.append("declared_goal")
+ for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:
+ value = item.get(field_name)
+ if not isinstance(value, list) or not value:
+ missing_fields.append(field_name)
+ if not item.get("evidence_refs") and not item.get("observation_refs"):
+ missing_fields.append("evidence_refs_or_observation_refs")
+ if missing_fields:
+ missing_by_label[label] = missing_fields
+ return missing_by_label
+
+
+def _metadata_output_path_roots(raw_metadata: object) -> list[str]:
+ roots: set[str] = set()
+ for label_roots in _metadata_output_path_roots_by_label(raw_metadata).values():
+ roots.update(label_roots)
+ return sorted(roots)
+
+
+def _metadata_output_path_roots_by_label(raw_metadata: object) -> dict[str, list[str]]:
+ roots_by_label: dict[str, set[str]] = {}
+ unlabeled_index = 0
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is None:
+ continue
+ label = str(item.get("block_label") or "").strip()
+ if not label:
+ unlabeled_index += 1
+ label = f"<unlabeled:{unlabeled_index}>"
+ roots = roots_by_label.setdefault(label, set())
+ roots.update(_metadata_item_output_path_roots(item))
+ return {label: sorted(roots) for label, roots in sorted(roots_by_label.items()) if roots}
+
+
+def _metadata_item_output_path_roots(item: Mapping[str, Any]) -> set[str]:
+ roots: set[str] = set()
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
+ for path in _artifact_string_list(row.get("goal_value_paths")):
+ root = path.split(".", 1)[0].split("[", 1)[0].strip()
+ if root:
+ roots.add(root)
+ schema = _parse_extraction_schema(row.get("extraction_schema"))
+ if schema is not None:
+ roots.update(_schema_property_roots(schema))
+ return roots
+
+
+def _metadata_reject_code_block_output_status(
+ workflow_yaml: str,
+ *,
+ raw_metadata: object,
+ labels: list[str],
+) -> dict[str, Mapping[str, object]]:
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ metadata_by_label: dict[str, Mapping[str, Any]] = {}
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is None:
+ continue
+ label = str(item.get("block_label") or "").strip()
+ if label and label not in metadata_by_label:
+ metadata_by_label[label] = item
+
+ status: dict[str, Mapping[str, object]] = {}
+ for label in labels:
+ block = code_blocks.get(label)
+ if block is None:
+ continue
+ block_code = str(block.get("code") or "")
+ metadata = metadata_by_label.get(label)
+ status[label] = {
+ "block_type": _enum_or_string_name(block.get("block_type")) or str(block.get("block_type") or ""),
+ "has_code": bool(block_code.strip()),
+ "declares_output_intent": _block_declares_output_intent(block),
+ "declares_output_roots": sorted(_metadata_item_output_path_roots(metadata)) if metadata else [],
+ "has_meaningful_output": _code_block_has_meaningful_output(block_code, metadata),
+ }
+ return status
+
+
+def _schema_property_roots(schema: Mapping[str, object]) -> set[str]:
+ properties = schema.get("properties")
+ if isinstance(properties, Mapping):
+ return {str(key) for key in properties if str(key)}
+ items = schema.get("items")
+ if isinstance(items, Mapping):
+ return _schema_property_roots(items)
+ return set()
+
+
+def _metadata_violation_categories(violations: list[str]) -> list[str]:
+ categories: list[str] = []
+ for violation in violations:
+ if "requires non-empty" in violation:
+ categories.append("missing_required_list")
+ elif "requires a non-empty `declared_goal`" in violation:
+ categories.append("missing_declared_goal")
+ elif "requires `evidence_refs` or `observation_refs`" in violation:
+ categories.append("missing_artifact_refs")
+ elif "requires `source_tool`" in violation:
+ categories.append("missing_source_tool")
+ elif "goal_value_paths" in violation:
+ categories.append("invalid_goal_value_paths")
+ elif "extraction_schema" in violation:
+ categories.append("invalid_extraction_schema")
+ elif "return" in violation or "output" in violation:
+ categories.append("invalid_output_shape")
+ else:
+ categories.append("metadata_contract_violation")
+ return categories
+
+
+def _unchanged_after_recorded_outcome_signature(
+ ctx: AgentContext,
+ *,
+ workflow_yaml: str,
+ code_artifact_metadata: object,
+) -> str | None:
+ latest = ctx.latest_recorded_build_test_outcome
+ if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
+ return None
+ candidate_signature = authored_structure_signature_from_workflow(workflow_yaml, code_artifact_metadata)
+ if candidate_signature is None or candidate_signature != latest.authored_structure_signature:
+ return None
+ return candidate_signature
+
+
+def _recorded_outcome_requires_output_candidate(ctx: AgentContext) -> bool:
+ latest = ctx.latest_recorded_build_test_outcome
+ if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
+ return False
+ if latest.phase != "persisted_block_run" or latest.reason_code != "outcome_not_demonstrated":
+ return False
+ if latest.missing_requested_output_facts:
+ return True
+ verification = getattr(ctx, "completion_verification_result", None)
+ if verification is None or getattr(verification, "status", None) != "evaluated":
+ return False
+ return not verification.is_fully_satisfied()
+
+
+def _recorded_outcome_missing_output_roots(ctx: AgentContext) -> set[str]:
+ latest = ctx.latest_recorded_build_test_outcome
+ roots: set[str] = set()
+ if isinstance(latest, RecordedBuildTestOutcome) and latest.reason_code == "outcome_not_demonstrated":
+ for fact in latest.missing_requested_output_facts:
+ root = str(fact.get("output_root") or "").strip()
+ if root:
+ roots.add(root)
+ if roots:
+ return roots
+ verification = getattr(ctx, "completion_verification_result", None)
+ if verification is None or getattr(verification, "status", None) != "evaluated":
+ return set()
+ for verdict in getattr(verification, "verdicts", []):
+ if getattr(verdict, "satisfied", False):
+ continue
+ output_path = str(getattr(verdict, "output_path", "") or "").strip()
+ root = output_path.split(".", 1)[0].split("[", 1)[0].strip()
+ if root:
+ roots.add(root)
+ return roots
+
+
+def _candidate_missing_required_output_roots(
+ workflow_yaml: str,
+ code_artifact_metadata: object,
+ *,
+ required_roots: set[str],
+ runtime_produced_roots: set[str] | None = None,
+) -> list[str]:
+ if not required_roots:
+ return []
+ declared_roots = set(_metadata_output_path_roots(code_artifact_metadata))
+ produced_roots = _workflow_yaml_produced_output_roots(workflow_yaml)
+ covered_roots = (declared_roots & produced_roots) | (runtime_produced_roots or set())
+ return sorted(required_roots - covered_roots)
+
+
+def _recorded_runtime_produced_output_roots(ctx: AgentContext, workflow_yaml: str) -> set[str]:
+ verified_outputs = getattr(ctx, "verified_block_outputs", None)
+ if not isinstance(verified_outputs, Mapping):
+ return set()
+ code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ roots: set[str] = set()
+ for label, output in verified_outputs.items():
+ if not isinstance(label, str) or label not in code_block_labels:
+ continue
+ roots.update(_meaningful_runtime_output_roots(output))
+ return roots
+
+
+def _meaningful_runtime_output_roots(value: object, *, prefix: str = "") -> set[str]:
+ roots: set[str] = set()
+ if isinstance(value, Mapping):
+ for raw_key, child in value.items():
+ if not isinstance(raw_key, str):
+ continue
+ key = raw_key.strip()
+ if not key or key == "evidence_text" or not _is_structural_runtime_output_key(key):
+ continue
+ path = f"{prefix}.{key}" if prefix else key
+ if _runtime_output_value_is_meaningful(child):
+ roots.add(path.split(".", 1)[0])
+ roots.update(_meaningful_runtime_output_roots(child, prefix=path))
+ return roots
+ if isinstance(value, list):
+ for item in value:
+ roots.update(_meaningful_runtime_output_roots(item, prefix=prefix))
+ return roots
+
+
+def _runtime_output_value_is_meaningful(value: object) -> bool:
+ if value is None:
+ return False
+ if isinstance(value, str):
+ return bool(value.strip())
+ if isinstance(value, Mapping):
+ return any(_runtime_output_value_is_meaningful(item) for item in value.values())
+ if isinstance(value, list):
+ return any(_runtime_output_value_is_meaningful(item) for item in value)
+ return True
+
+
+def _workflow_yaml_produced_output_roots(workflow_yaml: str) -> set[str]:
+ roots: set[str] = set()
+ for block in _workflow_yaml_code_blocks_by_label(workflow_yaml).values():
+ roots.update(_code_block_produced_output_roots(str(block.get("code") or "")))
+ return roots
+
+
+def _code_block_produced_output_roots(code: str) -> set[str]:
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return set()
+ scope_statements = list(_iter_top_level_scope(tree.body))
+ assigned_roots = _assigned_top_level_names(tree.body)
+ dict_assignments: dict[str, set[str]] = {}
+ helper_return_roots = _helper_function_literal_return_roots(tree.body)
+ returned_roots: set[str] = set()
+ for node in scope_statements:
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ dict_roots = _dict_literal_string_key_roots(node.value)
+ if isinstance(node.value, ast.Dict):
+ dict_assignments[node.targets[0].id] = dict_roots
+ elif isinstance(node, ast.Assign):
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.AnnAssign):
+ if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):
+ dict_assignments[node.target.id] = _dict_literal_string_key_roots(node.value)
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.AugAssign):
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.Return) and node.value is not None:
+ returned_roots.update(_return_output_roots(node.value, dict_assignments, helper_return_roots))
+ return returned_roots or assigned_roots
+
+
+def _apply_literal_dict_key_assignment(
+ dict_assignments: dict[str, set[str]],
+ node: ast.Assign | ast.AnnAssign | ast.AugAssign,
+) -> None:
+ targets: list[ast.expr] = []
+ if isinstance(node, ast.Assign):
+ targets = list(node.targets)
+ else:
+ targets = [node.target]
+ for target in targets:
+ assignment = _literal_dict_key_assignment(target)
+ if assignment is None:
+ continue
+ name, key = assignment
+ if name in dict_assignments:
+ dict_assignments[name].add(key)
+
+
+def _literal_dict_key_assignment(target: ast.expr) -> tuple[str, str] | None:
+ if not isinstance(target, ast.Subscript) or not isinstance(target.value, ast.Name):
+ return None
+ key_node = target.slice
+ if isinstance(key_node, ast.Constant) and isinstance(key_node.value, str) and key_node.value.strip():
+ return target.value.id, key_node.value.strip()
+ return None
+
+
+def _return_output_roots(
+ node: ast.expr,
+ dict_assignments: Mapping[str, set[str]],
+ helper_return_roots: Mapping[str, set[str]],
+) -> set[str]:
+ if isinstance(node, ast.Await):
+ return _return_output_roots(node.value, dict_assignments, helper_return_roots)
+ if isinstance(node, ast.Name):
+ return set(dict_assignments.get(node.id, set()))
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+ return set(helper_return_roots.get(node.func.id, set()))
+ return _dict_literal_string_key_roots(node)
+
+
+def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[str, set[str]]:
+ helpers: dict[str, set[str]] = {}
+ for statement in statements:
+ if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ dict_assignments: dict[str, set[str]] = {}
+ roots: set[str] = set()
+ for helper_statement in _iter_top_level_scope(statement.body):
+ if (
+ isinstance(helper_statement, ast.Assign)
+ and len(helper_statement.targets) == 1
+ and isinstance(helper_statement.targets[0], ast.Name)
+ ):
+ dict_roots = _dict_literal_string_key_roots(helper_statement.value)
+ if isinstance(helper_statement.value, ast.Dict):
+ dict_assignments[helper_statement.targets[0].id] = dict_roots
+ elif isinstance(helper_statement, ast.Assign):
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.AnnAssign):
+ if isinstance(helper_statement.target, ast.Name) and isinstance(helper_statement.value, ast.Dict):
+ dict_assignments[helper_statement.target.id] = _dict_literal_string_key_roots(
+ helper_statement.value
+ )
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.AugAssign):
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.Return) and helper_statement.value is not None:
+ roots.update(_dict_literal_string_key_roots(helper_statement.value))
+ if isinstance(helper_statement.value, ast.Name):
+ roots.update(dict_assignments.get(helper_statement.value.id, set()))
+ if roots:
+ helpers[statement.name] = roots
+ return helpers
+
+
+def _dict_literal_string_key_roots(node: ast.expr) -> set[str]:
+ if not isinstance(node, ast.Dict):
+ return set()
+ roots: set[str] = set()
+ for key in node.keys:
+ if isinstance(key, ast.Constant) and isinstance(key.value, str) and key.value.strip():
+ roots.add(key.value.strip())
+ return roots
+
+
+def _output_empty_code_block_labels(workflow_yaml: str, code_artifact_metadata: object) -> list[str]:
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ if not code_blocks:
+ return []
+ metadata_by_label = code_artifact_metadata if isinstance(code_artifact_metadata, Mapping) else {}
+ labels_without_output = [
+ label
+ for label, block in code_blocks.items()
+ if not _code_block_has_meaningful_output(
+ str(block.get("code") or ""),
+ metadata_by_label.get(label),
+ )
+ ]
+ return sorted(labels_without_output) if len(labels_without_output) == len(code_blocks) else []
+
+
+def _code_block_has_meaningful_output(code: str, artifact: object) -> bool:
+ try:
+ tree = ast.parse(code)
+ except SyntaxError:
+ return False
+
+ class ReturnVisitor(ast.NodeVisitor):
+ def __init__(self) -> None:
+ self.has_meaningful_return = False
+
+ def visit_Return(self, node: ast.Return) -> None:
+ if _return_value_is_meaningful(node.value):
+ self.has_meaningful_return = True
+
+ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+ return
+
+ def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
+ return
+
+ def visit_ClassDef(self, node: ast.ClassDef) -> None:
+ return
+
+ visitor = ReturnVisitor()
+ visitor.visit(tree)
+ if visitor.has_meaningful_return:
+ return True
+ if _code_block_has_top_level_return(tree):
+ return False
+ if isinstance(artifact, Mapping):
+ goal_roots = _artifact_goal_value_roots(artifact)
+ if goal_roots and _missing_declared_output_roots(code, goal_roots) is None:
+ return True
+ return False
+
+
+@dataclass(frozen=True)
+class _ScoutedSelectOptionInteraction:
+ selector: str
+ source_origin: str
+ value_present: bool
+ label_present: bool
+
+
+def _scouted_select_option_interactions(ctx: AgentContext) -> list[_ScoutedSelectOptionInteraction]:
+ trajectory = getattr(ctx, "scout_trajectory", None)
+ if not isinstance(trajectory, list):
+ return []
+ interactions: list[_ScoutedSelectOptionInteraction] = []
+ seen: set[tuple[str, str, bool, bool]] = set()
+ for interaction in trajectory:
+ if not isinstance(interaction, Mapping):
+ continue
+ if str(interaction.get("tool_name") or "").strip() != "select_option":
+ continue
+ selector = _safe_selector_repair_atom(interaction.get("selector"))
+ if not selector:
+ continue
+ raw_source_url = str(interaction.get("source_url") or "").strip()
+ source_origin = (url_origin(raw_source_url) or "") if raw_source_url else ""
+ entry = _ScoutedSelectOptionInteraction(
+ selector=selector,
+ source_origin=source_origin,
+ value_present=bool(str(interaction.get("value") or "").strip()),
+ label_present=bool(str(interaction.get("label") or interaction.get("option_label") or "").strip()),
+ )
+ key = (entry.selector, entry.source_origin, entry.value_present, entry.label_present)
+ if key in seen:
+ continue
+ seen.add(key)
+ interactions.append(entry)
+ return interactions
+
+
+def _select_option_text_click_repair_context(
+ workflow_yaml: str, ctx: AgentContext
+) -> CodeAuthoringRepairContext | None:
+ expected_interactions = _scouted_select_option_interactions(ctx)
+ if not expected_interactions:
+ return None
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ candidate_selectors = _workflow_select_option_call_selectors(code_blocks.values())
+ if None in candidate_selectors:
+ return None
+ missing_interactions = [
+ interaction for interaction in expected_interactions if interaction.selector not in candidate_selectors
+ ]
+ if not missing_interactions:
+ return None
+ for label, block in code_blocks.items():
+ tree = _parsed_code_tree(str(block.get("code") or ""))
+ if tree is None or not _code_contains_get_by_text_click(tree):
+ continue
+ missing_interaction = missing_interactions[0]
+ return CodeAuthoringRepairContext(
+ block_label=label,
+ reason_code="select_option_interaction_mismatch",
+ selector=missing_interaction.selector,
+ source_url=missing_interaction.source_origin or None,
+ current_origin=missing_interaction.source_origin or None,
+ binding_candidates=[
+ "expected_tool:select_option",
+ "authored_tool:get_by_text_click",
+ f"value_present:{missing_interaction.value_present}",
+ f"label_present:{missing_interaction.label_present}",
+ ],
+ repair_instruction=(
+ "Use the captured select element API for this interaction, for example "
+ "page.locator(selector).select_option(...), instead of clicking option text."
+ ),
+ )
+ return None
+
+
+def _workflow_select_option_call_selectors(code_blocks: Iterable[Mapping[str, Any]]) -> set[str | None]:
+ selectors: set[str | None] = set()
+ for block in code_blocks:
+ tree = _parsed_code_tree(str(block.get("code") or ""))
+ if tree is not None:
+ selectors.update(_select_option_call_selectors(tree))
+ return selectors
+
+
+def _parsed_code_tree(code: str) -> ast.AST | None:
+ try:
+ return ast.parse(code)
+ except SyntaxError:
+ return None
+
+
+def _code_contains_get_by_text_click(tree: ast.AST) -> bool:
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "click"
+ and _call_chain_contains_method(node.func.value, "get_by_text")
+ ):
+ return True
+ return False
+
+
+def _select_option_call_selectors(tree: ast.AST) -> set[str | None]:
+ locator_aliases = _locator_alias_selectors(tree)
+ selectors: set[str | None] = set()
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
+ continue
+ if node.func.attr != "select_option":
+ continue
+ resolved = _locator_receiver_selectors(node.func.value, locator_aliases)
+ if resolved:
+ selectors.update(resolved)
+ else:
+ selectors.add(None)
+ return selectors
+
+
+def _locator_alias_selectors(tree: ast.AST) -> dict[str, set[str]]:
+ aliases: dict[str, set[str]] = {}
+
+ def record(target: ast.AST, value: ast.AST | None) -> None:
+ if not isinstance(target, ast.Name) or value is None:
+ return
+ selectors = _locator_receiver_selectors(value, aliases)
+ if selectors:
+ aliases[target.id] = selectors
+
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ record(target, node.value)
+ continue
+ if isinstance(node, ast.AnnAssign):
+ record(node.target, node.value)
+ return aliases
+
+
+def _locator_receiver_selectors(node: ast.AST, aliases: Mapping[str, set[str]]) -> set[str]:
+ while isinstance(node, ast.Attribute) and node.attr in {"first", "last"}:
+ node = node.value
+ if isinstance(node, ast.Name):
+ return set(aliases.get(node.id, set()))
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
+ return set()
+ if node.func.attr in {"first", "last", "nth", "filter"}:
+ return _locator_receiver_selectors(node.func.value, aliases)
+ if node.func.attr != "locator" or not node.args:
+ return set()
+ receiver = node.func.value
+ if not isinstance(receiver, ast.Name) or receiver.id != "page":
+ return set()
+ selector = node.args[0]
+ if isinstance(selector, ast.Constant) and isinstance(selector.value, str):
+ value = selector.value.strip()
+ return {value} if value else set()
+ return set()
+
+
+def _call_chain_contains_method(node: ast.AST, method_name: str) -> bool:
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ if node.func.attr == method_name:
+ return True
+ return _call_chain_contains_method(node.func.value, method_name)
+ if isinstance(node, ast.Attribute):
+ return _call_chain_contains_method(node.value, method_name)
+ return False
+
+
+def _code_block_has_top_level_return(tree: ast.AST) -> bool:
+ body = tree.body if isinstance(tree, ast.Module) else []
+ return any(isinstance(statement, ast.Return) for statement in _iter_top_level_scope(body))
+
+
+def _return_value_is_meaningful(node: ast.expr | None) -> bool:
+ if node is None:
+ return False
+ if isinstance(node, ast.Constant) and node.value is None:
+ return False
+ if (
+ isinstance(node, (ast.Dict, ast.List, ast.Tuple, ast.Set))
+ and not getattr(node, "elts", None)
+ and not getattr(node, "keys", None)
+ ):
+ return False
+ return True
+
+
def _signal_is_churn(signal: CopilotToolBlockerSignal | None) -> bool:
return signal is not None and signal.internal_reason_code in _CHURN_REASON_CODES
@@ -1531,15 +2266,15 @@ def _existing_metadata_covers_output(label: str, existing_metadata: Any) -> bool
return isinstance(metadata, Mapping) and _artifact_declares_goal_values(metadata)
-def _missing_code_artifact_metadata_error(workflow_yaml: str, ctx: AgentContext, raw_metadata: Any) -> str | None:
+def _missing_code_artifact_metadata_labels(workflow_yaml: str, ctx: AgentContext, raw_metadata: Any) -> list[str]:
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
- return None
+ return []
parsed = parse_workflow_yaml(workflow_yaml)
if not isinstance(parsed, Mapping):
- return None
+ return []
code_blocks = _workflow_code_blocks(dict(parsed))
if not code_blocks:
- return None
+ return []
has_workflow_output = _workflow_declares_output_parameter(parsed)
output_intent_blocks = [block for block in code_blocks if _block_declares_output_intent(block)]
if has_workflow_output and not output_intent_blocks and len(code_blocks) == 1:
@@ -1550,16 +2285,17 @@ def _missing_code_artifact_metadata_error(workflow_yaml: str, ctx: AgentContext,
if has_workflow_output or _block_declares_output_intent(block)
]
if not labels:
- return None
+ return []
existing_metadata = getattr(ctx, "code_artifact_metadata", None)
if _raw_code_artifact_metadata_empty(raw_metadata):
- missing = [label for label in labels if not _existing_metadata_covers_output(label, existing_metadata)]
- else:
- missing = [
- label
- for label in labels
- if not _raw_metadata_covers_output_label(raw_metadata, label, candidate_labels=labels)
- ]
+ return [label for label in labels if not _existing_metadata_covers_output(label, existing_metadata)]
+ return [
+ label for label in labels if not _raw_metadata_covers_output_label(raw_metadata, label, candidate_labels=labels)
+ ]
+
+
+def _missing_code_artifact_metadata_error(workflow_yaml: str, ctx: AgentContext, raw_metadata: Any) -> str | None:
+ missing = _missing_code_artifact_metadata_labels(workflow_yaml, ctx, raw_metadata)
if not missing:
return None
joined = ", ".join(f"`{label}`" for label in missing)
@@ -2448,7 +3184,6 @@ def _string_parameter_row(default_value: str, key: str) -> dict[str, Any]:
def _required_string_parameter_row(key: str) -> dict[str, Any]:
- # No default_value: direct scout fills become required workflow inputs at runtime.
return {
"parameter_type": "workflow",
"workflow_parameter_type": "string",
@@ -2643,49 +3378,6 @@ def _is_credential_parameter(parameter: Mapping[str, Any]) -> bool:
)
-def _submitted_string_parameter_default(
- parameters: list[Any],
- *,
- synthesized_key: str,
- typed_length: int | None,
-) -> tuple[dict[str, Any] | None, str | None]:
- candidates: list[dict[str, Any]] = []
- for parameter in parameters:
- if not isinstance(parameter, dict):
- continue
- candidate_key = str(parameter.get("key") or "").strip()
- if not candidate_key or candidate_key == synthesized_key:
- continue
- if _is_credential_parameter(parameter):
- continue
- parameter_type = str(parameter.get("parameter_type") or "").lower()
- workflow_parameter_type = str(parameter.get("workflow_parameter_type") or "").lower()
- if parameter_type and parameter_type != "workflow":
- continue
- if workflow_parameter_type and workflow_parameter_type != "string":
- continue
- default_value = parameter.get("default_value")
- if isinstance(default_value, str):
- candidates.append(parameter)
-
- if len(candidates) != 1:
- return None, None
-
- candidate = candidates[0]
- default_value = str(candidate.get("default_value") or "")
- if _SECRET_LIKE_LITERAL_RE.search(default_value):
- return (
- None,
- f"Unable to bind synthesized parameter `{synthesized_key}`: submitted parameter default looks credential-like.",
- )
- if typed_length is not None and typed_length > 0 and len(default_value) != typed_length:
- return (
- None,
- f"Unable to bind synthesized parameter `{synthesized_key}`: submitted parameter default length does not match the scout record.",
- )
- return candidate, None
-
-
def _string_parameter_default_value(parameter: Mapping[str, Any]) -> str | None:
if _is_credential_parameter(parameter):
return None
@@ -2719,29 +3411,6 @@ def _matching_string_parameter_key_by_default(
return matches[0] if len(matches) == 1 else None
-def _matching_string_parameter_key_by_default_length(
- parameters: list[Any],
- *,
- default_length: int | None,
- exclude_key: str,
-) -> str | None:
- if default_length is None or default_length <= 0:
- return None
- matches: list[str] = []
- for parameter in parameters:
- if not isinstance(parameter, dict):
- continue
- key = str(parameter.get("key") or "").strip()
- if not key or key == exclude_key:
- continue
- default_value = _string_parameter_default_value(parameter)
- if not default_value or _SECRET_LIKE_LITERAL_RE.search(default_value):
- continue
- if len(default_value) == default_length:
- matches.append(key)
- return matches[0] if len(matches) == 1 else None
-
-
def _drop_parameter_key(parameters: list[Any], key: str) -> None:
parameters[:] = [
parameter
@@ -2779,53 +3448,10 @@ def _is_python_identifier(value: str) -> bool:
return value.isidentifier() and not keyword.iskeyword(value)
-def _combined_string_default_expression(
- parameters: list[Any],
- *,
- synthesized_default: str,
- typed_length: int | None,
-) -> tuple[list[str], str] | None:
- if not synthesized_default and typed_length is None:
- return None
-
- candidates: list[tuple[str, str]] = []
- for parameter in parameters:
- if not isinstance(parameter, dict):
- continue
- key = str(parameter.get("key") or "").strip()
- if not _is_python_identifier(key):
- continue
- default_value = _string_parameter_default_value(parameter)
- if not default_value or _SECRET_LIKE_LITERAL_RE.search(default_value):
- continue
- candidates.append((key, default_value))
-
- matches: list[tuple[list[str], str]] = []
- for first_index, (first_key, first_default) in enumerate(candidates):
- for last_key, last_default in candidates[first_index + 1 :]:
- combined_default = f"{first_default} {last_default}".strip()
- if synthesized_default:
- if combined_default != synthesized_default:
- continue
- elif len(combined_default) != typed_length:
- continue
- matches.append(([first_key, last_key], f'(str({first_key}) + " " + str({last_key}))'))
- if len(matches) > 1:
- LOG.debug(
- "copilot_synthesized_parameter_combined_default_ambiguous",
- match_count=len(matches),
- synthesized_default_present=bool(synthesized_default),
- typed_length=typed_length,
- )
- return None
- return matches[0] if matches else None
-
-
class _SynthesizedParameterReconciliation(NamedTuple):
parameter_keys: list[str]
violations: list[str]
aliases: dict[str, str]
- expressions: dict[str, str]
repair_context: CodeAuthoringRepairContext | None = None
@@ -2833,8 +3459,6 @@ def _apply_parameter_reconciliation_to_code(code: str, reconciliation: _Synthesi
reconciled = code
for old_key, new_key in reconciliation.aliases.items():
reconciled = _replace_python_identifier(reconciled, old_key, new_key)
- for old_key, expression in reconciliation.expressions.items():
- reconciled = _replace_python_identifier(reconciled, old_key, expression)
return reconciled
@@ -2847,9 +3471,18 @@ def _synthesized_parameter_binding_repair_context(
code_block: Mapping[str, Any],
synthesized_key: str,
parameter_keys: list[str],
+ scout_trajectory: list[ScoutedInteraction],
+ synthesized_parameters: list[dict[str, str]],
) -> CodeAuthoringRepairContext:
available_parameter_keys = sorted(_declared_string_workflow_parameter_keys(parsed))
binding_candidates = [synthesized_key] + [key for key in available_parameter_keys if key != synthesized_key]
+ matched_scout = _scout_interaction_for_synthesized_parameter(
+ synthesized_key=synthesized_key,
+ scout_trajectory=scout_trajectory,
+ synthesized_parameters=synthesized_parameters,
+ )
+ selector = _safe_selector_repair_atom(matched_scout.get("selector")) if matched_scout is not None else ""
+ source_url = str(matched_scout.get("source_url") or "").strip() if matched_scout is not None else ""
return CodeAuthoringRepairContext(
block_label=str(code_block.get("label") or ""),
reason_code=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
@@ -2857,6 +3490,8 @@ def _synthesized_parameter_binding_repair_context(
parameter_keys=list(parameter_keys),
available_parameter_keys=available_parameter_keys,
binding_candidates=binding_candidates,
+ selector=selector or None,
+ source_url=url_origin(source_url) if source_url else None,
repair_instruction=(
f"Declare and use workflow string parameter `{synthesized_key}` exactly, include it in parameter_keys, "
"reference it as a bare Python variable in code, and rerun via update_and_run_blocks."
@@ -2864,6 +3499,28 @@ def _synthesized_parameter_binding_repair_context(
)
+def _scout_interaction_for_synthesized_parameter(
+ *,
+ synthesized_key: str,
+ scout_trajectory: list[ScoutedInteraction],
+ synthesized_parameters: list[dict[str, str]],
+) -> ScoutedInteraction | None:
+ non_credential_keys = [
+ str(parameter.get("key") or "").strip()
+ for parameter in synthesized_parameters
+ if str(parameter.get("key") or "").strip() and not parameter.get("credential_id")
+ ]
+ typed_interactions = [
+ interaction for interaction in scout_trajectory if str(interaction.get("tool_name") or "") == "type_text"
+ ]
+ if non_credential_keys.count(synthesized_key) != 1:
+ return None
+ index = non_credential_keys.index(synthesized_key)
+ if index >= len(typed_interactions):
+ return None
+ return typed_interactions[index]
+
+
def _reconcile_synthesized_parameters(
*,
parsed: dict[str, Any],
@@ -2875,7 +3532,7 @@ def _reconcile_synthesized_parameters(
workflow_definition = parsed.get("workflow_definition")
if not isinstance(workflow_definition, dict):
return _SynthesizedParameterReconciliation(
- [], ["Unable to bind synthesized parameters: workflow_definition is missing."], {}, {}
+ [], ["Unable to bind synthesized parameters: workflow_definition is missing."], {}
)
parameters = workflow_definition.get("parameters")
if parameters is None:
@@ -2883,7 +3540,7 @@ def _reconcile_synthesized_parameters(
workflow_definition["parameters"] = parameters
if not isinstance(parameters, list):
return _SynthesizedParameterReconciliation(
- [], ["Unable to bind synthesized parameters: workflow_definition.parameters must be a list."], {}, {}
+ [], ["Unable to bind synthesized parameters: workflow_definition.parameters must be a list."], {}
)
existing_by_key = {
@@ -2893,7 +3550,6 @@ def _reconcile_synthesized_parameters(
parameter_keys: list[str] = []
violations: list[str] = []
aliases: dict[str, str] = {}
- expressions: dict[str, str] = {}
repair_context: CodeAuthoringRepairContext | None = None
non_credential_synthesized = [param for param in synthesized_parameters if not param.get("credential_id")]
typed_lengths = [
@@ -2902,13 +3558,26 @@ def _reconcile_synthesized_parameters(
if str(interaction.get("tool_name") or "") == "type_text"
]
+ def add_binding_violation(key: str, message: str) -> None:
+ nonlocal repair_context
+ violations.append(message)
+ if repair_context is None:
+ repair_context = _synthesized_parameter_binding_repair_context(
+ parsed=parsed,
+ code_block=code_block,
+ synthesized_key=key,
+ parameter_keys=parameter_keys,
+ scout_trajectory=scout_trajectory,
+ synthesized_parameters=synthesized_parameters,
+ )
+
for synthesized_param in synthesized_parameters:
key = str(synthesized_param.get("key") or "").strip()
if not key:
violations.append("Unable to bind synthesized parameter: parameter key is missing.")
continue
if key in parameter_keys:
- violations.append(f"Unable to bind synthesized parameter `{key}`: duplicate synthesized key.")
+ add_binding_violation(key, f"Unable to bind synthesized parameter `{key}`: duplicate synthesized key.")
continue
parameter_keys.append(key)
@@ -2916,6 +3585,15 @@ def _reconcile_synthesized_parameters(
existing = existing_by_key.get(key)
synthesized_default = str(synthesized_param.get("default_value") or "").strip()
typed_length = _coerce_positive_int(synthesized_param.get("typed_length"))
+ matched_scout = _scout_interaction_for_synthesized_parameter(
+ synthesized_key=key,
+ scout_trajectory=scout_trajectory,
+ synthesized_parameters=synthesized_parameters,
+ )
+ scout_typed_length = (
+ _coerce_positive_int(matched_scout.get("typed_length")) if matched_scout is not None else None
+ )
+ typed_length = typed_length or scout_typed_length
if credential_id:
if existing is not None:
if existing_credentials.get(key) != credential_id:
@@ -2939,31 +3617,23 @@ def _reconcile_synthesized_parameters(
default_value=synthesized_default,
exclude_key=key,
)
- if alias_key is not None:
- aliases[key] = alias_key
- parameter_keys[-1] = alias_key
- _drop_parameter_key(parameters, key)
- continue
- alias_key = _matching_string_parameter_key_by_default_length(
- parameters,
- default_length=typed_length,
- exclude_key=key,
- )
if alias_key is not None:
aliases[key] = alias_key
parameter_keys[-1] = alias_key
_drop_parameter_key(parameters, key)
continue
if _is_credential_parameter(existing):
- violations.append(
- f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed."
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed.",
)
elif synthesized_default:
existing_default = _string_parameter_default_value(existing)
if existing_default != synthesized_default:
- violations.append(
+ add_binding_violation(
+ key,
f"Unable to bind synthesized parameter `{key}`: "
- "submitted parameter default does not match the scout record."
+ "submitted parameter default does not match the scout record.",
)
continue
@@ -2972,8 +3642,9 @@ def _reconcile_synthesized_parameters(
# captured from live type_text scouting are fully screened by
# safe_typed_default_value before they enter the scout trajectory.
if _SECRET_LIKE_LITERAL_RE.search(synthesized_default):
- violations.append(
- f"Unable to bind synthesized parameter `{key}`: synthesized default looks credential-like."
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: synthesized default looks credential-like.",
)
continue
alias_key = _matching_string_parameter_key_by_default(
@@ -2985,91 +3656,33 @@ def _reconcile_synthesized_parameters(
aliases[key] = alias_key
parameter_keys[-1] = alias_key
continue
- combined_expression = _combined_string_default_expression(
- parameters,
- synthesized_default=synthesized_default,
- typed_length=typed_length,
- )
- if combined_expression is not None:
- expression_keys, expression = combined_expression
- parameter_keys.pop()
- for expression_key in expression_keys:
- if expression_key not in parameter_keys:
- parameter_keys.append(expression_key)
- expressions[key] = expression
- continue
parameters.append(_string_parameter_row(synthesized_default, key))
continue
if len(non_credential_synthesized) != 1:
- alias_key = _matching_string_parameter_key_by_default_length(
- parameters,
- default_length=typed_length,
- exclude_key=key,
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.",
)
- if alias_key is not None:
- aliases[key] = alias_key
- parameter_keys[-1] = alias_key
- continue
-
- combined_expression = _combined_string_default_expression(
- parameters,
- synthesized_default=synthesized_default,
- typed_length=typed_length,
- )
- if combined_expression is not None:
- expression_keys, expression = combined_expression
- parameter_keys.pop()
- for expression_key in expression_keys:
- if expression_key not in parameter_keys:
- parameter_keys.append(expression_key)
- expressions[key] = expression
- continue
-
- if len(non_credential_synthesized) != 1:
- # Fall through only after length aliasing and combined-expression binding both miss.
- violations.append(
- f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous."
- )
- if repair_context is None:
- repair_context = _synthesized_parameter_binding_repair_context(
- parsed=parsed,
- code_block=code_block,
- synthesized_key=key,
- parameter_keys=parameter_keys,
- )
- continue
-
- typed_length = typed_lengths[0] if len(typed_lengths) == 1 else None
- submitted_parameter, error = _submitted_string_parameter_default(
- parameters,
- synthesized_key=key,
- typed_length=typed_length,
- )
- if error:
- violations.append(error)
- continue
- if submitted_parameter is not None:
- submitted_parameter["parameter_type"] = "workflow"
- submitted_parameter["workflow_parameter_type"] = "string"
- submitted_parameter["key"] = key
continue
+ typed_length = typed_length or scout_typed_length or (typed_lengths[0] if len(typed_lengths) == 1 else None)
direct_fill_usage = _submitted_direct_fill_type_usage(submitted_code, key)
- if direct_fill_usage.matched and not direct_fill_usage.mismatched:
- parameters.append(_required_string_parameter_row(key))
- continue
if direct_fill_usage.matched and direct_fill_usage.mismatched:
- violations.append(
+ add_binding_violation(
+ key,
f"Unable to bind synthesized parameter `{key}`: submitted code mixes direct fills using `{key}` "
"with other browser-locator fill/type values. Use the synthesized parameter for every scout-input "
- "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values."
+ "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values.",
)
continue
+ if direct_fill_usage.matched:
+ parameters.append(_required_string_parameter_row(key))
+ continue
literal, error = _safe_singleton_literal_for_parameter(submitted_code, key, typed_length)
if error:
- violations.append(error)
+ add_binding_violation(key, error)
continue
parameters.append(
{
@@ -3083,7 +3696,7 @@ def _reconcile_synthesized_parameters(
code_block["parameter_keys"] = parameter_keys
# Submitted synthesized parameter rows are re-derived from scout evidence before persistence.
code_block.pop("parameters", None)
- return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, expressions, repair_context)
+ return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, repair_context)
def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult:
@@ -3267,8 +3880,6 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
}
if parameter_reconciliation.aliases:
substitutions["parameter_aliases"] = parameter_reconciliation.aliases
- if parameter_reconciliation.expressions:
- substitutions["parameter_expressions"] = parameter_reconciliation.expressions
if extraction_suffix:
substitutions["preserved_extraction_suffix"] = True
if preserve_submitted_extraction:
@@ -4259,6 +4870,13 @@ def _reject_schema_incompatibility(
falling through to repair churn), record it on the context, and reject without
persisting the incompatible draft."""
signal = build_schema_incompatibility_blocker_signal(incompatibility)
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="schema_incompatibility",
+ summary=signal.user_facing_reason,
+ structural_payload=incompatibility.to_summary_dict(),
+ block_labels=[incompatibility.block_label],
+ )
stash_blocker_signal(ctx, signal)
stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
ctx.latest_schema_incompatibility = incompatibility
@@ -4359,6 +4977,22 @@ async def _update_workflow(
params.get("code_artifact_metadata"),
)
if missing_metadata_error is not None:
+ missing_labels = _missing_code_artifact_metadata_labels(
+ workflow_yaml, ctx, params.get("code_artifact_metadata")
+ )
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary=missing_metadata_error,
+ structural_payload=_code_artifact_metadata_reject_payload(
+ workflow_yaml=workflow_yaml,
+ raw_metadata=params.get("code_artifact_metadata"),
+ offending_labels=[],
+ missing_labels=missing_labels,
+ violation_categories=["missing_code_artifact_metadata"],
+ ),
+ block_labels=missing_labels,
+ )
return reject(
error=missing_metadata_error,
user_facing_summary=_compiled_authoring_user_summary(),
@@ -4376,6 +5010,18 @@ async def _update_workflow(
code_artifact_metadata_error = normalization.error
if code_artifact_metadata_error is not None:
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary=code_artifact_metadata_error,
+ structural_payload=_code_artifact_metadata_reject_payload(
+ workflow_yaml=workflow_yaml,
+ raw_metadata=params.get("code_artifact_metadata"),
+ offending_labels=normalization.offending_labels,
+ violation_categories=_metadata_violation_categories(normalization.violations),
+ ),
+ block_labels=normalization.offending_labels,
+ )
if normalization.schema_incompatibilities:
incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)
if incompatibility is not None:
@@ -4398,6 +5044,13 @@ async def _update_workflow(
)
if code_safety_errors:
_set_code_authoring_repair_context(ctx, code_authoring_repair_context)
+ if code_authoring_repair_context is None:
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="code_safety_reject",
+ summary="Code authoring guardrail rejected the submitted code block.",
+ structural_payload=_code_safety_reject_payload(code_safety_errors),
+ )
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=credential_priority_reject)
# Per-label salvage keeps conforming metadata across a rejection; a
# rejected code block keeps nothing, since its yaml never becomes the
@@ -4465,6 +5118,12 @@ async def _update_workflow(
repair_context=code_authoring_repair_context,
)
if credential_scout_errors:
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="credential_scout_reject",
+ summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
+ structural_payload=_credential_scout_reject_payload(workflow_yaml),
+ )
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
return reject(
error="\n".join(credential_scout_errors),
@@ -4555,6 +5214,140 @@ async def _update_workflow(
# New data-write blocks default to surfacing failures rather than swallowing them.
workflow_yaml = default_data_write_continue_on_failure(workflow_yaml, ctx.workflow_yaml)
+ authored_structure_signature = _unchanged_after_recorded_outcome_signature(
+ ctx,
+ workflow_yaml=workflow_yaml,
+ code_artifact_metadata=getattr(ctx, "code_artifact_metadata", None),
+ )
+ if authored_structure_signature is not None:
+ block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="unchanged_after_recorded_outcome",
+ summary="The authored code and output structure are unchanged after the last recorded test outcome.",
+ structural_payload={
+ "reason_code": "unchanged_after_recorded_outcome",
+ "authored_structure_signature": authored_structure_signature,
+ "block_labels": block_labels,
+ },
+ authored_structure_signature=authored_structure_signature,
+ block_labels=block_labels,
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error=(
+ "Submitted workflow is unchanged after the last recorded test outcome. "
+ "Revise the code block or output metadata before testing again."
+ ),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
+
+ recorded_output_roots_required = _recorded_outcome_requires_output_candidate(ctx)
+ recorded_missing_output_roots = (
+ _recorded_outcome_missing_output_roots(ctx) if recorded_output_roots_required else set()
+ )
+ runtime_produced_output_roots = (
+ _recorded_runtime_produced_output_roots(ctx, workflow_yaml) if recorded_output_roots_required else set()
+ )
+ unresolved_recorded_output_roots = recorded_missing_output_roots - runtime_produced_output_roots
+
+ output_empty_labels = (
+ _output_empty_code_block_labels(workflow_yaml, getattr(ctx, "code_artifact_metadata", None))
+ if recorded_output_roots_required and (not recorded_missing_output_roots or unresolved_recorded_output_roots)
+ else []
+ )
+ if output_empty_labels:
+ authored_structure_signature = authored_structure_signature_from_workflow(
+ workflow_yaml,
+ getattr(ctx, "code_artifact_metadata", None),
+ )
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary="Submitted workflow does not return any keyed output after the last recorded test outcome.",
+ structural_payload={
+ "reason_code": "recorded_outcome_requires_output_candidate",
+ "authored_structure_signature": authored_structure_signature,
+ "empty_output_block_labels": output_empty_labels,
+ "recorded_reason_code": "outcome_not_demonstrated",
+ },
+ authored_structure_signature=authored_structure_signature,
+ block_labels=output_empty_labels,
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error=(
+ "Submitted workflow does not return any keyed output after the last recorded test outcome. "
+ "Add structured output for the unsatisfied completion criteria before testing again."
+ ),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
+
+ missing_output_roots = (
+ _candidate_missing_required_output_roots(
+ workflow_yaml,
+ getattr(ctx, "code_artifact_metadata", None),
+ required_roots=unresolved_recorded_output_roots,
+ runtime_produced_roots=runtime_produced_output_roots,
+ )
+ if recorded_output_roots_required
+ else []
+ )
+ if missing_output_roots:
+ authored_structure_signature = authored_structure_signature_from_workflow(
+ workflow_yaml,
+ getattr(ctx, "code_artifact_metadata", None),
+ )
+ block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary="Submitted workflow does not cover the missing requested output roots from the last recorded test outcome.",
+ structural_payload={
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "authored_structure_signature": authored_structure_signature,
+ "missing_output_roots": missing_output_roots,
+ "block_labels": block_labels,
+ "recorded_reason_code": "outcome_not_demonstrated",
+ },
+ authored_structure_signature=authored_structure_signature,
+ block_labels=block_labels,
+ missing_requested_output_facts=[
+ {
+ "output_path": root,
+ "output_root": root,
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ for root in missing_output_roots
+ ],
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error=(
+ "Submitted workflow does not cover the missing requested output roots from the last recorded test "
+ "outcome. Declare and produce structured output for those roots before testing again."
+ ),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
+
+ select_option_mismatch_context = _select_option_text_click_repair_context(workflow_yaml, ctx)
+ if select_option_mismatch_context is not None:
+ _set_code_authoring_repair_context(ctx, select_option_mismatch_context)
+ _record_code_authoring_guardrail_reject(ctx)
+ return {
+ "ok": False,
+ "error": (
+ "Submitted workflow replaces a captured select-option interaction with a text click. "
+ "Use the select element API for the captured selector before testing again."
+ ),
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ "data": _code_repair_progress_data(select_option_mismatch_context),
+ }
+
try:
# A code block renders code-first (goal + plain step timeline) only when it
# carries a `prompt`; the model authors the goal as artifact `declared_goal`
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index 19056103e..fff4c744a 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -6,10 +6,14 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
authored_structure_signature_from_workflow,
record_build_test_outcome,
+ recorded_outcome_from_author_time_reject,
+ recorded_outcome_from_authoring_repair_context,
recorded_outcome_from_loaded_result_evidence,
recorded_outcome_from_run_blocks_result,
)
+from skyvern.forge.sdk.copilot.code_block_preflight import SANDBOX_UNRESOLVED_NAME_REASON_CODE
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvidence, LoadedResultCompositionTarget
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
@@ -326,6 +330,166 @@ def test_outcome_not_demonstrated_keeps_authoritative_unsatisfied_criteria_ident
assert "address and statuses" not in payload_text
+def test_authoring_repair_context_produces_structural_recorded_outcome() -> None:
+ context = CodeAuthoringRepairContext(
+ block_label="search_records",
+ reason_code=SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ unresolved_names=["confirmation_number", "row_text"],
+ parameter_keys=["confirmation_number"],
+ available_parameter_keys=["confirmation_number"],
+ binding_candidates=["confirmation_number", "row_text"],
+ )
+
+ outcome = recorded_outcome_from_authoring_repair_context(context)
+
+ assert outcome.phase == "author_time_reject"
+ assert outcome.reason_code == "sandbox_unresolved_name"
+ assert outcome.structural_key is not None
+ assert (
+ outcome.structural_key
+ == recorded_outcome_from_authoring_repair_context(
+ context.model_copy(update={"unresolved_names": ["row_text", "confirmation_number"]})
+ ).structural_key
+ )
+
+
+def test_author_time_reject_structural_payloads_make_distinct_keys() -> None:
+ first = recorded_outcome_from_author_time_reject(
+ reason_code="schema_incompatibility",
+ structural_payload={
+ "block_label": "extract_record",
+ "incompatible_paths": ["records[].expiration_date"],
+ "known_output_paths": ["records[].name"],
+ },
+ )
+ second = recorded_outcome_from_author_time_reject(
+ reason_code="schema_incompatibility",
+ structural_payload={
+ "block_label": "extract_record",
+ "incompatible_paths": ["records[].license_number"],
+ "known_output_paths": ["records[].name"],
+ },
+ )
+
+ assert first.structural_key is not None
+ assert second.structural_key is not None
+ assert first.structural_key != second.structural_key
+
+
+def test_metadata_reject_preserves_missing_requested_output_facts() -> None:
+ outcome = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "missing_output_roots": ["address", "credentialing_status"],
+ "block_labels": ["lookup_provider_and_extract_credentials"],
+ },
+ missing_requested_output_facts=[
+ {
+ "output_path": "address",
+ "output_root": "address",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.is_authoritative is True
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "address",
+ "output_root": "address",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ assert outcome.structural_key_payload is not None
+ assert "address" in str(outcome.structural_key_payload)
+
+
+def test_author_time_reject_without_structural_payload_is_not_authoritative() -> None:
+ outcome = recorded_outcome_from_author_time_reject(
+ reason_code="code_safety_reject",
+ observed_evidence_summary="Rewrite the code without unsafe behavior.",
+ )
+
+ assert outcome.is_authoritative is False
+ assert outcome.structural_key is None
+
+
+def test_metadata_reject_key_uses_typed_fields_not_wording() -> None:
+ first = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "metadata_reject",
+ "offending_labels": ["search_registry"],
+ "required_fields": ["claimed_outcomes", "completion_criteria"],
+ "missing_fields_by_label": {"search_registry": ["claimed_outcomes"]},
+ "violation_categories": ["missing_required_list"],
+ },
+ observed_evidence_summary="Metadata requires non-empty claimed_outcomes.",
+ )
+ same_structure = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "metadata_reject",
+ "offending_labels": ["search_registry"],
+ "required_fields": ["claimed_outcomes", "completion_criteria"],
+ "missing_fields_by_label": {"search_registry": ["claimed_outcomes"]},
+ "violation_categories": ["missing_required_list"],
+ },
+ observed_evidence_summary="Different wording for the same typed metadata failure.",
+ )
+ changed_label = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "metadata_reject",
+ "offending_labels": ["extract_registry"],
+ "required_fields": ["claimed_outcomes", "completion_criteria"],
+ "missing_fields_by_label": {"extract_registry": ["claimed_outcomes"]},
+ "violation_categories": ["missing_required_list"],
+ },
+ )
+ changed_required_field = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "metadata_reject",
+ "offending_labels": ["search_registry"],
+ "required_fields": ["terminal_verifier_expectations"],
+ "missing_fields_by_label": {"search_registry": ["terminal_verifier_expectations"]},
+ "violation_categories": ["missing_required_list"],
+ },
+ )
+ changed_missing_field = recorded_outcome_from_author_time_reject(
+ reason_code="metadata_reject",
+ structural_payload={
+ "reason_code": "metadata_reject",
+ "offending_labels": ["search_registry"],
+ "required_fields": ["claimed_outcomes", "completion_criteria"],
+ "missing_fields_by_label": {"search_registry": ["completion_criteria"]},
+ "violation_categories": ["missing_required_list"],
+ },
+ )
+
+ assert first.structural_key == same_structure.structural_key
+ assert changed_label.structural_key != first.structural_key
+ assert changed_required_field.structural_key != first.structural_key
+ assert changed_missing_field.structural_key != first.structural_key
+
+
def test_runtime_block_failure_outcome_includes_bounded_page_state_and_run_id() -> None:
result = {
"ok": False,
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index cfc367dae..b0939e0d2 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -12,17 +12,24 @@ from types import SimpleNamespace
import pytest
import yaml
+from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
)
+from skyvern.forge.sdk.copilot.build_test_outcome import (
+ RecordedBuildTestOutcome,
+ authored_structure_signature_from_workflow,
+)
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
_sandbox_shim_surface,
strip_redundant_sandbox_imports,
)
+from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError
from skyvern.forge.sdk.copilot.code_block_synthesis import _get_by_role_expr, _get_by_role_expr_strict
+from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
@@ -42,7 +49,10 @@ from skyvern.forge.sdk.copilot.tools import (
)
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
-from skyvern.forge.sdk.copilot.tools.workflow_update import _strip_redundant_sandbox_imports_in_yaml
+from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ _code_safety_reject_payload,
+ _strip_redundant_sandbox_imports_in_yaml,
+)
from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind, _kind_for_blocker_signal
from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks
from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
@@ -1055,250 +1065,1163 @@ class TestCodeRepairProgressClassification:
assert result["data"]["progress_text"] != result["user_facing_summary"]
@pytest.mark.asyncio
- async def test_raw_conflict_marker_reject_carries_progress_surface_kind(
+ async def test_generated_code_preflight_string_reject_is_not_authoritative(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
- _stub_successful_update(monkeypatch)
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_code_block_safety_errors",
+ lambda workflow_yaml, prior_yaml: [
+ "Code block `search_registry` failed the generated-code preflight check: "
+ "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."
+ ],
+ )
ctx = _code_only_ctx()
- result = await _update_workflow({"workflow_yaml": f"<<<<<<< HEAD\n{_SAFE_CODE_YAML}"}, ctx)
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx)
+
assert result["ok"] is False
- assert result["data"]["surface_kind"] == "code_repair_progress"
- assert result["data"]["progress_text"]
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "code_safety_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is False
+ assert ctx.latest_recorded_build_test_outcome.structural_key is None
- @pytest.mark.asyncio
- async def test_credential_scout_reject_is_not_classified_as_progress(self) -> None:
- ctx = _code_only_ctx()
- ctx.scout_trajectory = []
- result = await _update_workflow(
- {
- "workflow_yaml": _credential_code_yaml(
- code="""
- await page.locator("#email").fill(login_credential.username)
- await page.locator("input[type='password']").fill(login_credential.password)
- await page.locator("#totpmfa").fill(login_credential.totp)
- await page.locator("input[type='submit']").click()
- await page.wait_for_load_state("load")
- """
+ def test_code_safety_payload_keeps_typed_security_error_authoritative(self) -> None:
+ payload = _code_safety_reject_payload(
+ [
+ CodeBlockSecurityError(
+ "Code block `search_registry` failed the Copilot code security check: page.request is not allowed.",
+ block_label="search_registry",
+ reason_code="AUTHOR_PAGE_REQUEST",
+ surface="page.request",
)
- },
- ctx,
+ ]
)
- assert result["ok"] is False
- assert result["data"]["failure_type"] == "missing_credential_or_init"
- assert result["data"].get("surface_kind") is None
+ assert payload == {
+ "code_safety_errors": [
+ {
+ "block_label": "search_registry",
+ "reason_code": "AUTHOR_PAGE_REQUEST",
+ "surface": "page.request",
+ }
+ ]
+ }
-class TestCodeBlockParameterPersistSeam:
- @pytest.mark.asyncio
- async def test_undeclared_parameter_key_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
- _stub_successful_update(monkeypatch)
- ctx = _code_only_ctx()
- ctx.workflow_yaml = _SAFE_CODE_YAML
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
- submitted = _directory_blocks_yaml(
- """
- - block_type: code
- label: search_directory
- parameter_keys: [address_or_postal_code]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- """
+ def test_code_safety_payload_rejects_string_only_preflight_authority(self) -> None:
+ payload = _code_safety_reject_payload(
+ [
+ "Code block `search_registry` failed the generated-code preflight check: "
+ "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."
+ ]
)
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
-
- assert result["ok"] is False
- assert "search_directory" in result["error"]
- assert "`address_or_postal_code`" in result["error"]
- assert "workflow_definition.parameters" in result["error"]
- assert ctx.workflow_yaml == _SAFE_CODE_YAML
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
+ assert payload is None
@pytest.mark.asyncio
- async def test_nested_code_block_undeclared_parameter_key_rejects(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_authoritative_recorded_outcome_rejects_unchanged_authored_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- submitted = _directory_blocks_yaml(
- """
- - block_type: loop
- label: retry_search
- loop_blocks:
- - block_type: code
- label: nested_directory_search
- parameter_keys: [address_or_postal_code]
- code: |
- await page.locator("#location").fill(str(address_or_postal_code))
- """
+ ctx.workflow_yaml = _code_yaml('await page.goto("https://example.com/old")')
+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ authored_structure_signature=signature,
)
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
- assert "nested_directory_search" in result["error"]
- assert "`address_or_postal_code`" in result["error"]
- assert ctx.workflow_yaml == ""
-
- def test_deeply_nested_parameter_contract_returns_validation_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
- block: dict[str, object] = {
- "block_type": "code",
- "label": "search_directory",
- "parameter_keys": ["search_location"],
- "code": "print(search_location)",
- }
- for index in range(1100):
- block = {
- "block_type": "loop",
- "label": f"loop_{index}",
- "loop_blocks": [block],
- }
- parsed = {
- "workflow_definition": {
- "parameters": [{"key": "search_location"}],
- "blocks": [block],
- }
- }
- monkeypatch.setattr(workflow_update_module, "parse_workflow_yaml", lambda _workflow_yaml: parsed)
+ assert "unchanged after the last recorded test outcome" in result["error"]
+ assert ctx.workflow_yaml == _code_yaml('await page.goto("https://example.com/old")')
+ assert ctx.has_staged_proposal is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "unchanged_after_recorded_outcome"
+ assert ctx.latest_recorded_build_test_outcome.structural_key is not None
- assert (
- workflow_update_module._code_block_parameter_contract_error("workflow_definition: {}")
- == "Workflow YAML nesting is too deep to validate."
- )
+ @pytest.mark.asyncio
+ async def test_repeated_unchanged_authored_candidate_keeps_same_reject_key_and_backstop(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ authored_structure_signature=signature,
+ )
+
+ first = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ first_count = ctx.code_authoring_guardrail_reject_count
+ second = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert first_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
+ assert first_count == 1
+ assert ctx.code_authoring_guardrail_reject_count == 2
@pytest.mark.asyncio
- async def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_authoritative_recorded_outcome_allows_changed_authored_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- submitted = _code_yaml(
- "await page.locator('#query').fill(str(provider_query))",
- parameter_keys=["provider_query"],
- workflow_param=True,
+ signature = authored_structure_signature_from_workflow(_SAFE_CODE_YAML)
+ changed_yaml = _code_yaml('await page.goto("https://example.com/search")\nvalue = "changed"')
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-outcome",
+ authored_structure_signature=signature,
)
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": changed_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
+ assert ctx.workflow_yaml == changed_yaml
+ assert ctx.has_staged_proposal is True
@pytest.mark.asyncio
- async def test_new_declared_workflow_parameter_key_is_accepted_on_later_edit(
+ async def test_authoritative_outcome_not_demonstrated_rejects_output_empty_changed_candidate(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.workflow_yaml = _code_yaml(
- "await page.locator('#query').fill(str(provider_query))",
- parameter_keys=["provider_query"],
- workflow_param=True,
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
+ ],
)
- submitted = _yaml(
- """
- title: Registry lookup
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
workflow_definition:
parameters:
- - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample Search}
- - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: Example City}
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: address_city_county_or_zip_code
blocks:
- block_type: code
- label: search_registry
- parameter_keys: [provider_query, search_location]
+ label: {label}
+ parameter_keys:
+ - address_city_county_or_zip_code
code: |
- await page.locator("#query").fill(str(provider_query))
- await page.locator("#location").fill(str(search_location))
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ await page.locator("#locInput").fill(str(address_city_county_or_zip_code))
+ return {{}}
"""
)
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
- assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
+ assert result["ok"] is False
+ assert "does not return any keyed output" in result["error"]
+ assert ctx.has_staged_proposal is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
- async def test_prior_block_output_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_authoritative_outcome_not_demonstrated_allows_implicit_keyed_output_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- submitted = _directory_blocks_yaml(
- """
- - block_type: code
- label: search_registry
- code: |
- return {"records": []}
- - block_type: code
- label: summarize_registry
- parameter_keys: [search_registry_output]
- code: |
- print(search_registry_output)
+ label = "lookup_provider_and_extract_credentials"
+ ctx.completion_verification_result = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["npi", "locations"],
+ verdicts=[
+ CriterionVerdict(criterion_id="npi", state="unsatisfied", reason_code="no_evidence"),
+ CriterionVerdict(criterion_id="locations", state="unsatisfied", reason_code="no_evidence"),
+ ],
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["npi", "locations"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["npi", "locations"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ npi = "1234567890"
+ locations = [{{"address": "123 Main St"}}]
"""
)
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.workflow_yaml == submitted
+ assert ctx.workflow_yaml == candidate_yaml
- @pytest.mark.parametrize(
- ("submitted", "label", "key"),
- [
- (
- _directory_blocks_yaml(
- """
- - block_type: code
- label: summarize_registry
- parameter_keys: [search_registry_output]
- code: |
- print(search_registry_output)
- - block_type: code
- label: search_registry
- code: |
- return {"records": []}
- """
- ),
- "summarize_registry",
- "search_registry_output",
- ),
- (
- _directory_blocks_yaml(
- """
- - block_type: loop
- label: retry_search
- loop_blocks:
- - block_type: code
- label: nested_directory_search
- parameter_keys: [retry_search_output]
- code: |
- print(retry_search_output)
- """
- ),
- "nested_directory_search",
- "retry_search_output",
- ),
- ],
- )
@pytest.mark.asyncio
- async def test_unavailable_output_parameter_key_rejects(
- self, monkeypatch: pytest.MonkeyPatch, submitted: str, label: str, key: str
+ async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ ],
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ address = "North Carolina, USA"
+ """
+ )
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
- assert label in result["error"]
- assert f"`{key}`" in result["error"]
- assert ctx.workflow_yaml == ""
+ assert "does not cover the missing requested output roots" in result["error"]
+ assert ctx.has_staged_proposal is False
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
- @pytest.mark.parametrize("marker", ["<<<<<<< HEAD", "======="])
@pytest.mark.asyncio
- async def test_raw_workflow_yaml_conflict_marker_rejects_before_persist(
- self, monkeypatch: pytest.MonkeyPatch, marker: str
+ async def test_metadata_reject_persists_missing_roots_for_next_author_context(
+ self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- ctx.workflow_yaml = _SAFE_CODE_YAML
- submitted = f"{marker}\n{_SAFE_CODE_YAML}"
+ label = "lookup_provider_and_extract_credentials"
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ return {{"search_options": []}}
+ """
+ )
- result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "address",
+ "output_root": "address",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ rendered = agent_module._recorded_build_test_outcome_prompt(ctx)
+ assert "missing_requested_output_facts:" in rendered
+ assert "output_root=address" in rendered
+ assert "output_root=credentialing_status" in rendered
+ assert "value_status=no_typed_value" in rendered
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_when_last_run_produced_required_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.verified_block_outputs[label] = {
+ "address": "North Carolina, USA",
+ "credentialing_status": "Active",
+ "locations": [{"address": "North Carolina, USA"}],
+ "statuses": ["Active"],
+ }
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ address = "North Carolina, USA"
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_still_rejects_empty_last_run_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.verified_block_outputs[label] = {
+ "address": "",
+ "credentialing_status": "",
+ "locations": [],
+ "statuses": [],
+ }
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "does not return any keyed output" in result["error"]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_candidate_covering_missing_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [
+ {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
+ ],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["address", "credentialing_status", "locations", "statuses"]}
+ ],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "locations", "output_root": "locations", "value_status": "empty_typed_value"},
+ {"output_path": "statuses", "output_root": "statuses", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ address = "North Carolina, USA"
+ credentialing_status = "unknown"
+ locations = []
+ statuses = []
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_returning_missing_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ async def extract():
+ return {{"address": "North Carolina", "credentialing_status": "unknown"}}
+
+ return await extract()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_local_dict_output_roots(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ async def extract():
+ result = {{"address": "North Carolina", "credentialing_status": "unknown"}}
+ return result
+
+ return await extract()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_top_level_literal_key_dict_updates(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ result = {{"address": "North Carolina"}}
+ result["credentialing_status"] = "unknown"
+ return result
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_helper_literal_key_dict_updates(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["address", "credentialing_status"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ async def extract():
+ result = {{}}
+ result["address"] = "North Carolina"
+ result["credentialing_status"] = "unknown"
+ return result
+
+ return await extract()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == candidate_yaml
+
+ @pytest.mark.asyncio
+ async def test_captured_select_option_rejects_authored_text_click_plan_selection(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans?session=private",
+ "value": "gold",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Plan lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
+ blocks:
+ - block_type: code
+ label: plan_selection
+ parameter_keys:
+ - plan
+ code: |
+ main = page.locator("main")
+ await main.get_by_text(str(plan), exact=True).first.click()
+ return {"selected_plan": str(plan)}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "select-option interaction with a text click" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "select_option_interaction_mismatch"
+ assert repair_context["block_label"] == "plan_selection"
+ assert repair_context["selector"] == "#planSelect"
+ assert repair_context["source_url"] == "https://example.com"
+ assert "private" not in str(repair_context)
+ assert ctx.last_code_authoring_repair_context is not None
+ assert ctx.last_code_authoring_repair_context.reason_code == "select_option_interaction_mismatch"
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "code_safety_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert ctx.has_staged_proposal is False
+
+ @pytest.mark.asyncio
+ async def test_captured_select_option_allows_authored_select_api(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans",
+ "value": "gold",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Plan lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
+ blocks:
+ - block_type: code
+ label: plan_selection
+ parameter_keys:
+ - plan
+ code: |
+ await page.locator("#planSelect").select_option(label=str(plan))
+ return {"selected_plan": str(plan)}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_captured_select_option_allows_later_unrelated_text_click(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans",
+ "value": "gold",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Plan lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
+ blocks:
+ - block_type: code
+ label: plan_selection
+ parameter_keys:
+ - plan
+ code: |
+ await page.locator("#planSelect").select_option(label=str(plan))
+ return {"selected_plan": str(plan)}
+ - block_type: code
+ label: continue_from_plan
+ code: |
+ await page.get_by_text("Continue", exact=True).click()
+ return {"continued": True}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_runtime_hidden_option_failure_rejects_repeated_text_click_structure(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "select_option",
+ "selector": "#planSelect",
+ "source_url": "https://example.com/plans",
+ "value": "gold",
+ "trajectory_index": 0,
+ }
+ ]
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ structural_failure_identity="runtime:hidden-option-click",
+ authored_structure_signature="authored:previous-hidden-option-click",
+ )
+
+ def submitted_yaml(return_value: str) -> str:
+ return _yaml(
+ f"""
+ title: Plan lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: plan
+ blocks:
+ - block_type: code
+ label: plan_selection
+ parameter_keys:
+ - plan
+ code: |
+ main = page.locator("main")
+ await main.get_by_text(str(plan), exact=True).first.click()
+ return {{"selected_plan": "{return_value}"}}
+ """
+ )
+
+ first = await _update_workflow({"workflow_yaml": submitted_yaml("first")}, ctx, allow_missing_credentials=True)
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ first_count = ctx.code_authoring_guardrail_reject_count
+ second = await _update_workflow(
+ {"workflow_yaml": submitted_yaml("second")}, ctx, allow_missing_credentials=True
+ )
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert first_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
+ assert first_count == 1
+ assert ctx.code_authoring_guardrail_reject_count == 2
+ assert ctx.has_staged_proposal is False
+
+ @pytest.mark.asyncio
+ async def test_raw_conflict_marker_reject_carries_progress_surface_kind(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": f"<<<<<<< HEAD\n{_SAFE_CODE_YAML}"}, ctx)
+ assert result["ok"] is False
+ assert result["data"]["surface_kind"] == "code_repair_progress"
+ assert result["data"]["progress_text"]
+
+ @pytest.mark.asyncio
+ async def test_credential_scout_reject_is_not_classified_as_progress(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ result = await _update_workflow(
+ {
+ "workflow_yaml": _credential_code_yaml(
+ code="""
+ await page.locator("#email").fill(login_credential.username)
+ await page.locator("input[type='password']").fill(login_credential.password)
+ await page.locator("#totpmfa").fill(login_credential.totp)
+ await page.locator("input[type='submit']").click()
+ await page.wait_for_load_state("load")
+ """
+ )
+ },
+ ctx,
+ )
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert result["data"].get("surface_kind") is None
+
+
+class TestCodeBlockParameterPersistSeam:
+ @pytest.mark.asyncio
+ async def test_undeclared_parameter_key_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _SAFE_CODE_YAML
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
+ submitted = _directory_blocks_yaml(
+ """
+ - block_type: code
+ label: search_directory
+ parameter_keys: [address_or_postal_code]
+ code: |
+ await page.locator("#location").fill(str(address_or_postal_code))
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "search_directory" in result["error"]
+ assert "`address_or_postal_code`" in result["error"]
+ assert "workflow_definition.parameters" in result["error"]
+ assert ctx.workflow_yaml == _SAFE_CODE_YAML
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+
+ @pytest.mark.asyncio
+ async def test_nested_code_block_undeclared_parameter_key_rejects(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _directory_blocks_yaml(
+ """
+ - block_type: loop
+ label: retry_search
+ loop_blocks:
+ - block_type: code
+ label: nested_directory_search
+ parameter_keys: [address_or_postal_code]
+ code: |
+ await page.locator("#location").fill(str(address_or_postal_code))
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "nested_directory_search" in result["error"]
+ assert "`address_or_postal_code`" in result["error"]
+ assert ctx.workflow_yaml == ""
+
+ def test_deeply_nested_parameter_contract_returns_validation_error(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ block: dict[str, object] = {
+ "block_type": "code",
+ "label": "search_directory",
+ "parameter_keys": ["search_location"],
+ "code": "print(search_location)",
+ }
+ for index in range(1100):
+ block = {
+ "block_type": "loop",
+ "label": f"loop_{index}",
+ "loop_blocks": [block],
+ }
+ parsed = {
+ "workflow_definition": {
+ "parameters": [{"key": "search_location"}],
+ "blocks": [block],
+ }
+ }
+ monkeypatch.setattr(workflow_update_module, "parse_workflow_yaml", lambda _workflow_yaml: parsed)
+
+ assert (
+ workflow_update_module._code_block_parameter_contract_error("workflow_definition: {}")
+ == "Workflow YAML nesting is too deep to validate."
+ )
+
+ @pytest.mark.asyncio
+ async def test_declared_workflow_string_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _code_yaml(
+ "await page.locator('#query').fill(str(provider_query))",
+ parameter_keys=["provider_query"],
+ workflow_param=True,
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+
+ @pytest.mark.asyncio
+ async def test_new_declared_workflow_parameter_key_is_accepted_on_later_edit(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _code_yaml(
+ "await page.locator('#query').fill(str(provider_query))",
+ parameter_keys=["provider_query"],
+ workflow_param=True,
+ )
+ submitted = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_query, default_value: Sample Search}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location, default_value: Example City}
+ blocks:
+ - block_type: code
+ label: search_registry
+ parameter_keys: [provider_query, search_location]
+ code: |
+ await page.locator("#query").fill(str(provider_query))
+ await page.locator("#location").fill(str(search_location))
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+
+ @pytest.mark.asyncio
+ async def test_prior_block_output_parameter_key_is_accepted(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _directory_blocks_yaml(
+ """
+ - block_type: code
+ label: search_registry
+ code: |
+ return {"records": []}
+ - block_type: code
+ label: summarize_registry
+ parameter_keys: [search_registry_output]
+ code: |
+ print(search_registry_output)
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml == submitted
+
+ @pytest.mark.parametrize(
+ ("submitted", "label", "key"),
+ [
+ (
+ _directory_blocks_yaml(
+ """
+ - block_type: code
+ label: summarize_registry
+ parameter_keys: [search_registry_output]
+ code: |
+ print(search_registry_output)
+ - block_type: code
+ label: search_registry
+ code: |
+ return {"records": []}
+ """
+ ),
+ "summarize_registry",
+ "search_registry_output",
+ ),
+ (
+ _directory_blocks_yaml(
+ """
+ - block_type: loop
+ label: retry_search
+ loop_blocks:
+ - block_type: code
+ label: nested_directory_search
+ parameter_keys: [retry_search_output]
+ code: |
+ print(retry_search_output)
+ """
+ ),
+ "nested_directory_search",
+ "retry_search_output",
+ ),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_unavailable_output_parameter_key_rejects(
+ self, monkeypatch: pytest.MonkeyPatch, submitted: str, label: str, key: str
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert label in result["error"]
+ assert f"`{key}`" in result["error"]
+ assert ctx.workflow_yaml == ""
+
+ @pytest.mark.parametrize("marker", ["<<<<<<< HEAD", "======="])
+ @pytest.mark.asyncio
+ async def test_raw_workflow_yaml_conflict_marker_rejects_before_persist(
+ self, monkeypatch: pytest.MonkeyPatch, marker: str
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _SAFE_CODE_YAML
+ submitted = f"{marker}\n{_SAFE_CODE_YAML}"
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is False
assert f"conflict marker `{marker}`" in result["error"]
@@ -1486,6 +2409,9 @@ class TestCompiledAuthoringImposition:
assert ctx.workflow_yaml == ""
assert "authoring_repair_context" not in result["data"]
assert ctx.last_code_authoring_repair_context is None
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
async def test_output_intent_rejects_partial_metadata_without_output_label(
@@ -1526,6 +2452,113 @@ class TestCompiledAuthoringImposition:
assert "must pass `code_artifact_metadata`" in result["error"]
assert ctx.workflow_yaml == ""
+ @pytest.mark.asyncio
+ async def test_enter_directory_location_output_block_metadata_passes_before_persist(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: enter_directory_location
+ prompt: Enter the directory location and output the selected address.
+ code: |
+ await page.locator("#locInput").fill("North Carolina, USA")
+ address = "North Carolina, USA"
+ """
+ )
+ metadata = [
+ {
+ "block_label": "enter_directory_location",
+ "declared_goal": "Enter the directory location and return the address.",
+ "claimed_outcomes": [
+ {
+ "id": "claim:address",
+ "status": "observed_not_verified",
+ "goal_value_paths": ["address"],
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "id": "expectation:address",
+ "goal_value_paths": ["address"],
+ }
+ ],
+ }
+ ]
+
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
+
+ assert result["ok"] is True
+ assert ctx.workflow_yaml
+ assert "enter_directory_location" in ctx.code_artifact_metadata
+ assert ctx.latest_recorded_build_test_outcome is None
+
+ @pytest.mark.asyncio
+ async def test_invalid_metadata_records_typed_outcome_without_prose_payload(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ submitted = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ records = [{"number": "REC-001"}]
+ """
+ )
+ metadata = [
+ {
+ "block_label": "search_registry",
+ "claimed_outcomes": [{"id": "claim:rows", "status": "satisfied"}],
+ }
+ ]
+
+ result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
+
+ assert result["ok"] is False
+ assert "Artifact metadata" in result["error"]
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ structural_payload = workflow_update_module._code_artifact_metadata_reject_payload(
+ workflow_yaml=submitted,
+ raw_metadata=metadata,
+ offending_labels=["search_registry"],
+ violation_categories=["missing_declared_goal", "missing_required_list", "missing_artifact_refs"],
+ )
+ assert structural_payload is not None
+ assert structural_payload["offending_labels"] == ["search_registry"]
+ assert structural_payload["missing_fields_by_label"] == {
+ "search_registry": [
+ "declared_goal",
+ "page_dependencies",
+ "completion_criteria",
+ "terminal_verifier_expectations",
+ "evidence_refs_or_observation_refs",
+ ]
+ }
+ assert structural_payload["code_block_output_status"] == {
+ "search_registry": {
+ "block_type": "code",
+ "has_code": True,
+ "declares_output_intent": False,
+ "declares_output_roots": [],
+ "has_meaningful_output": False,
+ }
+ }
+ payload_text = str(structural_payload)
+ assert "requires non-empty" not in payload_text
+ assert "claim:rows" not in payload_text
+
@pytest.mark.asyncio
async def test_imposition_preserves_submitted_extraction_suffix(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -1954,47 +2987,144 @@ class TestCompiledAuthoringImposition:
assert result["ok"] is False
assert "submitted code mixes direct fills using `provider_name`" in result["error"]
assert ctx.workflow_yaml == ""
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["provider_name"]
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+
+ @pytest.mark.asyncio
+ async def test_typed_default_contract_reject_clears_stale_repair_context(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._typed_default_ctx()
+ submitted = _yaml(
+ """
+ title: Product lookup
+ workflow_definition:
+ parameters: invalid
+ blocks:
+ - block_type: code
+ label: search_catalog
+ code: |
+ await page.locator("#search").fill("example_sku_123")
+ """
+ )
+ ctx.workflow_yaml = submitted
+ ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "workflow_definition.parameters must be a list" in result["error"]
assert "authoring_repair_context" not in result["data"]
assert ctx.last_code_authoring_repair_context is None
@pytest.mark.asyncio
- async def test_typed_default_contract_reject_clears_stale_repair_context(
+ async def test_unknown_computed_parameter_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["provider_name"]
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert ctx.workflow_yaml == ""
+
+ @pytest.mark.asyncio
+ async def test_single_synthesized_key_without_bindable_fill_records_repair_context(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
- ctx = self._typed_default_ctx()
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#location",
+ "source_url": "https://example.com/directory",
+ "typed_length": 5,
+ "role": "textbox",
+ "accessible_name": "Address City County or Zip Code",
+ "trajectory_index": 0,
+ }
+ ]
submitted = _yaml(
"""
- title: Product lookup
+ title: Directory lookup
workflow_definition:
- parameters: invalid
blocks:
- block_type: code
- label: search_catalog
+ label: search_directory
code: |
- await page.locator("#search").fill("example_sku_123")
+ await page.locator("#submit").click()
"""
)
- ctx.workflow_yaml = submitted
- ctx.last_code_authoring_repair_context = _stale_unresolved_repair_context()
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is False
- assert "workflow_definition.parameters must be a list" in result["error"]
- assert "authoring_repair_context" not in result["data"]
- assert ctx.last_code_authoring_repair_context is None
+ assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["selector"] == "#location"
+ assert repair_context["source_url"] == "https://example.com"
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+ assert ctx.workflow_yaml == ""
@pytest.mark.asyncio
- async def test_unknown_computed_parameter_rejects_before_persist(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_synthesized_parameter_repair_context_uses_safe_selector_atom(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
- ctx = self._provider_search_ctx()
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ raw_selector = "#location-" + ("x" * 180)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": raw_selector,
+ "source_url": "https://example.com/directory?query=raw",
+ "typed_length": 5,
+ "role": "textbox",
+ "accessible_name": "Address City County or Zip Code",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_directory
+ code: |
+ await page.locator("#submit").click()
+ """
+ )
- result = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is False
- assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
- assert ctx.workflow_yaml == ""
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["selector"] in (None, "")
+ assert repair_context["source_url"] == "https://example.com"
+ assert raw_selector not in str(repair_context)
@pytest.mark.asyncio
async def test_multi_input_missing_synthesized_parameter_rejects_with_repair_context(
@@ -2169,7 +3299,7 @@ class TestCompiledAuthoringImposition:
}
@pytest.mark.asyncio
- async def test_synthesized_parameter_aliases_by_typed_length_to_declared_default(
+ async def test_synthesized_parameter_does_not_alias_by_typed_length_to_declared_default(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -2214,19 +3344,19 @@ class TestCompiledAuthoringImposition:
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["search_location", "provider_first_name"]
- assert "str(search_location)" in block["code"]
- assert "address_or_postal_code" not in block["code"]
- assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
- "address_or_postal_code": "search_location"
- }
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `address_or_postal_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_or_postal_code"]
+ assert ctx.workflow_yaml == ""
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
- async def test_synthesized_provider_search_key_rewrites_to_declared_first_last_inputs(
+ async def test_synthesized_provider_search_key_does_not_compose_declared_first_last_inputs(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -2238,6 +3368,7 @@ class TestCompiledAuthoringImposition:
"selector": "#locInput",
"source_url": "https://example.com/find-care",
"typed_length": 17,
+ "typed_value": "Example City, USA",
"role": "textbox",
"accessible_name": "Address or postal code",
"trajectory_index": 0,
@@ -2274,21 +3405,16 @@ class TestCompiledAuthoringImposition:
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["search_location", "provider_first_name", "provider_last_name"]
- assert "str(search_location)" in block["code"]
- assert '(str(provider_first_name) + " " + str(provider_last_name))' in block["code"]
- assert "address_or_postal_code" not in block["code"]
- assert "provider_name_or_identifier" not in block["code"]
- assert result["data"]["imposed_substitutions"]["parameter_aliases"] == {
- "address_or_postal_code": "search_location"
- }
- assert result["data"]["imposed_substitutions"]["parameter_expressions"] == {
- "provider_name_or_identifier": ('(str(provider_first_name) + " " + str(provider_last_name))')
- }
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `provider_name_or_identifier`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["provider_name_or_identifier"]
+ assert ctx.workflow_yaml == ""
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
def test_identifier_rewrite_skips_string_literals_and_comments(self) -> None:
source = (
@@ -2300,11 +3426,11 @@ class TestCompiledAuthoringImposition:
rewritten = workflow_update_module._replace_python_identifier(
source,
"provider_query",
- '(str(provider_first_name) + " " + str(provider_last_name))',
+ "provider_name",
)
ast.parse(rewritten)
- assert 'str((str(provider_first_name) + " " + str(provider_last_name)))' in rewritten
+ assert 'await page.locator("#providerSearch").fill(str(provider_name))' in rewritten
assert "# provider_query should stay readable in comments" in rewritten
assert '"provider_query should stay readable in strings"' in rewritten
@@ -2325,35 +3451,124 @@ class TestCompiledAuthoringImposition:
' await page.locator("#providerSearch").fill("")\n'
)
- def test_ambiguous_combined_default_match_logs_and_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None:
- debug_events: list[tuple[str, dict[str, object]]] = []
- monkeypatch.setattr(
- workflow_update_module,
- "LOG",
- SimpleNamespace(debug=lambda event, **kwargs: debug_events.append((event, kwargs))),
+ @pytest.mark.asyncio
+ async def test_single_synthesized_location_key_rejects_unique_declared_search_location_without_exact_default(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#locInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 19,
+ "role": "textbox",
+ "accessible_name": "Address City County or Zip Code",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ parameters:
+ - {key: plan, default_value: "Coastal Complete Health"}
+ - {key: provider_first_name, default_value: "Jordan"}
+ - {key: provider_last_name, default_value: "Avery"}
+ - {key: provider_npi, default_value: "1457803926"}
+ - {key: search_location, default_value: "North Carolina, USA"}
+ blocks:
+ - block_type: code
+ label: open_directory_plan_selection
+ parameter_keys:
+ - address_city_county_or_zip_code
+ code: |
+ await page.locator("#planContinue").click()
+ """
)
- result = workflow_update_module._combined_string_default_expression(
- [
- {"key": "first_name", "default_value": "Given"},
- {"key": "last_name", "default_value": "Family"},
- {"key": "given_name", "default_value": "Given"},
- {"key": "family_name", "default_value": "Family"},
- ],
- synthesized_default="Given Family",
- typed_length=None,
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["available_parameter_keys"] == [
+ "plan",
+ "provider_first_name",
+ "provider_last_name",
+ "provider_npi",
+ "search_location",
+ ]
+ assert ctx.workflow_yaml == ""
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+
+ @pytest.mark.asyncio
+ async def test_single_synthesized_location_key_rejects_unique_declared_search_location_without_default(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#locInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 19,
+ "role": "textbox",
+ "accessible_name": "Address City County or Zip Code",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: plan}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_first_name}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_last_name}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: provider_npi}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location}
+ blocks:
+ - block_type: code
+ label: open_directory_plan_selection
+ parameter_keys:
+ - address_city_county_or_zip_code
+ code: |
+ await page.locator("#planContinue").click()
+ """
)
- assert result is None
- assert debug_events == [
- (
- "copilot_synthesized_parameter_combined_default_ambiguous",
- {"match_count": 3, "synthesized_default_present": True, "typed_length": None},
- )
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["available_parameter_keys"] == [
+ "plan",
+ "provider_first_name",
+ "provider_last_name",
+ "provider_npi",
+ "search_location",
]
+ assert ctx.workflow_yaml == ""
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
- async def test_synthesized_provider_search_key_rewrites_to_short_first_last_inputs(
+ async def test_single_synthesized_location_key_rejects_ambiguous_declared_location_alias_without_defaults(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -2362,11 +3577,11 @@ class TestCompiledAuthoringImposition:
ctx.scout_trajectory = [
{
"tool_name": "type_text",
- "selector": "#providerSearch",
+ "selector": "#locInput",
"source_url": "https://example.com/find-care",
- "typed_length": 12,
+ "typed_length": 19,
"role": "textbox",
- "accessible_name": "Provider name or identifier",
+ "accessible_name": "Address City County or Zip Code",
"trajectory_index": 0,
}
]
@@ -2375,30 +3590,77 @@ class TestCompiledAuthoringImposition:
title: Directory lookup
workflow_definition:
parameters:
- - {key: first_name, default_value: "Given"}
- - {key: last_name, default_value: "Family"}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: search_location}
+ - {parameter_type: workflow, workflow_parameter_type: string, key: service_location}
blocks:
- block_type: code
- label: search_directory
+ label: open_directory_plan_selection
parameter_keys:
- - provider_name_or_identifier
+ - address_city_county_or_zip_code
code: |
- await page.locator("#providerSearch").fill(str(provider_name_or_identifier))
+ await page.locator("#planContinue").click()
"""
)
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["first_name", "last_name"]
- assert '(str(first_name) + " " + str(last_name))' in block["code"]
- assert "provider_name_or_identifier" not in block["code"]
- assert result["data"]["imposed_substitutions"]["parameter_expressions"] == {
- "provider_name_or_identifier": ('(str(first_name) + " " + str(last_name))')
- }
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["available_parameter_keys"] == ["search_location", "service_location"]
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
+
+ @pytest.mark.asyncio
+ async def test_single_synthesized_location_key_keeps_authoritative_reject_when_length_alias_ambiguous(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#locInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 19,
+ "role": "textbox",
+ "accessible_name": "Address City County or Zip Code",
+ "trajectory_index": 0,
+ }
+ ]
+ submitted = _yaml(
+ """
+ title: Directory lookup
+ workflow_definition:
+ parameters:
+ - {key: search_location, default_value: "North Carolina, USA"}
+ - {key: alternate_location, default_value: "South Carolina, USA"}
+ blocks:
+ - block_type: code
+ label: open_directory_plan_selection
+ parameter_keys:
+ - address_city_county_or_zip_code
+ code: |
+ await page.locator("#planContinue").click()
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `address_city_county_or_zip_code`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["block_label"] == "open_directory_plan_selection"
+ assert repair_context["unresolved_names"] == ["address_city_county_or_zip_code"]
+ assert repair_context["available_parameter_keys"] == ["alternate_location", "search_location"]
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
async def test_single_local_string_constant_is_lifted_for_synthesized_key(
@@ -2425,7 +3687,7 @@ class TestCompiledAuthoringImposition:
]
@pytest.mark.asyncio
- async def test_single_submitted_string_parameter_is_adopted_for_synthesized_key(
+ async def test_single_submitted_string_parameter_is_not_adopted_by_length_for_synthesized_key(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
@@ -2434,22 +3696,17 @@ class TestCompiledAuthoringImposition:
result = await _update_workflow({"workflow_yaml": _SUBMITTED_COMPUTED_PARAMETER_YAML}, ctx)
- assert result["ok"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
synthesized_key = "search_by_doctor_name_or_specialty_hospital_procedure_and_more"
- assert f'await page.locator("#provInput").fill(str({synthesized_key}))' in block["code"]
- assert block["parameter_keys"] == [synthesized_key]
- assert parsed["workflow_definition"]["parameters"] == [
- {
- "parameter_type": "workflow",
- "workflow_parameter_type": "string",
- "key": synthesized_key,
- "default_value": "Sample Search",
- }
- ]
- assert result["data"]["imposed_substitutions"]["parameter_keys"] == [synthesized_key]
+ assert result["ok"] is False
+ assert f"Unable to bind synthesized parameter `{synthesized_key}`" in result["error"]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair_context["unresolved_names"] == [synthesized_key]
+ assert ctx.workflow_yaml == ""
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.reason_code == "synthesized_parameter_binding_ambiguous"
+ assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
@pytest.mark.asyncio
async def test_mixed_literal_and_computed_fill_rejects_before_persist(
@@ -3293,12 +4550,12 @@ def _terminal_challenge_signal() -> CopilotToolBlockerSignal:
class TestCodeAuthoringGuardrailChurnBackstop:
@pytest.mark.asyncio
- async def test_counter_climbs_on_distinct_rejects_and_resets_on_accept(
+ async def test_counter_climbs_on_repeated_recorded_outcome_and_resets_on_accept(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
ctx = _code_only_ctx()
for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
- result = await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(index)}, ctx)
+ result = await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
assert result["ok"] is False
assert ctx.code_authoring_guardrail_reject_count == index + 1
assert ctx.blocker_signal is None
@@ -3340,8 +4597,8 @@ class TestCodeAuthoringGuardrailChurnBackstop:
@pytest.mark.asyncio
async def test_nth_reject_stashes_churn_signal_and_resolves_to_loop_halt(self) -> None:
ctx = _code_only_ctx()
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(index)}, ctx)
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
churn = ctx.blocker_signal
@@ -3358,8 +4615,8 @@ class TestCodeAuthoringGuardrailChurnBackstop:
ctx.blocker_signal = terminal
ctx.latest_tool_blocker_signal = terminal
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(index)}, ctx)
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
assert ctx.blocker_signal is terminal
@@ -3556,20 +4813,13 @@ class TestCodeAuthoringGuardrailChurnBackstop:
]
await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ assert ctx.code_authoring_guardrail_reject_count == 1
assert ctx.last_code_authoring_reject_was_credential_priority is False
- with pytest.raises(CopilotTurnHalt) as excinfo:
- _check_enforcement(ctx)
- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ assert ctx.blocker_signal is None
+ assert _check_enforcement(ctx) is None
@pytest.mark.asyncio
- async def test_non_credential_reject_halts_immediately_when_count_pre_charged(self) -> None:
- # Credential-priority rejects can push the shared counter into the 4..8 band where the
- # credential bound still defers; a later non-credential reject flips the flag and halts at
- # once on the non-credential N=4, since the count is already past it.
+ async def test_non_credential_reject_resets_when_recorded_outcome_changes_after_precharge(self) -> None:
ctx = _code_only_ctx()
ctx.scout_trajectory = []
@@ -3590,11 +4840,31 @@ class TestCodeAuthoringGuardrailChurnBackstop:
]
await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS + 2
+ assert ctx.code_authoring_guardrail_reject_count == 1
assert ctx.last_code_authoring_reject_was_credential_priority is False
- with pytest.raises(CopilotTurnHalt) as excinfo:
- _check_enforcement(ctx)
- assert excinfo.value.halt.kind is TurnHaltKind.LOOP_DETECTED
+ assert ctx.blocker_signal is None
+ assert _check_enforcement(ctx) is None
+
+ @pytest.mark.asyncio
+ async def test_changed_authoring_recorded_outcome_resets_guardrail_backstop(self) -> None:
+ ctx = _code_only_ctx()
+
+ await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ await _update_workflow({"workflow_yaml": _code_yaml("value = missing_second_name()")}, ctx)
+
+ assert first_key is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key != first_key
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert ctx.blocker_signal is None
+
+ @pytest.mark.asyncio
+ async def test_identical_authoring_recorded_outcome_still_halts_at_backstop(self) -> None:
+ ctx = _code_only_ctx()
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _code_yaml("value = missing_first_name()")}, ctx)
+
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
churn = ctx.blocker_signal
assert isinstance(churn, CopilotToolBlockerSignal)
assert churn.internal_reason_code == "code_authoring_guardrail_churn"
@@ -3602,8 +4872,8 @@ class TestCodeAuthoringGuardrailChurnBackstop:
@pytest.mark.asyncio
async def test_accept_after_latched_churn_clears_both_signals(self, monkeypatch: pytest.MonkeyPatch) -> None:
ctx = _code_only_ctx()
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
- await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(index)}, ctx)
+ for _ in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ await _update_workflow({"workflow_yaml": _distinct_guardrail_yaml(0)}, ctx)
churn = ctx.blocker_signal
assert isinstance(churn, CopilotToolBlockerSignal)
assert churn.internal_reason_code == "code_authoring_guardrail_churn"
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 52dc84508..440a920f0 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -1414,7 +1414,7 @@ class TestPreflightSurfacesSyntaxError:
diagnostics = preflight_code_block(code, parameter_keys=())
assert sum(1 for d in diagnostics if d.code == "BROAD_DOCUMENT_BODY_TEXT_WAIT") == 1
- assert any("localized result/detail" in d.message for d in diagnostics)
+ assert any("localized container" in d.message for d in diagnostics)
def test_broad_body_text_wait_for_function_keyword_expression_surfaces_selection_diagnostic(self) -> None:
code = (
@@ -1436,6 +1436,182 @@ class TestPreflightSurfacesSyntaxError:
assert not any(d.code == "BROAD_DOCUMENT_BODY_TEXT_WAIT" for d in diagnostics)
+ def test_global_get_by_text_wait_for_surfaces_ambiguous_locator_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'await page.get_by_text("NATIONAL PROVIDER IDENTIFIER").wait_for(timeout=5000)\n',
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ def test_global_get_by_text_wait_for_alias_surfaces_ambiguous_locator_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\nawait target.wait_for(timeout=5000)\n',
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ "if ready:\n"
+ ' target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ " await target.wait_for(timeout=5000)\n",
+ "for _ in [1]:\n"
+ ' target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ " await target.wait_for(timeout=5000)\n",
+ "try:\n"
+ ' target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ " await target.wait_for(timeout=5000)\n"
+ "except Exception:\n"
+ " pass\n",
+ ],
+ )
+ def test_nested_get_by_text_alias_wait_surfaces_ambiguous_locator_diagnostic(self, code: str) -> None:
+ diagnostics = preflight_code_block(code, parameter_keys=("ready",))
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ def test_rebound_get_by_text_alias_does_not_surface_stale_ambiguous_locator_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ 'target = page.locator("#safe")\n'
+ "await target.wait_for(timeout=5000)\n",
+ parameter_keys=(),
+ )
+
+ assert not any(d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR" for d in diagnostics)
+
+ def test_get_by_text_alias_wait_before_rebind_surfaces_ambiguous_locator_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ "await target.wait_for(timeout=5000)\n"
+ 'target = page.locator("#safe")\n',
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ def test_nested_earlier_get_by_text_rebind_does_not_clear_later_global_alias(self) -> None:
+ diagnostics = preflight_code_block(
+ "async def helper():\n"
+ ' target = page.locator("#safe")\n'
+ 'target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ "await target.wait_for(timeout=5000)\n",
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ def test_nested_later_get_by_text_rebind_does_not_clear_outer_global_alias(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.get_by_text("NATIONAL PROVIDER IDENTIFIER")\n'
+ "async def helper():\n"
+ ' target = page.locator("#safe")\n'
+ "await target.wait_for(timeout=5000)\n",
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR") == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'await page.locator("table").wait_for(state="visible", timeout=15000)\n',
+ 'await page.locator("table").first.wait_for(state="visible", timeout=15000)\n',
+ 'table = page.locator("table")\nawait table.first.wait_for(state="visible", timeout=15000)\n',
+ ],
+ )
+ def test_broad_global_table_wait_for_surfaces_selection_diagnostic(self, code: str) -> None:
+ diagnostics = preflight_code_block(code, parameter_keys=())
+
+ assert sum(1 for d in diagnostics if d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR") == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'if ready:\n target = page.locator("table")\n await target.wait_for(timeout=5000)\n',
+ 'for _ in [1]:\n target = page.locator("table")\n await target.wait_for(timeout=5000)\n',
+ "try:\n"
+ ' target = page.locator("table")\n'
+ " await target.wait_for(timeout=5000)\n"
+ "except Exception:\n"
+ " pass\n",
+ ],
+ )
+ def test_nested_table_alias_wait_surfaces_broad_table_diagnostic(self, code: str) -> None:
+ diagnostics = preflight_code_block(code, parameter_keys=("ready",))
+
+ assert sum(1 for d in diagnostics if d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR") == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'await page.locator("table tbody tr").first.wait_for(state="visible", timeout=15000)\n',
+ 'await page.locator("#coastalCard").locator("table").first.wait_for(state="visible", timeout=15000)\n',
+ 'await page.locator("table").get_by_role("row").first.wait_for(state="visible", timeout=15000)\n',
+ 'await page.locator("table").filter(has_text="Credentialed").wait_for(timeout=15000)\n',
+ 'table = page.locator("table").filter(has_text="Credentialed")\nawait table.wait_for(timeout=15000)\n',
+ ],
+ )
+ def test_scoped_or_narrowed_table_wait_for_is_allowed(self, code: str) -> None:
+ diagnostics = preflight_code_block(code, parameter_keys=())
+
+ assert not any(d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR" for d in diagnostics)
+
+ def test_rebound_table_alias_does_not_surface_stale_broad_table_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.locator("table")\ntarget = page.locator("#safe")\nawait target.wait_for(timeout=5000)\n',
+ parameter_keys=(),
+ )
+
+ assert not any(d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR" for d in diagnostics)
+
+ def test_table_alias_wait_before_rebind_surfaces_broad_table_diagnostic(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.locator("table")\nawait target.wait_for(timeout=5000)\ntarget = page.locator("#safe")\n',
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR") == 1
+
+ def test_nested_earlier_table_rebind_does_not_clear_later_global_alias(self) -> None:
+ diagnostics = preflight_code_block(
+ "async def helper():\n"
+ ' target = page.locator("#safe")\n'
+ 'target = page.locator("table")\n'
+ "await target.wait_for(timeout=5000)\n",
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR") == 1
+
+ def test_nested_later_table_rebind_does_not_clear_outer_global_alias(self) -> None:
+ diagnostics = preflight_code_block(
+ 'target = page.locator("table")\n'
+ "async def helper():\n"
+ ' target = page.locator("#safe")\n'
+ "await target.wait_for(timeout=5000)\n",
+ parameter_keys=(),
+ )
+
+ assert sum(1 for d in diagnostics if d.code == "BROAD_GLOBAL_TABLE_WAIT_FOR") == 1
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'await page.locator("main").get_by_text("NATIONAL PROVIDER IDENTIFIER").wait_for(timeout=5000)\n',
+ 'await page.get_by_text("NATIONAL PROVIDER IDENTIFIER").first.wait_for(timeout=5000)\n',
+ 'await page.get_by_text("NATIONAL PROVIDER IDENTIFIER").nth(0).wait_for(timeout=5000)\n',
+ 'await page.get_by_text("NATIONAL PROVIDER IDENTIFIER").filter(visible=True).wait_for(timeout=5000)\n',
+ ],
+ )
+ def test_scoped_or_narrowed_get_by_text_wait_for_is_allowed(self, code: str) -> None:
+ diagnostics = preflight_code_block(code, parameter_keys=())
+
+ assert not any(d.code == "GLOBAL_GET_BY_TEXT_WAIT_FOR" for d in diagnostics)
+
def test_non_page_wait_for_function_does_not_surface_body_text_diagnostic(self) -> None:
code = """
diagnostics = []
@@ -1462,7 +1638,7 @@ class TestPreflightSurfacesSyntaxError:
errors = _code_block_safety_errors(workflow_yaml, None)
assert any("failed the generated-code preflight check" in str(error) for error in errors)
- assert any("localized result/detail" in str(error) for error in errors)
+ assert any("localized container" in str(error) for error in errors)
def test_broad_container_record_scan_surfaces_row_extraction_diagnostic(self) -> None:
code = """