Skip to content

Automate browser based workflows with AI

SKY-11591: author-time output contract for Copilot build-time repair (#7069)

Andrew Neilsonac0bfa20b65d
diff --git a/skyvern-frontend/src/routes/workflows/studio/runview/RunPaneHeader.tsx b/skyvern-frontend/src/routes/workflows/studio/runview/RunPaneHeader.tsx
index ccdf8c045..b831a8a25 100644
--- a/skyvern-frontend/src/routes/workflows/studio/runview/RunPaneHeader.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/runview/RunPaneHeader.tsx
@@ -207,7 +207,8 @@ export function RunPaneActions() {
type="button"
aria-label="API & Webhooks"
className={cn(
- "inline-flex h-7 items-center gap-1.5 rounded-md border border-border px-2 text-[11px] font-medium",
+ "inline-flex h-7 shrink-0 items-center gap-1.5 whitespace-nowrap rounded-md border border-border text-[11px] font-medium",
+ compact ? "w-7 justify-center px-0" : "px-2",
"text-muted-foreground hover:bg-accent hover:text-accent-foreground",
"focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring",
)}
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b9bf0380e..014382308 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -679,6 +679,40 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
lines.append(
f"page_challenges: {_render_authoring_repair_prompt_list(repair_context.page_challenge_summaries)}"
)
+ if repair_context.reason_code == "metadata_reject":
+ if repair_context.runtime_failure_reason:
+ lines.append(
+ f"runtime_failure_reason: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_reason)}"
+ )
+ if repair_context.runtime_failure_class:
+ lines.append(
+ f"runtime_failure_class: {_clean_authoring_repair_prompt_atom(repair_context.runtime_failure_class)}"
+ )
+ if repair_context.metadata_contract_source:
+ lines.append(
+ "metadata_contract_source: "
+ f"{_clean_authoring_repair_prompt_atom(repair_context.metadata_contract_source)}"
+ )
+ if repair_context.metadata_contract_reason_code:
+ lines.append(
+ "metadata_contract_reason_code: "
+ f"{_clean_authoring_repair_prompt_atom(repair_context.metadata_contract_reason_code)}"
+ )
+ if repair_context.required_goal_value_paths:
+ lines.append(
+ "required_goal_value_paths: "
+ f"{_render_authoring_repair_prompt_list(repair_context.required_goal_value_paths)}"
+ )
+ if repair_context.required_extraction_schema_paths:
+ lines.append(
+ "required_extraction_schema_paths: "
+ f"{_render_authoring_repair_prompt_list(repair_context.required_extraction_schema_paths)}"
+ )
+ if repair_context.required_code_return_paths:
+ lines.append(
+ "required_code_return_paths: "
+ f"{_render_authoring_repair_prompt_list(repair_context.required_code_return_paths)}"
+ )
selector_alternative_lines = _render_selector_repair_alternatives(repair_context.selector_alternatives)
if selector_alternative_lines:
lines.append("same_page_selector_alternatives:")
@@ -730,6 +764,11 @@ def _code_authoring_repair_context_prompt(ctx: CopilotContext | None) -> str:
"For missing prior block outputs, bind to an actual available_output_key or repair the producing/current "
"code block so the output exists; do not create a workflow parameter for missing_output_key."
)
+ if repair_context.reason_code == "metadata_reject":
+ lines.append(
+ "For metadata rejects, author code_artifact_metadata with goal_value_paths, valid extraction_schema, "
+ "and code return paths matching required requested output child paths; rerun update_and_run_blocks."
+ )
lines.append(_clean_authoring_repair_prompt_atom(repair_context.repair_instruction, max_chars=260))
return "\n\n" + "\n".join(line for line in lines if line)
@@ -1403,6 +1442,15 @@ def _make_agent_result(
result.apply_without_review = _should_apply_code_only_success_without_review(ctx, result.proposal_disposition)
if ctx is not None and result.completion_criteria_turn_state is None:
result.completion_criteria_turn_state = getattr(ctx, "completion_criteria_turn_state", None)
+ if ctx is not None and result.code_artifact_metadata is None:
+ evidence_metadata = getattr(
+ getattr(ctx, "workflow_verification_evidence", None), "code_artifact_metadata", None
+ )
+ ctx_metadata = getattr(ctx, "code_artifact_metadata", None)
+ if isinstance(evidence_metadata, dict) and evidence_metadata:
+ result.code_artifact_metadata = evidence_metadata
+ elif isinstance(ctx_metadata, dict) and ctx_metadata:
+ result.code_artifact_metadata = ctx_metadata
return result
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 47e085d08..f0c037977 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -83,13 +83,19 @@ class RecordedBuildTestOutcome(BaseModel):
page_evidence_refs: list[str] = Field(default_factory=list)
evidence_refs: list[str] = Field(default_factory=list)
missing_requested_output_facts: list[dict[str, object]] = Field(default_factory=list)
+ runtime_output_repair_facts: list[dict[str, object]] = Field(default_factory=list)
authored_structure_signature: str | None = None
display_text: str = ""
key_provenance: dict[str, str] = Field(default_factory=dict)
@property
def structural_key_payload(self) -> dict[str, object] | None:
- if not (self.structural_failure_identity or self.verified_progress_marker or self.page_evidence_refs):
+ if not (
+ self.structural_failure_identity
+ or self.verified_progress_marker
+ or self.page_evidence_refs
+ or self.runtime_output_repair_facts
+ ):
return None
return {
"version": _STRUCTURAL_KEY_VERSION,
@@ -103,6 +109,7 @@ class RecordedBuildTestOutcome(BaseModel):
"page_evidence_refs": sorted(self.page_evidence_refs),
"evidence_refs": sorted(self.evidence_refs),
"missing_requested_output_facts": self.missing_requested_output_facts,
+ "runtime_output_repair_facts": self.runtime_output_repair_facts,
}
@property
@@ -635,6 +642,7 @@ def recorded_outcome_from_run_blocks_result(
recorded_run_outcome: RecordedRunOutcome | None = None,
completion_verification: CompletionVerificationResult | None = None,
authored_structure_signature: str | None = None,
+ registered_output_parameter_payloads: Sequence[Mapping[str, object]] | None = None,
) -> RecordedBuildTestOutcome | None:
data = _dict(result.get("data"))
workflow_run_id = _safe_str(data.get("workflow_run_id"))
@@ -645,6 +653,15 @@ def recorded_outcome_from_run_blocks_result(
output_refs = _output_evidence_refs(blocks)
verification_identity = _completion_verification_identity(completion_verification)
missing_output_facts = _missing_requested_output_facts(completion_verification, blocks)
+ authoritative_workflow_run_id = (
+ recorded_run_outcome.workflow_run_id if recorded_run_outcome is not None else None
+ ) or workflow_run_id
+ runtime_output_facts = _runtime_output_repair_facts(
+ completion_verification,
+ blocks,
+ registered_output_parameter_payloads or _mapping_list(data.get("registered_output_parameter_values")),
+ authoritative_workflow_run_id,
+ )
if recorded_run_outcome is not None:
reason_code = _run_outcome_reason_code(recorded_run_outcome)
if reason_code == "terminal_challenge_blocker":
@@ -713,6 +730,7 @@ def recorded_outcome_from_run_blocks_result(
page_evidence_refs=page_refs,
evidence_refs=evidence_refs,
missing_requested_output_facts=missing_output_facts,
+ runtime_output_repair_facts=runtime_output_facts,
authored_structure_signature=authored_structure_signature,
observed_evidence_summary=recorded_run_outcome.display_reason or "",
key_provenance={
@@ -720,6 +738,7 @@ def recorded_outcome_from_run_blocks_result(
"page_evidence_refs": "bounded post-run page evidence",
"evidence_refs": "run output structure",
"missing_requested_output_facts": "CompletionVerificationResult unsatisfied output paths and run output shape",
+ "runtime_output_repair_facts": "same-run registered output parameters and completion verdicts",
},
)
run_status = _safe_str(data.get("overall_status"))
@@ -1170,6 +1189,51 @@ def _missing_requested_output_facts(
return sorted(facts, key=lambda item: str(item.get("output_path") or ""))
+def _runtime_output_repair_facts(
+ completion_verification: CompletionVerificationResult | None,
+ blocks: Sequence[Mapping[str, object]],
+ registered_output_parameter_payloads: Sequence[Mapping[str, object]],
+ workflow_run_id: str,
+) -> list[dict[str, object]]:
+ if completion_verification is None or completion_verification.status != "evaluated" or not workflow_run_id:
+ return []
+ facts: list[dict[str, object]] = []
+ for verdict in completion_verification.verdicts:
+ if verdict.satisfied or not verdict.output_path:
+ continue
+ output_path = _bounded_ref(verdict.output_path)
+ if not _output_path_has_child(output_path):
+ continue
+ output_root = _output_path_root(output_path)
+ if not output_root:
+ continue
+ values, evidence_refs, block_labels = _runtime_output_values_for_path(
+ blocks,
+ registered_output_parameter_payloads,
+ workflow_run_id,
+ output_path,
+ )
+ value_status = _runtime_output_value_status(values, verdict)
+ fact: dict[str, object] = {
+ "workflow_run_id": _bounded_ref(workflow_run_id),
+ "output_path": output_path,
+ "output_root": output_root,
+ "criterion_id": _bounded_ref(verdict.criterion_id),
+ "reason_code": _bounded_ref(verdict.reason_code),
+ "value_status": value_status,
+ }
+ if len(block_labels) == 1:
+ fact["block_label"] = block_labels[0]
+ if verdict.grounding_mode:
+ fact["grounding_mode"] = verdict.grounding_mode
+ if verdict.expected_output_shape:
+ fact["expected_output_shape"] = _bounded_ref(verdict.expected_output_shape)
+ if evidence_refs:
+ fact["evidence_refs"] = evidence_refs
+ facts.append(fact)
+ return sorted(facts, key=lambda item: str(item.get("output_path") or ""))
+
+
def _has_presence_only_output_evidence(verdict: CriterionVerdict) -> bool:
return (
verdict.reason_code == "structurally_abstained"
@@ -1183,6 +1247,10 @@ def _output_path_root(output_path: str) -> str:
return _bounded_ref(output_path.split(".", 1)[0].split("[", 1)[0])
+def _output_path_has_child(output_path: str) -> bool:
+ return "." in output_path or "[]" in output_path
+
+
def _output_path_value_status(
blocks: Sequence[Mapping[str, object]],
output_path: str,
@@ -1207,6 +1275,73 @@ def _output_path_value_status(
return "typed_value_unverified"
+def _runtime_output_values_for_path(
+ blocks: Sequence[Mapping[str, object]],
+ registered_output_parameter_payloads: Sequence[Mapping[str, object]],
+ workflow_run_id: str,
+ output_path: str,
+) -> tuple[list[object], list[str], list[str]]:
+ values: list[object] = []
+ evidence_refs: list[str] = []
+ block_labels: list[str] = []
+ for item in registered_output_parameter_payloads:
+ item_run_id = _safe_str(item.get("workflow_run_id"))
+ if item_run_id != workflow_run_id:
+ continue
+ value, present = _registered_output_value_for_path(item, output_path)
+ if not present:
+ continue
+ values.append(value)
+ label = _bounded_ref(item.get("block_label"))
+ key = _bounded_ref(item.get("output_parameter_key"))
+ if label:
+ block_labels.append(label)
+ if label or key:
+ evidence_refs.append(f"registered_output:{label or 'unknown'}:{key or output_path}")
+ for block in blocks:
+ extracted = block.get("extracted_data")
+ if extracted is None:
+ continue
+ value, present = _value_at_output_path(extracted, output_path)
+ if not present:
+ continue
+ values.append(value)
+ label = _bounded_ref(block.get("label"))
+ if label:
+ block_labels.append(label)
+ evidence_refs.append(f"output:{label}")
+ return values, list(dict.fromkeys(evidence_refs)), sorted(dict.fromkeys(block_labels))
+
+
+def _registered_output_value_for_path(item: Mapping[str, object], output_path: str) -> tuple[object | None, bool]:
+ value = item.get("value")
+ key = _safe_str(item.get("output_parameter_key"))
+ if key == output_path:
+ return value, True
+ if output_path.startswith("output.") and key == output_path.split(".", 1)[1]:
+ return value, True
+ if isinstance(value, Mapping):
+ return _value_at_output_path(value, output_path)
+ return None, False
+
+
+def _runtime_output_value_status(values: Sequence[object], verdict: CriterionVerdict) -> str:
+ if values:
+ if any(value is None for value in values):
+ return "null"
+ expected_shape = (verdict.expected_output_shape or "").casefold()
+ if expected_shape in {"string", "str"} and any(not isinstance(value, str) for value in values):
+ return "type_mismatch"
+ if expected_shape in {"array", "list"} and any(not isinstance(value, list) for value in values):
+ return "shape_mismatch"
+ if expected_shape in {"object", "dict"} and any(not isinstance(value, Mapping) for value in values):
+ return "shape_mismatch"
+ return "no_typed_value" if not verdict.has_exact_value else "type_mismatch"
+ if verdict.reason_code == "structurally_abstained":
+ return "structural_abstained"
+ return "no_typed_value"
+
+
def _value_at_output_path(value: object, output_path: str) -> tuple[object | None, bool]:
current = value
for segment in [part for part in re.split(r"\.|\[\]", output_path) if part]:
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index c38e6e5b5..bb2b4809b 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -171,6 +171,11 @@ class CodeAuthoringRepairContext(BaseModel):
missing_output_key: str | None = None
available_output_keys: list[str] = Field(default_factory=list)
current_block_parameter_keys: list[str] = Field(default_factory=list)
+ required_goal_value_paths: list[str] = Field(default_factory=list)
+ required_extraction_schema_paths: list[str] = Field(default_factory=list)
+ required_code_return_paths: list[str] = Field(default_factory=list)
+ metadata_contract_source: str = ""
+ metadata_contract_reason_code: str = ""
failed_block_status: str | None = None
workflow_run_id: str | None = None
current_origin: str | None = None
@@ -462,6 +467,7 @@ class AgentResult:
staged_workflow_yaml: str | None = None
staged_workflow: Workflow | None = None
has_staged_proposal: bool = False
+ code_artifact_metadata: dict[str, dict[str, Any]] | None = None
# Set when ``_update_workflow`` wrote canonical mid-turn (param / top-level
# settings changes); terminal handlers roll back on non-auto-accept.
canonical_was_persisted_due_to_param_change: bool = False
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 56442ad01..dd412a09f 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -2229,8 +2229,8 @@ def _ambiguous_bare_selector_rescout_signal_state(ctx: Any, tool_name: str) -> s
return "allow"
-def _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:
- if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS:
+def _should_block_tool_after_unresolved_recorded_outcome(ctx: Any, tool_name: str) -> bool:
+ if tool_name not in _SYNTHESIZED_BLOCK_PERSISTENCE_MUTATING_TOOLS and tool_name != "update_workflow":
return False
if not _turn_intent_can_update_and_run_without_user_input(getattr(ctx, "turn_intent", None)):
return False
@@ -2247,18 +2247,13 @@ def _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx: Any, tool
def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToolBlockerSignal | None:
- if tool_name in _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS:
- return None
- ambiguous_selector_rescout_state = _ambiguous_bare_selector_rescout_signal_state(ctx, tool_name)
- if ambiguous_selector_rescout_state == "allow":
- return None
- if _should_block_mutating_tool_after_unresolved_recorded_outcome(ctx, tool_name):
+ if _should_block_tool_after_unresolved_recorded_outcome(ctx, tool_name):
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
agent_steering_text=(
"The last recorded test outcome is authoritative and still has unsatisfied completion criteria. "
f"Call {SYNTHESIZED_BLOCK_PERSISTENCE_TOOL} with a materially changed authored workflow now; "
- "do not spend the repair attempt on browser interaction."
+ "do not spend the repair attempt on another standalone tool call."
),
user_facing_reason="I need to revise and test the workflow code instead of interacting with the page.",
recovery_hint="retry_with_different_tool",
@@ -2269,6 +2264,11 @@ def synthesized_block_persistence_signal(ctx: Any, tool_name: str) -> CopilotToo
blocked_tool=tool_name,
extra={"recorded_outcome_reason_code": "outcome_not_demonstrated"},
)
+ if tool_name in _SYNTHESIZED_BLOCK_PERSISTENCE_ALLOWED_TOOLS:
+ return None
+ ambiguous_selector_rescout_state = _ambiguous_bare_selector_rescout_signal_state(ctx, tool_name)
+ if ambiguous_selector_rescout_state == "allow":
+ return None
if (
ambiguous_selector_rescout_state != "block"
and not _should_force_synthesized_block_persistence(ctx)
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 8b9234ab1..9ccb97aef 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -298,6 +298,10 @@ class AgentContext:
# Latest typed reached-download target from the scout steer; the synthesizer compiles the terminal
# expect_download step from it. Selector is the observed download link, not necessarily a trajectory click.
reached_download_target: ReachedDownloadTarget | None = None
+ # Author-time output-contract cross-turn state, keyed by the contract signature; set lazily by workflow_update.
+ output_contract_pinned_block_label_by_signature: dict[str, str] = field(default_factory=dict)
+ output_contract_reject_count_by_signature: dict[str, int] = field(default_factory=dict)
+ runtime_output_repair_attempt_by_signature: dict[str, bool] = field(default_factory=dict)
synthesized_block_offered: bool = False
synthesized_block_offered_trajectory_len: int = 0
synthesized_block_offered_goal_complete: bool = False
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 0aaeb7f2d..a048fb9e1 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -243,9 +243,14 @@ from .workflow_update import BlockObservationRef as BlockObservationRef
from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadata
from .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argument
from .workflow_update import _code_block_safety_errors as _code_block_safety_errors
+from .workflow_update import (
+ _impose_output_contract_envelope_after_steering as _impose_output_contract_envelope_after_steering,
+)
+from .workflow_update import _metadata_contract_run_preflight_reject as _metadata_contract_run_preflight_reject
from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
from .workflow_update import _record_workflow_update_result as _record_workflow_update_result
+from .workflow_update import _scaffold_metadata_contract_for_update as _scaffold_metadata_contract_for_update
from .workflow_update import _update_workflow as _update_workflow
LOG = structlog.get_logger()
@@ -283,7 +288,7 @@ async def update_workflow_tool(
refs, observation refs, and terminal verifier expectations.
"""
copilot_ctx = ctx.context
- serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
+ serialized_code_artifact_metadata: object = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
arguments = {
"workflow_yaml": workflow_yaml,
@@ -596,7 +601,7 @@ async def update_and_run_blocks_tool(
copilot_ctx = ctx.context
copilot_ctx.completion_verification_result = None
handler_start = time.monotonic()
- serialized_code_artifact_metadata = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
+ serialized_code_artifact_metadata: object = _code_artifact_metadata_as_tool_argument(code_artifact_metadata)
normalized_block_observation_refs = normalize_block_observation_refs(block_observation_refs)
arguments = {
"workflow_yaml": workflow_yaml,
@@ -614,6 +619,45 @@ async def update_and_run_blocks_tool(
if authority_error:
return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", authority_error)
+ workflow_yaml, imposed_code_artifact_metadata, envelope_imposed = _impose_output_contract_envelope_after_steering(
+ copilot_ctx,
+ workflow_yaml,
+ serialized_code_artifact_metadata,
+ )
+ if envelope_imposed:
+ serialized_code_artifact_metadata = imposed_code_artifact_metadata
+ arguments["workflow_yaml"] = workflow_yaml
+ arguments["code_artifact_metadata"] = serialized_code_artifact_metadata
+
+ scaffolded_code_artifact_metadata, scaffold_applied = _scaffold_metadata_contract_for_update(
+ copilot_ctx,
+ workflow_yaml,
+ serialized_code_artifact_metadata,
+ )
+ if scaffold_applied:
+ serialized_code_artifact_metadata = scaffolded_code_artifact_metadata
+ arguments["code_artifact_metadata"] = serialized_code_artifact_metadata
+
+ metadata_contract_preflight_reject = _metadata_contract_run_preflight_reject(
+ copilot_ctx,
+ workflow_yaml,
+ serialized_code_artifact_metadata,
+ )
+ if metadata_contract_preflight_reject is not None:
+ record_tool_step_result_for_ctx(
+ copilot_ctx,
+ "update_and_run_blocks",
+ arguments,
+ metadata_contract_preflight_reject,
+ )
+ _record_diagnosis_repair_contract(
+ copilot_ctx,
+ source_tool="update_and_run_blocks",
+ result=metadata_contract_preflight_reject,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", metadata_contract_preflight_reject)
+ return json.dumps(sanitized)
+
loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
if loop_error:
return _diagnosis_repair_tool_error(copilot_ctx, "update_and_run_blocks", loop_error)
@@ -650,10 +694,13 @@ async def update_and_run_blocks_tool(
"block_observation_refs": normalized_block_observation_refs,
"raw_block_observation_refs": block_observation_refs,
"code_artifact_metadata": serialized_code_artifact_metadata,
- "raw_code_artifact_metadata": code_artifact_metadata,
+ "raw_code_artifact_metadata": serialized_code_artifact_metadata
+ if scaffold_applied or envelope_imposed
+ else code_artifact_metadata,
},
copilot_ctx,
allow_missing_credentials=skip_run_after_update,
+ allow_static_output_uncertainty=True,
)
_record_workflow_update_result(copilot_ctx, update_result, prior_definition)
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 96000bf25..266a89c96 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -1,4 +1,5 @@
import time
+from collections.abc import Mapping
from typing import Any
import structlog
@@ -47,7 +48,7 @@ from skyvern.forge.sdk.copilot.reached_download_target import (
ReachedDownloadTarget,
derive_from_block_outputs,
)
-from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion
+from skyvern.forge.sdk.copilot.request_policy import CompletionCriterion, is_fallback_floor_criterion
from skyvern.forge.sdk.copilot.terminal_predicates import outcome_fully_verified
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
@@ -73,6 +74,9 @@ LOG = structlog.get_logger()
_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(f"output.{key}" for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
+_AUTHORED_OUTPUT_CONTRACT_CRITERION_ID_PREFIX = "__copilot_authored_output__"
+_AUTHORED_OUTPUT_CONTRACT_MISSING_CRITERION_ID = "__copilot_authored_output_contract_missing"
+_AUTHORED_OUTPUT_CONTRACT_MISSING_PATH = "output.__copilot_missing_authored_output_contract__"
_VALIDATION_REVIEW_OUTPUT_CONTRACT_HINT = (
" For validation-only pre-submit Review pages, do not repair by returning only booleans such as "
"pre_submit_review_reached, submit_control_visible, submit_or_finalize_clicked, or per-field *_verified flags. "
@@ -173,7 +177,112 @@ def _completion_verification_criteria(copilot_ctx: Any) -> list[CompletionCriter
policy = _completion_request_policy(copilot_ctx)
# A method-mandated criterion asserts HOW the goal was reached; the outcome
# judge sees only end-state evidence.
- return policy.graded_completion_criteria() if policy is not None else []
+ criteria = policy.graded_completion_criteria() if policy is not None else []
+ authored_output_criteria = _authored_output_contract_criteria(copilot_ctx)
+ if authored_output_criteria and (not criteria or all(is_fallback_floor_criterion(c) for c in criteria)):
+ return authored_output_criteria
+ if _accepted_staged_output_contract_missing(copilot_ctx) and (
+ not criteria or all(is_fallback_floor_criterion(c) for c in criteria)
+ ):
+ return [_authored_output_contract_missing_criterion()]
+ return criteria
+
+
+def _authored_output_contract_criteria(copilot_ctx: Any) -> list[CompletionCriterion]:
+ paths = _authored_output_contract_paths(copilot_ctx)
+ return [
+ CompletionCriterion(
+ id=f"{_AUTHORED_OUTPUT_CONTRACT_CRITERION_ID_PREFIX}{path.replace('[]', '_items').replace('.', '_')}",
+ outcome=f"The run output includes the authored output contract path {path}.",
+ implicit=True,
+ level="run",
+ output_path=path,
+ )
+ for path in paths
+ ]
+
+
+def _authored_output_contract_paths(copilot_ctx: Any) -> list[str]:
+ if _accepted_staged_proposal_present(copilot_ctx):
+ return sorted(_authored_output_contract_metadata_paths(_accepted_staged_output_contract_metadata(copilot_ctx)))
+ repair_context_paths = _authored_output_contract_repair_context_paths(copilot_ctx)
+ if repair_context_paths:
+ return sorted(repair_context_paths)
+ return sorted(_authored_output_contract_metadata_paths(_accepted_staged_output_contract_metadata(copilot_ctx)))
+
+
+def _accepted_staged_output_contract_metadata(copilot_ctx: Any) -> object:
+ evidence = getattr(copilot_ctx, "workflow_verification_evidence", None)
+ metadata = getattr(evidence, "code_artifact_metadata", None)
+ if _authored_output_contract_metadata_paths(metadata):
+ return metadata
+ return getattr(copilot_ctx, "code_artifact_metadata", None)
+
+
+def _authored_output_contract_metadata_paths(metadata: object) -> set[str]:
+ paths: set[str] = set()
+ if isinstance(metadata, Mapping):
+ for artifact in metadata.values():
+ if not isinstance(artifact, Mapping):
+ continue
+ for row_group_key in ("claimed_outcomes", "terminal_verifier_expectations"):
+ row_group = artifact.get(row_group_key)
+ if not isinstance(row_group, list):
+ continue
+ for row in row_group:
+ if not isinstance(row, Mapping):
+ continue
+ paths.update(_authored_output_contract_paths_from_list(row.get("goal_value_paths")))
+ return paths
+
+
+def _accepted_staged_output_contract_missing(copilot_ctx: Any) -> bool:
+ return (
+ _accepted_staged_proposal_present(copilot_ctx)
+ and not _authored_output_contract_paths(copilot_ctx)
+ and bool(_authored_output_contract_repair_context_paths(copilot_ctx))
+ )
+
+
+def _accepted_staged_proposal_present(copilot_ctx: Any) -> bool:
+ return bool(getattr(copilot_ctx, "has_staged_proposal", False) or getattr(copilot_ctx, "staged_workflow", None))
+
+
+def _authored_output_contract_missing_criterion() -> CompletionCriterion:
+ return CompletionCriterion(
+ id=_AUTHORED_OUTPUT_CONTRACT_MISSING_CRITERION_ID,
+ outcome="The accepted staged workflow exposes an authored output contract.",
+ implicit=True,
+ level="run",
+ output_path=_AUTHORED_OUTPUT_CONTRACT_MISSING_PATH,
+ )
+
+
+def _authored_output_contract_repair_context_paths(copilot_ctx: Any) -> set[str]:
+ repair_context = getattr(copilot_ctx, "last_code_authoring_repair_context", None)
+ paths: set[str] = set()
+ for attr in ("required_goal_value_paths", "required_extraction_schema_paths", "required_code_return_paths"):
+ paths.update(_authored_output_contract_paths_from_list(getattr(repair_context, attr, None)))
+ return paths
+
+
+def _authored_output_contract_paths_from_list(value: object) -> set[str]:
+ if not isinstance(value, list):
+ return set()
+ return {path for raw_path in value for path in [_authored_output_contract_path(raw_path)] if path}
+
+
+def _authored_output_contract_path(value: object) -> str:
+ if not isinstance(value, str):
+ return ""
+ path = value.strip()
+ if path == "output." or path == "output":
+ return ""
+ if path.startswith("output."):
+ return path
+ if path.startswith("_") or not path.replace("_", "").isalnum():
+ return ""
+ return f"output.{path}"
def _split_criteria_by_plane(criteria: list[Any]) -> tuple[list[CompletionCriterion], list[CompletionCriterion]]:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 7d6f6389d..24b2fabaf 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -2585,6 +2585,12 @@ def _record_adjudicated_build_test_outcome(
completion_verification: CompletionVerificationResult | None,
recorded_run_outcome: RecordedRunOutcome | None,
) -> None:
+ result_data = result.get("data")
+ registered_output_parameter_payloads = (
+ result_data.get("registered_output_parameter_values")
+ if isinstance(result_data, dict) and isinstance(result_data.get("registered_output_parameter_values"), list)
+ else None
+ )
record_build_test_outcome(
copilot_ctx,
recorded_outcome_from_run_blocks_result(
@@ -2596,6 +2602,7 @@ def _record_adjudicated_build_test_outcome(
getattr(copilot_ctx, "workflow_yaml", None),
getattr(copilot_ctx, "code_artifact_metadata", None),
),
+ registered_output_parameter_payloads=registered_output_parameter_payloads,
),
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 6d085e6e4..ee49a2aff 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import ast
import copy
+import hashlib
import io
import json
import keyword
@@ -50,6 +51,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_BARE_TAG_RE,
_SYNTHESIZED_BLOCK_LABEL,
SynthesisDiagnostics,
+ SynthesizedCodeBlock,
_get_by_role_expr_strict,
_is_positional_selector,
_parse_role_name,
@@ -71,6 +73,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
code_authoring_churn_stop_signal,
credential_priority_authoring_churn_stop_signal,
+ synthesized_persistence_reopened_after_failed_run,
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
from skyvern.forge.sdk.copilot.narration import CODE_REPAIR_PROGRESS_SURFACE_KIND, CODE_REPAIR_PROGRESS_TEXT
@@ -432,6 +435,7 @@ def _normalize_code_artifact_metadata_detailed(
impose_defaults: bool = False,
scout_trajectory: list[ScoutedInteraction] | None = None,
verified_runtime_output_paths_by_label: Mapping[str, set[str]] | None = None,
+ advisory_declared_output_return_shape_labels: set[str] | None = None,
) -> CodeArtifactNormalization:
"""Normalize submitted artifact metadata at the persist seam.
@@ -445,6 +449,7 @@ def _normalize_code_artifact_metadata_detailed(
items = _code_artifact_metadata_items(raw_metadata)
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
trajectory: list[ScoutedInteraction] = scout_trajectory or []
+ advisory_return_shape_labels = advisory_declared_output_return_shape_labels or set()
violations: list[str] = []
offending_labels: list[str] = []
schema_incompatibilities: list[SchemaIncompatibility] = []
@@ -539,7 +544,7 @@ def _normalize_code_artifact_metadata_detailed(
block_code,
require_declared_output=require_declared_output,
)
- if return_shape_error is not None:
+ if return_shape_error is not None and label not in advisory_return_shape_labels:
item_violations.append(return_shape_error)
schema_conformance_error = _extraction_schema_conformance_error(label, dumped, block_code)
if schema_conformance_error is not None:
@@ -1346,6 +1351,24 @@ def _metadata_output_path_roots(raw_metadata: object) -> list[str]:
return sorted(roots)
+def _metadata_output_paths(raw_metadata: object) -> set[str]:
+ paths: set[str] = set()
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is not None:
+ paths.update(_metadata_item_output_paths(item))
+ return paths
+
+
+def _metadata_extraction_schema_paths(raw_metadata: object) -> set[str]:
+ paths: set[str] = set()
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is not None:
+ paths.update(_metadata_item_extraction_schema_paths(item))
+ return paths
+
+
def _metadata_output_path_roots_by_label(raw_metadata: object) -> dict[str, list[str]]:
roots_by_label: dict[str, set[str]] = {}
for label, paths in _metadata_output_paths_by_label(raw_metadata).items():
@@ -1383,15 +1406,21 @@ def _metadata_item_output_path_roots(item: Mapping[str, Any]) -> set[str]:
def _metadata_item_output_paths(item: Mapping[str, Any]) -> set[str]:
- """Return declared output paths plus schema property roots used for static root matching."""
paths: set[str] = set()
for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
for row in _artifact_rows(item.get(field_name)):
paths.update(_artifact_goal_value_paths(row.get("goal_value_paths")))
+ paths.update(_metadata_item_extraction_schema_paths(item))
+ return paths
+
+
+def _metadata_item_extraction_schema_paths(item: Mapping[str, Any]) -> set[str]:
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
schema = _parse_extraction_schema(row.get("extraction_schema"))
if schema is not None:
- paths.update(_schema_property_roots(schema))
- return paths
+ return _schema_property_paths(schema)
+ return set()
def _metadata_reject_code_block_output_status(
@@ -1437,6 +1466,26 @@ def _schema_property_roots(schema: Mapping[str, object]) -> set[str]:
return set()
+def _schema_property_paths(schema: Mapping[str, object], *, prefix: str = "") -> set[str]:
+ properties = schema.get("properties")
+ if isinstance(properties, Mapping):
+ paths: set[str] = set()
+ for raw_key, child in properties.items():
+ key = str(raw_key).strip()
+ if not key:
+ continue
+ path = f"{prefix}.{key}" if prefix else key
+ paths.add(path)
+ if isinstance(child, Mapping):
+ paths.update(_schema_property_paths(child, prefix=path))
+ return paths
+ items = schema.get("items")
+ if isinstance(items, Mapping):
+ array_prefix = f"{prefix}[]" if prefix else ""
+ return _schema_property_paths(items, prefix=array_prefix)
+ return set()
+
+
def _metadata_violation_categories(violations: list[str]) -> list[str]:
categories: list[str] = []
for violation in violations:
@@ -1493,9 +1542,13 @@ def _recorded_outcome_missing_output_paths(ctx: AgentContext) -> set[str]:
paths: set[str] = set()
if isinstance(latest, RecordedBuildTestOutcome) and latest.reason_code == "outcome_not_demonstrated":
for fact in latest.missing_requested_output_facts:
- path = str(fact.get("output_path") or fact.get("output_root") or "").strip()
+ path = str(fact.get("output_path") or "").strip()
if path:
paths.add(path)
+ continue
+ root = str(fact.get("output_root") or "").strip()
+ if root:
+ paths.add(root)
if paths:
return paths
verification = getattr(ctx, "completion_verification_result", None)
@@ -1510,99 +1563,1632 @@ def _recorded_outcome_missing_output_paths(ctx: AgentContext) -> set[str]:
return paths
-def _candidate_missing_required_output_paths(
- workflow_yaml: str,
- code_artifact_metadata: object,
+def _requested_output_child_paths(ctx: AgentContext) -> set[str]:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return set()
+ request_policy = getattr(ctx, "request_policy", None)
+ criteria_source = getattr(request_policy, "graded_completion_criteria", None)
+ criteria = criteria_source() if callable(criteria_source) else getattr(request_policy, "completion_criteria", [])
+ if not isinstance(criteria, list):
+ return set()
+ paths: set[str] = set()
+ for criterion in criteria:
+ if getattr(criterion, "level", None) == "definition":
+ continue
+ if getattr(criterion, "method_mandated", False):
+ continue
+ if getattr(criterion, "kind", None) == "validation_classification":
+ continue
+ path = _canonical_requested_output_path(getattr(criterion, "output_path", None))
+ if path and _output_path_has_child(path):
+ paths.add(path)
+ return paths
+
+
+def _canonical_requested_output_path(value: object) -> str:
+ if not isinstance(value, str):
+ return ""
+ path = value.strip()
+ if path == "$":
+ return ""
+ if path.startswith("$."):
+ path = path[2:]
+ elif path.startswith("$["):
+ path = path[1:]
+ path = path.replace("[*]", "[]")
+ path = re.sub(r"\[\d+\]", "[]", path)
+ return ".".join(part for part in path.split(".") if part)
+
+
+def _required_child_output_paths_for_authoring(ctx: AgentContext) -> tuple[set[str], str, str]:
+ if _recorded_outcome_requires_output_candidate(ctx):
+ return (
+ {path for path in _recorded_outcome_missing_output_paths(ctx) if _output_path_has_child(path)},
+ "recorded_outcome",
+ "recorded_outcome_missing_output_coverage",
+ )
+ return (
+ _requested_output_child_paths(ctx),
+ "requested_output_contract",
+ "requested_output_contract_missing_output_coverage",
+ )
+
+
+def _missing_requested_output_facts(paths: Iterable[str], *, reason_code: str) -> list[dict[str, object]]:
+ return [
+ {
+ "output_path": path,
+ "output_root": _output_path_root(path),
+ "reason_code": reason_code,
+ "value_status": "no_typed_value",
+ }
+ for path in sorted(paths)
+ ]
+
+
+def _single_repair_block_label(block_labels: list[str]) -> str:
+ labels = [str(label).strip() for label in block_labels if str(label).strip()]
+ return labels[0] if len(labels) == 1 else ""
+
+
+def _metadata_repair_contract(
+ *,
+ block_labels: list[str],
+ required_paths: Iterable[str],
+ source: str,
+ reason_code: str,
+) -> dict[str, object] | None:
+ paths = sorted(dict.fromkeys(str(path).strip() for path in required_paths if str(path).strip()))
+ block_label = _single_repair_block_label(block_labels)
+ if not paths or not block_label:
+ return None
+ return {
+ "block_label": block_label,
+ "required_goal_value_paths": paths,
+ "required_extraction_schema_paths": paths,
+ "required_code_return_paths": paths,
+ "source": source,
+ "reason_code": reason_code,
+ }
+
+
+def _metadata_output_repair_context(
+ *,
+ block_labels: list[str],
+ required_paths: Iterable[str],
+ coverage_reason_code: str,
+ source: str,
+ summary: str,
+) -> CodeAuthoringRepairContext | None:
+ paths = sorted(dict.fromkeys(str(path).strip() for path in required_paths if str(path).strip()))
+ block_label = _single_repair_block_label(block_labels)
+ if not paths or not block_label:
+ return None
+ path_text = ", ".join(paths)
+ return CodeAuthoringRepairContext(
+ block_label=block_label,
+ reason_code="metadata_reject",
+ runtime_failure_class=coverage_reason_code,
+ runtime_failure_reason=summary,
+ required_goal_value_paths=paths,
+ required_extraction_schema_paths=paths,
+ required_code_return_paths=paths,
+ metadata_contract_source=source,
+ metadata_contract_reason_code=coverage_reason_code,
+ repair_instruction=(
+ "Declare code_artifact_metadata goal_value_paths and extraction_schema for required output paths "
+ f"{path_text}, make the code return those paths, then rerun update_and_run_blocks."
+ ),
+ )
+
+
+_METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE = "metadata_contract_required_before_run"
+_SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE = "separated_spine_shape_required"
+_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"
+_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = "output_contract_reject_budget_exhausted"
+_MAX_OUTPUT_CONTRACT_STEERING_REJECTS = 2
+_MAX_OUTPUT_CONTRACT_REJECTS = 4
+
+
+@dataclass(frozen=True)
+class _OutputContractEvaluation:
+ block_label: str
+ artifact_id: str
+ required_paths: set[str]
+ source: str
+ reason_code: str
+ missing_metadata_paths: list[str]
+ missing_schema_paths: list[str]
+ missing_return_paths: list[str]
+ shape_violations: list[str]
+ canonical_signature: str
+ payload: dict[str, Any]
+ repair_context: CodeAuthoringRepairContext | None
+ can_attempt_run: bool = False
+
+ @property
+ def has_deficiencies(self) -> bool:
+ return bool(
+ self.missing_metadata_paths
+ or self.missing_schema_paths
+ or self.missing_return_paths
+ or self.shape_violations
+ )
+
+
+@dataclass(frozen=True)
+class _RuntimeOutputRepairContract:
+ required_paths: set[str]
+ facts: list[dict[str, Any]]
+ workflow_run_id: str
+ source: str = "runtime_output_repair"
+ reason_code: str = "runtime_output_repair_required"
+
+
+def _metadata_contract_required_paths(paths: Iterable[str]) -> set[str]:
+ return {
+ path
+ for raw_path in paths
+ for path in [_canonical_requested_output_path(str(raw_path))]
+ if path and _output_path_has_child(path)
+ }
+
+
+def _path_segments(path: str) -> list[tuple[str, bool]]:
+ segments: list[tuple[str, bool]] = []
+ for raw_part in path.split("."):
+ part = raw_part.strip()
+ if not part:
+ continue
+ if part.endswith("[]"):
+ segments.append((part[:-2], True))
+ continue
+ if "[]" in part:
+ name = part.replace("[]", "")
+ if name:
+ segments.append((name, True))
+ continue
+ segments.append((part, False))
+ return segments
+
+
+def _schema_template_for_required_paths(required_paths: Iterable[str]) -> dict[str, Any]:
+ schema: dict[str, Any] = {"type": "object", "properties": {}, "required": []}
+
+ def ensure_required(container: dict[str, Any], key: str) -> None:
+ required = container.setdefault("required", [])
+ if isinstance(required, list) and key not in required:
+ required.append(key)
+
+ for path in sorted(_metadata_contract_required_paths(required_paths)):
+ segments = _path_segments(path)
+ container = schema
+ for index, (key, is_array) in enumerate(segments):
+ if not key:
+ break
+ properties = container.setdefault("properties", {})
+ if not isinstance(properties, dict):
+ break
+ ensure_required(container, key)
+ is_leaf = index == len(segments) - 1
+ if is_array:
+ child = properties.setdefault(
+ key,
+ {"type": "array", "items": {"type": "object", "properties": {}, "required": []}},
+ )
+ if isinstance(child, dict):
+ child["type"] = "array"
+ items = child.setdefault("items", {"type": "object", "properties": {}, "required": []})
+ if isinstance(items, dict):
+ items.setdefault("type", "object")
+ items.setdefault("properties", {})
+ items.setdefault("required", [])
+ container = items
+ continue
+ child = properties.setdefault(key, {} if is_leaf else {"type": "object", "properties": {}, "required": []})
+ if isinstance(child, dict) and not is_leaf:
+ child.setdefault("type", "object")
+ child.setdefault("properties", {})
+ child.setdefault("required", [])
+ container = child
+ return schema
+
+
+def _schema_template_text_for_required_paths(required_paths: Iterable[str]) -> str:
+ return json.dumps(_schema_template_for_required_paths(required_paths), sort_keys=True)
+
+
+def _metadata_contract_template(
*,
+ block_label: str,
+ required_paths: set[str],
+ source: str,
+ reason_code: str,
+) -> dict[str, Any]:
+ artifact_id = _artifact_id_for_block_label(block_label)
+ schema_text = _schema_template_text_for_required_paths(required_paths)
+ paths = sorted(required_paths)
+ return {
+ "block_label": block_label,
+ "artifact_id": artifact_id,
+ "declared_goal": "Return the requested structured output paths.",
+ "claimed_outcomes": [
+ {
+ "id": f"claim:{artifact_id}",
+ "scope": "outcome",
+ "status": "observed_not_verified",
+ "goal_value_paths": paths,
+ "extraction_schema": schema_text,
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "id": f"expectation:{artifact_id}",
+ "goal_value_paths": paths,
+ "extraction_schema": schema_text,
+ "source": source,
+ "reason_code": reason_code,
+ }
+ ],
+ }
+
+
+def _return_skeleton_for_required_paths(required_paths: Iterable[str]) -> str:
+ paths = sorted(_metadata_contract_required_paths(required_paths))
+ roots = sorted({_output_path_root(path) for path in paths if _output_path_root(path)})
+ if not roots:
+ return ""
+ if roots == ["output"]:
+ child_names = sorted(
+ {
+ child
+ for path in paths
+ if (child := _output_path_direct_child(path, "output")) and _return_scaffold_name_is_safe(child)
+ }
+ )
+ if child_names:
+ pairs = ", ".join(f'"{name}": {name}' for name in child_names)
+ return f'return {{"output": {{{pairs}}}}}'
+ return "return output"
+ if len(roots) == 1 and _return_scaffold_name_is_safe(roots[0]):
+ root = roots[0]
+ return f'return {{"{root}": {root}}}'
+ pairs = ", ".join(f'"{root}": {root}' for root in roots if _return_scaffold_name_is_safe(root))
+ return f"return {{{pairs}}}" if pairs else ""
+
+
+def _metadata_item_for_block_label(raw_metadata: object, block_label: str) -> Mapping[str, Any] | None:
+ for raw_item in _code_artifact_metadata_items(raw_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is None:
+ continue
+ if str(item.get("block_label") or "").strip() == block_label:
+ return item
+ return None
+
+
+def _metadata_has_mapping_item(raw_metadata: object) -> bool:
+ return any(_raw_metadata_item_mapping(item) is not None for item in _code_artifact_metadata_items(raw_metadata))
+
+
+def _metadata_item_goal_value_paths(item: Mapping[str, Any] | None) -> set[str]:
+ if item is None:
+ return set()
+ paths: set[str] = set()
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
+ paths.update(
+ path
+ for raw_path in _artifact_goal_value_paths(row.get("goal_value_paths"))
+ for path in [_canonical_requested_output_path(raw_path)]
+ if path
+ )
+ return paths
+
+
+def _metadata_item_effective_schema_text(item: Mapping[str, Any] | None, required_paths: set[str]) -> str:
+ if item is None or not required_paths:
+ return ""
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
+ schema_text = row.get("extraction_schema")
+ schema = _parse_extraction_schema(schema_text)
+ if schema is not None:
+ return str(schema_text or "").strip() if required_paths <= _schema_property_paths(schema) else ""
+ return ""
+
+
+def _active_metadata_repair_block_label(ctx: AgentContext) -> str:
+ repair_context = getattr(ctx, "last_code_authoring_repair_context", None)
+ if not isinstance(repair_context, CodeAuthoringRepairContext):
+ return ""
+ if repair_context.reason_code != "metadata_reject":
+ return ""
+ return str(repair_context.block_label or "").strip()
+
+
+def _output_metadata_owner_labels(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
required_paths: set[str],
) -> list[str]:
if not required_paths:
return []
- required_paths = {path for path in (str(item).strip() for item in required_paths) if path}
- declared_paths_by_label = {
- label: set(paths) for label, paths in _metadata_output_paths_by_label(code_artifact_metadata).items()
- }
- produced_by_label = _workflow_yaml_produced_output_roots_by_label(workflow_yaml)
- covered_paths: set[str] = set()
- abstained_declared_paths: set[str] = set()
- for label, declared_paths in declared_paths_by_label.items():
- produced = produced_by_label.get(label)
- if produced is None:
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ owners: set[str] = set()
+ for raw_item in _code_artifact_metadata_items(raw_code_artifact_metadata):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is None:
continue
- for required_path in required_paths & declared_paths:
- if _top_level_path_segment(required_path) in produced.roots:
- covered_paths.add(required_path)
- if produced.abstained:
- abstained_declared_paths.update(required_paths & declared_paths)
- missing_paths = required_paths - covered_paths
- if missing_paths:
- missing_paths -= abstained_declared_paths
- return sorted(missing_paths)
+ label = str(item.get("block_label") or "").strip()
+ if label in code_blocks and required_paths <= _metadata_item_goal_value_paths(item):
+ owners.add(label)
+ repair_label = _active_metadata_repair_block_label(ctx)
+ if repair_label in code_blocks:
+ owners.add(repair_label)
+ for label, block in code_blocks.items():
+ if required_paths <= _code_block_produced_output_paths(str(block.get("code") or "")):
+ owners.add(label)
+ return sorted(owners)
-class _ProducedOutputRoots(NamedTuple):
- roots: set[str]
- abstained: bool = False
+def _scout_spine_requires_separated_blocks(ctx: AgentContext) -> bool:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return False
+ scout_trajectory = getattr(ctx, "scout_trajectory", None)
+ if not isinstance(scout_trajectory, list) or len(scout_trajectory) < 2:
+ return False
+ synthesized = synthesize_code_block(
+ scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if synthesized is None or synthesized.diagnostics.truncated:
+ return False
+ mutations, _, ambiguous = _browser_surface_for_code(synthesized.code)
+ return not ambiguous and len(mutations) >= 2
-def _workflow_yaml_produced_output_roots_by_label(workflow_yaml: str) -> dict[str, _ProducedOutputRoots]:
- return {
- label: _code_block_produced_output_roots(str(block.get("code") or ""))
- for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items()
+def _output_contract_scope_key(ctx: AgentContext | None) -> str:
+ if ctx is None:
+ return ""
+ turn_id = str(getattr(ctx, "turn_id", "") or "").strip()
+ if turn_id:
+ return f"turn:{turn_id}"
+ workflow_permanent_id = str(getattr(ctx, "workflow_permanent_id", "") or "").strip()
+ turn_state = getattr(ctx, "completion_criteria_turn_state", None)
+ active_set_id = str(getattr(turn_state, "active_set_id", "") or "").strip()
+ if workflow_permanent_id and active_set_id:
+ return f"workflow:{workflow_permanent_id}:criteria_set:{active_set_id}"
+ decision = getattr(turn_state, "decision", None)
+ epoch = getattr(decision, "epoch", None)
+ if workflow_permanent_id and epoch is not None:
+ return f"workflow:{workflow_permanent_id}:criteria_epoch:{epoch}"
+ return ""
+
+
+def _stable_output_contract_key(scope_key: str, required_paths: set[str]) -> str:
+ scope_key = scope_key.strip()
+ if not scope_key:
+ return ""
+ payload = {
+ "scope": scope_key,
+ "required_paths": sorted(required_paths),
}
+ return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
-def _code_block_produced_output_roots(code: str) -> _ProducedOutputRoots:
- try:
- tree = ast.parse(textwrap.dedent(code).strip() or "pass")
- except SyntaxError:
- return _ProducedOutputRoots(set(), True)
- scope_statements = list(_iter_top_level_scope(tree.body))
- assigned_roots = _assigned_top_level_names(tree.body)
- dict_assignments: dict[str, set[str]] = {}
- dynamic_dict_assignment_names: set[str] = set()
- helper_return_roots = _helper_function_literal_return_roots(tree.body)
- returned_roots: set[str] = set()
- abstained = False
- saw_return = False
- for node in scope_statements:
- if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
- if isinstance(node.value, ast.Dict):
- produced = _dict_literal_output_roots(node.value)
- dict_assignments[node.targets[0].id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(node.targets[0].id)
- elif isinstance(node, ast.Assign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.AnnAssign):
- if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):
- produced = _dict_literal_output_roots(node.value)
- dict_assignments[node.target.id] = produced.roots
- if produced.abstained:
- dynamic_dict_assignment_names.add(node.target.id)
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.AugAssign):
- _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
- elif isinstance(node, ast.Return):
- saw_return = True
- if node.value is not None:
- returned = _return_output_roots(
- node.value,
- dict_assignments,
- dynamic_dict_assignment_names,
- helper_return_roots,
- )
- returned_roots.update(returned.roots)
- abstained = abstained or returned.abstained
- roots = returned_roots if saw_return else assigned_roots
- return _ProducedOutputRoots(roots, abstained)
+def _output_contract_author_time_structural_payload(
+ ctx: AgentContext,
+ required_paths: set[str],
+ *,
+ block_label: str = "",
+ deficiency_family: str = "output_contract_unsatisfied",
+) -> Mapping[str, object] | None:
+ signature = _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths)
+ if not signature:
+ return None
+ payload: dict[str, object] = {
+ "version": "metadata_reject_output_contract:v1",
+ "canonical_output_contract_signature": signature,
+ "canonical_required_child_paths": sorted(required_paths),
+ "deficiency_family": deficiency_family,
+ }
+ if block_label.strip():
+ payload["block_label"] = block_label.strip()
+ return payload
+
+
+def _output_contract_signature(
+ *,
+ ctx: AgentContext | None = None,
+ scope_key: str = "",
+ workflow_yaml: str,
+ source: str,
+ reason_code: str,
+ required_paths: set[str],
+) -> str:
+ return _stable_output_contract_key(scope_key or _output_contract_scope_key(ctx), required_paths)
+
+
+def _runtime_output_contract_signature(runtime_contract: _RuntimeOutputRepairContract | None) -> str:
+ if runtime_contract is None:
+ return ""
+ payload = {
+ "workflow_run_id": runtime_contract.workflow_run_id,
+ "required_paths": sorted(runtime_contract.required_paths),
+ "facts": runtime_contract.facts,
+ }
+ return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+def _default_output_contract_block_label(workflow_yaml: str) -> str:
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ return next(iter(code_blocks)) if len(code_blocks) == 1 else ""
+
+
+def _output_contract_pin_key(ctx: AgentContext, workflow_yaml: str, required_paths: set[str]) -> str:
+ return _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths)
+
+
+def _pinned_output_contract_block_label(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ required_paths: set[str],
+) -> str:
+ pinned_by_contract = getattr(ctx, "output_contract_pinned_block_label_by_signature", None)
+ if not isinstance(pinned_by_contract, Mapping):
+ return ""
+ pin_key = _output_contract_pin_key(ctx, workflow_yaml, required_paths)
+ if not pin_key:
+ return ""
+ label = str(pinned_by_contract.get(pin_key) or "").strip()
+ return label if label in _workflow_yaml_code_blocks_by_label(workflow_yaml) else ""
+
+
+def _pin_output_contract_block_label(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ required_paths: set[str],
+ label: str,
+) -> None:
+ label = label.strip()
+ if not label:
+ return
+ pinned_by_contract = getattr(ctx, "output_contract_pinned_block_label_by_signature", None)
+ if not isinstance(pinned_by_contract, dict):
+ pinned_by_contract = {}
+ pin_key = _output_contract_pin_key(ctx, workflow_yaml, required_paths)
+ if not pin_key:
+ return
+ pinned_by_contract.setdefault(pin_key, label)
+ ctx.output_contract_pinned_block_label_by_signature = pinned_by_contract
+
+
+def _target_output_contract_block_label(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+ required_paths: set[str],
+) -> tuple[str, list[str]]:
+ pinned_label = _pinned_output_contract_block_label(ctx, workflow_yaml, required_paths)
+ if pinned_label:
+ return pinned_label, [pinned_label]
+ owner_labels = _output_metadata_owner_labels(ctx, workflow_yaml, raw_code_artifact_metadata, required_paths)
+ if len(owner_labels) == 1:
+ _pin_output_contract_block_label(ctx, workflow_yaml, required_paths, owner_labels[0])
+ return owner_labels[0], owner_labels
+ default_label = _default_output_contract_block_label(workflow_yaml)
+ if default_label and not owner_labels:
+ _pin_output_contract_block_label(ctx, workflow_yaml, required_paths, default_label)
+ return default_label, [default_label]
+ return "", owner_labels
+
+
+def _runtime_output_repair_contract_from_recorded_outcome(ctx: AgentContext) -> _RuntimeOutputRepairContract | None:
+ outcome = getattr(ctx, "latest_recorded_build_test_outcome", None)
+ if not isinstance(outcome, RecordedBuildTestOutcome):
+ return None
+ if not (
+ outcome.is_authoritative
+ and outcome.phase == "persisted_block_run"
+ and outcome.reason_code == "outcome_not_demonstrated"
+ and outcome.workflow_run_id
+ ):
+ return None
+ facts: list[dict[str, Any]] = []
+ required_paths: set[str] = set()
+ block_labels: set[str] = set()
+ for raw_fact in outcome.runtime_output_repair_facts:
+ if not isinstance(raw_fact, Mapping):
+ return None
+ if str(raw_fact.get("workflow_run_id") or "").strip() != outcome.workflow_run_id:
+ return None
+ path = _canonical_requested_output_path(str(raw_fact.get("output_path") or ""))
+ if not path or not _output_path_has_child(path):
+ return None
+ fact = dict(raw_fact)
+ fact["output_path"] = path
+ fact["output_root"] = _output_path_root(path)
+ label = str(fact.get("block_label") or "").strip()
+ if label:
+ block_labels.add(label)
+ required_paths.add(path)
+ facts.append(fact)
+ if not facts or not required_paths or len(block_labels) > 1:
+ return None
+ return _RuntimeOutputRepairContract(
+ required_paths=required_paths,
+ facts=sorted(facts, key=lambda item: str(item.get("output_path") or "")),
+ workflow_run_id=outcome.workflow_run_id,
+ )
+
+
+def _output_contract_required_paths_source(ctx: AgentContext) -> tuple[set[str], str, str]:
+ runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
+ if runtime_contract is not None:
+ return runtime_contract.required_paths, runtime_contract.source, runtime_contract.reason_code
+ required_paths, source, reason_code = _required_child_output_paths_for_authoring(ctx)
+ repair_context = getattr(ctx, "last_code_authoring_repair_context", None)
+ if required_paths or not isinstance(repair_context, CodeAuthoringRepairContext):
+ return required_paths, source, reason_code
+ if repair_context.reason_code != "metadata_reject":
+ return required_paths, source, reason_code
+ required_paths = _metadata_contract_required_paths(
+ [
+ *repair_context.required_goal_value_paths,
+ *repair_context.required_extraction_schema_paths,
+ *repair_context.required_code_return_paths,
+ ]
+ )
+ source = str(repair_context.metadata_contract_source or "").strip() or "metadata_reject"
+ reason_code = (
+ str(repair_context.metadata_contract_reason_code or "").strip()
+ or str(repair_context.runtime_failure_class or "").strip()
+ or "metadata_reject"
+ )
+ return required_paths, source, reason_code
+
+
+def _code_return_is_static_advisory_candidate(code: str, required_paths: set[str]) -> bool:
+ if not required_paths:
+ return False
+ produced = _code_block_produced_output_roots(code)
+ if not produced.abstained:
+ return False
+ if _code_block_produced_output_paths(code):
+ return False
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return False
+ for node in _iter_top_level_scope(tree.body):
+ if isinstance(node, ast.Return) and node.value is not None and _expr_is_flat_string(node.value, set()):
+ return False
+ return True
+
+
+def _evaluate_output_contract_for_code_block(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+ *,
+ allow_static_return_advisory: bool = False,
+) -> _OutputContractEvaluation | None:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return None
+ runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
+ required_paths, source, reason_code = _output_contract_required_paths_source(ctx)
+ if not required_paths:
+ return None
+ effective_metadata = raw_code_artifact_metadata
+ if not _metadata_has_mapping_item(effective_metadata):
+ existing_metadata = getattr(ctx, "code_artifact_metadata", None)
+ if _metadata_has_mapping_item(existing_metadata):
+ effective_metadata = existing_metadata
+ block_label, owner_labels = _target_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ effective_metadata,
+ required_paths,
+ )
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ target_block = code_blocks.get(block_label) if block_label else None
+ target_metadata = _metadata_item_for_block_label(effective_metadata, block_label) if block_label else None
+ submitted_goal_paths = _metadata_item_goal_value_paths(target_metadata)
+ submitted_schema_paths = _metadata_item_extraction_schema_paths(target_metadata) if target_metadata else set()
+ submitted_code_paths = (
+ _code_block_produced_output_paths(str(target_block.get("code") or "")) if target_block is not None else set()
+ )
+ missing_metadata_paths = sorted(required_paths - submitted_goal_paths)
+ missing_schema_paths = sorted(required_paths - submitted_schema_paths)
+ missing_return_paths = sorted(required_paths - submitted_code_paths)
+ shape_violations: list[str] = []
+ if not block_label:
+ shape_violations.append("ambiguous_output_owner" if owner_labels else "missing_output_owner")
+ elif target_block is None:
+ shape_violations.append("missing_output_block")
+ elif required_paths and _scout_spine_requires_separated_blocks(ctx):
+ synthesized = synthesize_code_block(
+ ctx.scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if synthesized is not None and _browser_surface_contains_full_action_spine(
+ str(target_block.get("code") or ""), synthesized.code
+ ):
+ shape_violations.append(_SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE)
+
+ signature = _output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source=source,
+ reason_code=reason_code,
+ required_paths=required_paths,
+ )
+ post_steering_static_return_advisory = (
+ allow_static_return_advisory
+ and bool(missing_return_paths)
+ and not missing_metadata_paths
+ and not missing_schema_paths
+ and not shape_violations
+ and target_block is not None
+ and _output_contract_reject_count(ctx, signature) >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS
+ )
+ static_return_advisory = (
+ allow_static_return_advisory
+ and bool(missing_return_paths)
+ and not missing_metadata_paths
+ and not missing_schema_paths
+ and not shape_violations
+ and target_block is not None
+ and (
+ post_steering_static_return_advisory
+ or _code_return_is_static_advisory_candidate(str(target_block.get("code") or ""), required_paths)
+ )
+ )
+ effective_missing_return_paths = [] if static_return_advisory else missing_return_paths
+ runtime_signature = _runtime_output_contract_signature(runtime_contract)
+ artifact_id = _artifact_id_for_block_label(block_label) if block_label else ""
+ metadata_repair_contract = (
+ _metadata_repair_contract(
+ block_labels=[block_label],
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ )
+ if block_label
+ else None
+ )
+ repair = _metadata_output_repair_context(
+ block_labels=[block_label] if block_label else [],
+ required_paths=required_paths,
+ coverage_reason_code=reason_code,
+ source=source,
+ summary="Submitted workflow does not satisfy the requested output contract.",
+ )
+ missing_paths = sorted(
+ set(missing_metadata_paths)
+ | set(missing_schema_paths)
+ | set(effective_missing_return_paths)
+ | (required_paths if shape_violations else set())
+ )
+ payload: dict[str, Any] = {
+ "reason_code": _OUTPUT_CONTRACT_REJECT_REASON_CODE,
+ "block_label": block_label,
+ "artifact_id": artifact_id,
+ "canonical_required_child_paths": sorted(required_paths),
+ "source": source,
+ "metadata_contract_source": source,
+ "metadata_contract_reason_code": reason_code,
+ "missing_goal_value_paths": missing_metadata_paths,
+ "missing_extraction_schema_paths": missing_schema_paths,
+ "missing_code_return_paths": effective_missing_return_paths,
+ "static_return_advisory_paths": missing_return_paths if static_return_advisory else [],
+ "post_steering_static_return_advisory": post_steering_static_return_advisory,
+ "shape_violations": shape_violations,
+ "can_attempt_run": static_return_advisory,
+ "reject_reason": "" if static_return_advisory else _OUTPUT_CONTRACT_REJECT_REASON_CODE,
+ "canonical_output_contract_signature": signature,
+ "canonical_runtime_output_contract_signature": runtime_signature,
+ "runtime_output_workflow_run_id": runtime_contract.workflow_run_id if runtime_contract is not None else "",
+ "runtime_output_repair_facts": runtime_contract.facts if runtime_contract is not None else [],
+ "output_owner_labels": owner_labels,
+ "metadata_repair_contract": metadata_repair_contract,
+ "satisfying_templates": {
+ "code_artifact_metadata": (
+ _metadata_contract_template(
+ block_label=block_label,
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ )
+ if block_label
+ else None
+ ),
+ "extraction_schema": _schema_template_for_required_paths(required_paths),
+ "return_skeleton": _return_skeleton_for_required_paths(required_paths),
+ },
+ "missing_requested_output_facts": _missing_requested_output_facts(
+ missing_paths,
+ reason_code=reason_code,
+ ),
+ }
+ progress_data = _code_repair_progress_data(
+ repair,
+ missing_requested_output_facts=payload["missing_requested_output_facts"],
+ metadata_repair_contract=metadata_repair_contract,
+ )
+ progress_data.update(payload)
+ return _OutputContractEvaluation(
+ block_label=block_label,
+ artifact_id=artifact_id,
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ missing_metadata_paths=missing_metadata_paths,
+ missing_schema_paths=missing_schema_paths,
+ missing_return_paths=effective_missing_return_paths,
+ shape_violations=shape_violations,
+ canonical_signature=signature,
+ payload=progress_data,
+ repair_context=repair,
+ can_attempt_run=static_return_advisory,
+ )
+
+
+def _record_output_contract_reject(
+ ctx: AgentContext,
+ evaluation: _OutputContractEvaluation,
+ *,
+ summary: str,
+) -> dict[str, Any]:
+ count = _record_output_contract_family_reject(
+ ctx,
+ evaluation.required_paths,
+ reject_family=str(evaluation.payload.get("reason_code") or _OUTPUT_CONTRACT_REJECT_REASON_CODE),
+ )
+ payload = dict(evaluation.payload)
+ payload["output_contract_reject_count"] = count
+ payload["output_contract_reject_budget"] = _MAX_OUTPUT_CONTRACT_REJECTS
+ if count >= _MAX_OUTPUT_CONTRACT_REJECTS:
+ payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
+ payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
+ structural_payload = _output_contract_author_time_structural_payload(
+ ctx,
+ evaluation.required_paths,
+ block_label=evaluation.block_label,
+ )
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary=summary,
+ structural_payload=structural_payload or payload,
+ block_labels=[evaluation.block_label] if evaluation.block_label else [],
+ missing_requested_output_facts=payload.get("missing_requested_output_facts")
+ if isinstance(payload.get("missing_requested_output_facts"), list)
+ else None,
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return payload
+
+
+def _record_output_contract_family_reject(
+ ctx: AgentContext,
+ required_paths: set[str],
+ *,
+ reject_family: str,
+) -> int:
+ if not required_paths:
+ return 0
+ scope_key = _output_contract_scope_key(ctx)
+ signature = _stable_output_contract_key(scope_key, required_paths)
+ if not signature:
+ LOG.info(
+ "copilot_output_contract_reject_count_unscoped",
+ reject_family=reject_family,
+ canonical_required_child_paths=sorted(required_paths),
+ )
+ return 0
+ count_by_signature = getattr(ctx, "output_contract_reject_count_by_signature", None)
+ if not isinstance(count_by_signature, dict):
+ count_by_signature = {}
+ count = int(count_by_signature.get(signature, 0) or 0) + 1
+ count_by_signature[signature] = count
+ ctx.output_contract_reject_count_by_signature = count_by_signature
+ LOG.info(
+ "copilot_output_contract_reject_counted",
+ output_contract_scope_key=scope_key,
+ canonical_output_contract_signature=signature,
+ output_contract_reject_count=count,
+ output_contract_reject_budget=_MAX_OUTPUT_CONTRACT_REJECTS,
+ reject_family=reject_family,
+ canonical_required_child_paths=sorted(required_paths),
+ )
+ return count
+
+
+def _output_contract_reject_result(
+ evaluation: _OutputContractEvaluation,
+ *,
+ payload: dict[str, Any] | None = None,
+ tool_name: str = "update_workflow",
+) -> dict[str, Any]:
+ data = payload or evaluation.payload
+ if data.get("reason_code") == _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE:
+ error = (
+ "The workflow output contract repair budget is exhausted for this canonical requested-output contract. "
+ "Return with the typed output-contract payload instead of trying another variant."
+ )
+ else:
+ path_text = ", ".join(str(path) for path in data.get("canonical_required_child_paths", []) or [])
+ path_suffix = f" Required requested output paths: {path_text}." if path_text else ""
+ error = (
+ f"{tool_name} cannot proceed until the submitted workflow satisfies the requested output contract. "
+ "Use the returned code_artifact_metadata, extraction_schema, and return skeleton templates exactly for "
+ "the canonical required child paths." + path_suffix
+ )
+ return {
+ "ok": False,
+ "error": error,
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ "data": data,
+ }
+
+
+def _ensure_metadata_contract_rows(
+ item: dict[str, Any],
+ *,
+ required_paths: set[str],
+ schema_text: str,
+) -> None:
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ rows = _artifact_mutable_rows(item.get(field_name))
+ if not rows:
+ item[field_name] = [{"goal_value_paths": sorted(required_paths)}]
+ rows = _artifact_mutable_rows(item.get(field_name))
+ for row in rows:
+ if not _artifact_goal_value_paths(row.get("goal_value_paths")):
+ row["goal_value_paths"] = sorted(required_paths)
+ if schema_text and not str(row.get("extraction_schema") or "").strip():
+ row["extraction_schema"] = schema_text
+
+
+def _metadata_item_declares_extraction_schema(item: Mapping[str, Any] | None) -> bool:
+ if item is None:
+ return False
+ for field_name in ("claimed_outcomes", "terminal_verifier_expectations"):
+ for row in _artifact_rows(item.get(field_name)):
+ if str(row.get("extraction_schema") or "").strip():
+ return True
+ return False
+
+
+def _output_contract_scaffold_target_label(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+ required_paths: set[str],
+ *,
+ allow_missing_static_return: bool = False,
+) -> str:
+ label, owner_labels = _target_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths,
+ )
+ if not label or owner_labels != [label]:
+ return ""
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ target_block = code_blocks.get(label)
+ if target_block is None:
+ return ""
+ target_code = str(target_block.get("code") or "")
+ if not allow_missing_static_return and not required_paths <= _code_block_produced_output_paths(target_code):
+ return ""
+ if _scout_spine_requires_separated_blocks(ctx):
+ synthesized = synthesize_code_block(
+ ctx.scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if synthesized is not None and _browser_surface_contains_full_action_spine(target_code, synthesized.code):
+ return ""
+ return label
+
+
+def _apply_metadata_contract_scaffold(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+ *,
+ required_paths: set[str],
+ source: str,
+ reason_code: str,
+ allow_missing_static_return: bool = False,
+) -> object:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return raw_code_artifact_metadata
+ if not required_paths:
+ return raw_code_artifact_metadata
+ label = _output_contract_scaffold_target_label(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths,
+ allow_missing_static_return=allow_missing_static_return,
+ )
+ if not label:
+ return raw_code_artifact_metadata
+ existing_item = _metadata_item_for_block_label(raw_code_artifact_metadata, label)
+ prior_item = _metadata_item_for_block_label(getattr(ctx, "code_artifact_metadata", None), label)
+ if _metadata_item_declares_extraction_schema(existing_item) and not _metadata_item_effective_schema_text(
+ existing_item, required_paths
+ ):
+ return raw_code_artifact_metadata
+ if _metadata_item_declares_extraction_schema(prior_item) and not _metadata_item_effective_schema_text(
+ prior_item, required_paths
+ ):
+ return raw_code_artifact_metadata
+ schema_text = (
+ _metadata_item_effective_schema_text(existing_item, required_paths)
+ or _metadata_item_effective_schema_text(
+ prior_item,
+ required_paths,
+ )
+ or _schema_template_text_for_required_paths(required_paths)
+ )
+ items = [
+ copy.deepcopy(item)
+ for item in _code_artifact_metadata_items(raw_code_artifact_metadata)
+ if _raw_metadata_item_mapping(item) is not None
+ ]
+ target_index: int | None = None
+ for index, raw_item in enumerate(items):
+ item = _raw_metadata_item_mapping(raw_item)
+ if item is not None and str(item.get("block_label") or "").strip() == label:
+ target_index = index
+ break
+ if target_index is None:
+ target: dict[str, Any] = {"block_label": label}
+ items.append(target)
+ else:
+ target = dict(items[target_index])
+ items[target_index] = target
+ target["block_label"] = label
+ target["artifact_id"] = _artifact_id_for_block_label(label)
+ _ensure_metadata_contract_rows(target, required_paths=required_paths, schema_text=schema_text)
+ return items
+
+
+def _scaffold_metadata_contract_for_update(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+) -> tuple[object, bool]:
+ required_paths, source, reason_code = _output_contract_required_paths_source(ctx)
+ if not required_paths:
+ return raw_code_artifact_metadata, False
+ signature = _output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source=source,
+ reason_code=reason_code,
+ required_paths=required_paths,
+ )
+ scaffolded = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ allow_missing_static_return=_output_contract_reject_count(ctx, signature)
+ >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS,
+ )
+ return scaffolded, scaffolded is not raw_code_artifact_metadata
+
+
+def _apply_metadata_contract_schema_to_workflow_yaml(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+) -> str:
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return workflow_yaml
+ required_paths, source, reason_code = _output_contract_required_paths_source(ctx)
+ if not required_paths:
+ return workflow_yaml
+ label, owner_labels = _target_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths,
+ )
+ if not label or owner_labels != [label]:
+ return workflow_yaml
+ metadata_item = _metadata_item_for_block_label(raw_code_artifact_metadata, label)
+ schema_text = _metadata_item_effective_schema_text(metadata_item, required_paths)
+ if not schema_text:
+ return workflow_yaml
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return workflow_yaml
+ applied = False
+ for block in _workflow_code_blocks(parsed):
+ if str(block.get("label") or "").strip() != label:
+ continue
+ if str(block.get("extraction_schema") or "").strip():
+ return workflow_yaml
+ block["extraction_schema"] = schema_text
+ applied = True
+ break
+ if not applied:
+ return workflow_yaml
+ LOG.info(
+ "copilot_output_contract_schema_projected_to_workflow",
+ block_label=label,
+ canonical_required_child_paths=sorted(required_paths),
+ source=source,
+ reason_code=reason_code,
+ )
+ return yaml.safe_dump(parsed, sort_keys=False)
+
+
+def _workflow_needs_contract_readback_persist(
+ ctx: AgentContext,
+ prior_workflow: Workflow | None,
+ workflow: Workflow,
+ *,
+ allow_static_output_uncertainty: bool,
+) -> bool:
+ if not allow_static_output_uncertainty:
+ return False
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return False
+ if prior_workflow is None:
+ return False
+ definition = getattr(workflow, "workflow_definition", None)
+ if not getattr(definition, "blocks", None):
+ return False
+ prior_definition = getattr(prior_workflow, "workflow_definition", None)
+ if getattr(prior_definition, "blocks", None):
+ return False
+ return True
+
+
+def _output_contract_reject_count(ctx: AgentContext, signature: str) -> int:
+ count_by_signature = getattr(ctx, "output_contract_reject_count_by_signature", None)
+ if not isinstance(count_by_signature, Mapping):
+ return 0
+ try:
+ return int(count_by_signature.get(signature, 0) or 0)
+ except (TypeError, ValueError):
+ return 0
+
+
+def _runtime_output_repair_attempt_key(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ required_paths: set[str],
+ source: str,
+ reason_code: str,
+) -> str:
+ runtime_contract = _runtime_output_repair_contract_from_recorded_outcome(ctx)
+ outcome = getattr(ctx, "latest_recorded_build_test_outcome", None)
+ if runtime_contract is None or not isinstance(outcome, RecordedBuildTestOutcome):
+ return ""
+ payload = {
+ "output_contract_signature": _output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source=source,
+ reason_code=reason_code,
+ required_paths=required_paths,
+ ),
+ "runtime_output_contract_signature": _runtime_output_contract_signature(runtime_contract),
+ "recorded_structural_key": outcome.structural_key,
+ "authored_structure_signature": outcome.authored_structure_signature,
+ }
+ return hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode("utf-8")).hexdigest()
+
+
+def _runtime_output_repair_attempt_recorded(ctx: AgentContext, attempt_key: str) -> bool:
+ if not attempt_key:
+ return True
+ attempts = getattr(ctx, "runtime_output_repair_attempt_by_signature", None)
+ return isinstance(attempts, Mapping) and bool(attempts.get(attempt_key))
+
+
+def _record_runtime_output_repair_attempt(ctx: AgentContext, attempt_key: str) -> None:
+ if not attempt_key:
+ return
+ attempts = getattr(ctx, "runtime_output_repair_attempt_by_signature", None)
+ if not isinstance(attempts, dict):
+ attempts = {}
+ attempts[attempt_key] = True
+ ctx.runtime_output_repair_attempt_by_signature = attempts
+
+
+def _impose_output_contract_envelope_after_steering(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+) -> tuple[str, object, bool]:
+ required_paths, source, reason_code = _output_contract_required_paths_source(ctx)
+ if not required_paths:
+ return workflow_yaml, raw_code_artifact_metadata, False
+ signature = _output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source=source,
+ reason_code=reason_code,
+ required_paths=required_paths,
+ )
+ runtime_attempt_key = _runtime_output_repair_attempt_key(ctx, workflow_yaml, required_paths, source, reason_code)
+ if _output_contract_reject_count(
+ ctx, signature
+ ) < _MAX_OUTPUT_CONTRACT_STEERING_REJECTS and _runtime_output_repair_attempt_recorded(ctx, runtime_attempt_key):
+ return workflow_yaml, raw_code_artifact_metadata, False
+ label, owner_labels = _target_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths,
+ )
+ if not label or owner_labels != [label]:
+ LOG.info(
+ "copilot_output_contract_envelope_scaffold_bailed",
+ reason="ambiguous_or_invalid_output_owner",
+ canonical_output_contract_signature=signature,
+ )
+ return workflow_yaml, raw_code_artifact_metadata, False
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return workflow_yaml, raw_code_artifact_metadata, False
+ code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
+ target_block = code_blocks.get(label)
+ if target_block is None:
+ return workflow_yaml, raw_code_artifact_metadata, False
+ target_code = str(target_block.get("code") or "")
+ if _scout_spine_requires_separated_blocks(ctx):
+ synthesized = synthesize_code_block(
+ ctx.scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=getattr(ctx, "reached_download_target", None),
+ )
+ if synthesized is not None and _browser_surface_contains_full_action_spine(target_code, synthesized.code):
+ LOG.info(
+ "copilot_output_contract_envelope_scaffold_bailed",
+ reason="separated_spine_shape_violation",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ )
+ return workflow_yaml, raw_code_artifact_metadata, False
+ keyed_code, violations = _extraction_code_with_required_static_return(target_code, required_paths=required_paths)
+ if violations:
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ allow_missing_static_return=True,
+ )
+ scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
+ if scaffolded:
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_envelope_imposed_after_steering",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ steering_reject_count=_output_contract_reject_count(ctx, signature),
+ return_envelope_applied=False,
+ metadata_scaffold_applied=True,
+ )
+ LOG.info(
+ "copilot_output_contract_envelope_scaffold_bailed",
+ reason="static_return_envelope_unavailable",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ violations=violations,
+ )
+ return workflow_yaml, scaffolded_metadata, scaffolded
+ changed_code = keyed_code != textwrap.dedent(target_code).strip()
+ if changed_code:
+ for block in _workflow_code_blocks(parsed):
+ if str(block.get("label") or "").strip() == label:
+ block["code"] = keyed_code.rstrip() + "\n"
+ break
+ workflow_yaml = yaml.safe_dump(parsed, sort_keys=False)
+ scaffolded_metadata = _apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ )
+ scaffolded = scaffolded_metadata is not raw_code_artifact_metadata
+ applied = changed_code or scaffolded
+ if applied:
+ _record_runtime_output_repair_attempt(ctx, runtime_attempt_key)
+ LOG.info(
+ "copilot_output_contract_envelope_imposed_after_steering",
+ block_label=label,
+ canonical_output_contract_signature=signature,
+ steering_reject_count=_output_contract_reject_count(ctx, signature),
+ return_envelope_applied=changed_code,
+ metadata_scaffold_applied=scaffolded,
+ )
+ return workflow_yaml, scaffolded_metadata, applied
+
+
+def _metadata_contract_run_preflight_reject(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+) -> dict[str, Any] | None:
+ workflow_yaml, raw_code_artifact_metadata, _ = _impose_output_contract_envelope_after_steering(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ )
+ raw_code_artifact_metadata, _ = _scaffold_metadata_contract_for_update(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ )
+ evaluation = _evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ raw_code_artifact_metadata,
+ allow_static_return_advisory=True,
+ )
+ if evaluation is None or not evaluation.has_deficiencies or evaluation.can_attempt_run:
+ return None
+ payload = _record_output_contract_reject(
+ ctx,
+ evaluation,
+ summary="Submitted workflow does not satisfy the requested output contract before run.",
+ )
+ payload = dict(payload)
+ payload["output_contract_reason_code"] = payload.get("reason_code")
+ payload["reason_code"] = _METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE
+ payload["reject_reason"] = _METADATA_CONTRACT_REQUIRED_BEFORE_RUN_REASON_CODE
+ block_label = evaluation.block_label or "the target output block"
+ return {
+ "ok": False,
+ "error": (
+ "update_and_run_blocks cannot attempt a run until structurally complete "
+ f"`code_artifact_metadata` is submitted for `{block_label}`."
+ ),
+ "user_facing_summary": _compiled_authoring_user_summary(),
+ "data": payload,
+ }
+
+
+def _output_path_root(path: str) -> str:
+ return path.split(".", 1)[0].split("[", 1)[0].strip()
+
+
+def _output_path_has_child(path: str) -> bool:
+ return "." in path or "[" in path
+
+
+def _candidate_missing_required_output_paths(
+ workflow_yaml: str,
+ code_artifact_metadata: object,
+ *,
+ required_paths: set[str],
+) -> list[str]:
+ if not required_paths:
+ return []
+ required_paths = {path for path in (str(item).strip() for item in required_paths) if path}
+ declared_paths_by_label = {
+ label: set(paths) for label, paths in _metadata_output_paths_by_label(code_artifact_metadata).items()
+ }
+ produced_by_label = _workflow_yaml_produced_output_roots_by_label(workflow_yaml)
+ covered_paths: set[str] = set()
+ abstained_declared_paths: set[str] = set()
+ for label, declared_paths in declared_paths_by_label.items():
+ produced = produced_by_label.get(label)
+ if produced is None:
+ continue
+ for required_path in required_paths & declared_paths:
+ if _top_level_path_segment(required_path) in produced.roots:
+ covered_paths.add(required_path)
+ if produced.abstained:
+ abstained_declared_paths.update(required_paths & declared_paths)
+ missing_paths = required_paths - covered_paths
+ if missing_paths:
+ missing_paths -= abstained_declared_paths
+ return sorted(missing_paths)
+
+
+def _recorded_runtime_produced_output_roots(ctx: AgentContext, workflow_yaml: str) -> set[str]:
+ verified_outputs = getattr(ctx, "verified_block_outputs", None)
+ if not isinstance(verified_outputs, Mapping):
+ return set()
+ code_block_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ roots: set[str] = set()
+ for label, output in verified_outputs.items():
+ if not isinstance(label, str) or label not in code_block_labels:
+ continue
+ roots.update(_meaningful_runtime_output_roots(output))
+ return roots
+
+
+def _meaningful_runtime_output_roots(value: object, *, prefix: str = "") -> set[str]:
+ roots: set[str] = set()
+ if isinstance(value, Mapping):
+ for raw_key, child in value.items():
+ if not isinstance(raw_key, str):
+ continue
+ key = raw_key.strip()
+ if not key or key == "evidence_text" or not _is_structural_runtime_output_key(key):
+ continue
+ path = f"{prefix}.{key}" if prefix else key
+ if _runtime_output_value_is_meaningful(child):
+ roots.add(path.split(".", 1)[0])
+ roots.update(_meaningful_runtime_output_roots(child, prefix=path))
+ return roots
+ if isinstance(value, list):
+ for item in value:
+ roots.update(_meaningful_runtime_output_roots(item, prefix=prefix))
+ return roots
+
+
+def _runtime_output_value_is_meaningful(value: object) -> bool:
+ if value is None:
+ return False
+ if isinstance(value, str):
+ return bool(value.strip())
+ if isinstance(value, Mapping):
+ return any(_runtime_output_value_is_meaningful(item) for item in value.values())
+ if isinstance(value, list):
+ return any(_runtime_output_value_is_meaningful(item) for item in value)
+ return True
+
+
+class _ProducedOutputRoots(NamedTuple):
+ roots: set[str]
+ abstained: bool = False
+
+
+def _workflow_yaml_produced_output_roots_by_label(workflow_yaml: str) -> dict[str, _ProducedOutputRoots]:
+ return {
+ label: _code_block_produced_output_roots(str(block.get("code") or ""))
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items()
+ }
+
+
+def _workflow_yaml_produced_output_roots(workflow_yaml: str) -> set[str]:
+ roots: set[str] = set()
+ for produced in _workflow_yaml_produced_output_roots_by_label(workflow_yaml).values():
+ roots.update(produced.roots)
+ return roots
+
+
+def _workflow_yaml_produced_output_paths(workflow_yaml: str) -> set[str]:
+ paths: set[str] = set()
+ for block in _workflow_yaml_code_blocks_by_label(workflow_yaml).values():
+ paths.update(_code_block_produced_output_paths(str(block.get("code") or "")))
+ return paths
+
+
+def _code_block_produced_output_roots(code: str) -> _ProducedOutputRoots:
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return _ProducedOutputRoots(set(), True)
+ scope_statements = list(_iter_top_level_scope(tree.body))
+ assigned_roots = _assigned_top_level_names(tree.body)
+ dict_assignments: dict[str, set[str]] = {}
+ dynamic_dict_assignment_names: set[str] = set()
+ helper_return_roots = _helper_function_literal_return_roots(tree.body)
+ returned_roots: set[str] = set()
+ abstained = False
+ saw_return = False
+ for node in scope_statements:
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ if isinstance(node.value, ast.Dict):
+ produced = _dict_literal_output_roots(node.value)
+ dict_assignments[node.targets[0].id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(node.targets[0].id)
+ elif isinstance(node, ast.Assign):
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
+ elif isinstance(node, ast.AnnAssign):
+ if isinstance(node.target, ast.Name) and isinstance(node.value, ast.Dict):
+ produced = _dict_literal_output_roots(node.value)
+ dict_assignments[node.target.id] = produced.roots
+ if produced.abstained:
+ dynamic_dict_assignment_names.add(node.target.id)
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
+ elif isinstance(node, ast.AugAssign):
+ _apply_literal_dict_key_assignment(dict_assignments, dynamic_dict_assignment_names, node)
+ elif isinstance(node, ast.Return):
+ saw_return = True
+ if node.value is not None:
+ returned = _return_output_roots(
+ node.value,
+ dict_assignments,
+ dynamic_dict_assignment_names,
+ helper_return_roots,
+ )
+ returned_roots.update(returned.roots)
+ abstained = abstained or returned.abstained
+ roots = returned_roots if saw_return else assigned_roots
+ return _ProducedOutputRoots(roots, abstained)
+
+
+def _code_block_produced_output_paths(code: str) -> set[str]:
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return set()
+ scope_statements = list(_iter_top_level_scope(tree.body))
+ dict_assignments: dict[str, set[str]] = {}
+ helper_return_paths = _helper_function_literal_return_paths(tree.body)
+ returned_paths: set[str] = set()
+ for node in scope_statements:
+ if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name):
+ dict_paths = _dict_literal_string_key_paths(node.value, dict_assignments)
+ if dict_paths:
+ dict_assignments[node.targets[0].id] = dict_paths
+ elif isinstance(node, ast.Assign):
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.AnnAssign):
+ if isinstance(node.target, ast.Name) and node.value is not None:
+ dict_assignments[node.target.id] = _dict_literal_string_key_paths(node.value, dict_assignments)
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.AugAssign):
+ _apply_literal_dict_key_assignment(dict_assignments, node)
+ elif isinstance(node, ast.Return) and node.value is not None:
+ returned_paths.update(_return_output_paths(node.value, dict_assignments, helper_return_paths))
+ return returned_paths
+
+
+def _top_level_simple_assignment_names(code: str) -> set[str]:
+ try:
+ tree = ast.parse(textwrap.dedent(code).strip() or "pass")
+ except SyntaxError:
+ return set()
+ names: set[str] = set()
+ for node in tree.body:
+ if isinstance(node, ast.Assign):
+ for target in node.targets:
+ if isinstance(target, ast.Name):
+ names.add(target.id)
+ elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
+ names.add(node.target.id)
+ return names
+
+
+def _output_path_direct_child(path: str, root: str) -> str:
+ if not path.startswith(root + "."):
+ return ""
+ child = path[len(root) + 1 :]
+ return re.split(r"[.\[]", child, maxsplit=1)[0].strip()
+
+
+def _return_scaffold_name_is_safe(name: str) -> bool:
+ return name.isidentifier() and not keyword.iskeyword(name)
+
+
+def _extraction_code_with_required_static_return(
+ code: str,
+ *,
+ required_paths: set[str],
+) -> tuple[str, list[str]]:
+ stripped_code = textwrap.dedent(code).strip()
+ if not stripped_code or not required_paths:
+ return stripped_code, []
+ if required_paths <= _code_block_produced_output_paths(stripped_code):
+ return stripped_code, []
+ try:
+ tree = ast.parse(stripped_code)
+ except SyntaxError:
+ return stripped_code, []
+ if any(isinstance(node, ast.Return) for node in _iter_top_level_scope(tree.body)):
+ replacement = _replace_direct_child_local_return(stripped_code, tree, required_paths)
+ if replacement and required_paths <= _code_block_produced_output_paths(replacement):
+ return replacement, []
+ missing = sorted(required_paths - _code_block_produced_output_paths(stripped_code))
+ return stripped_code, [
+ "Unable to impose synthesized code block: selected output extraction does not return a keyed "
+ f"structure covering required output path(s): {', '.join(missing)}."
+ ]
+ assigned_names = _top_level_simple_assignment_names(stripped_code)
+ roots = {_output_path_root(path) for path in required_paths if _output_path_root(path)}
+ candidate = ""
+ if len(roots) == 1:
+ root = next(iter(roots))
+ if _return_scaffold_name_is_safe(root) and root in assigned_names:
+ candidate = stripped_code + f'\nreturn {{"{root}": {root}}}'
+ elif root == "output" and "output" in assigned_names:
+ candidate = stripped_code + "\nreturn output"
+ elif root == "output":
+ child_names = sorted(
+ {
+ child
+ for path in required_paths
+ if (child := _output_path_direct_child(path, "output"))
+ and _return_scaffold_name_is_safe(child)
+ and child in assigned_names
+ }
+ )
+ required_child_names = {
+ child
+ for path in required_paths
+ if (child := _output_path_direct_child(path, "output")) and _return_scaffold_name_is_safe(child)
+ }
+ if child_names and set(child_names) == required_child_names:
+ child_pairs = ", ".join(f'"{name}": {name}' for name in child_names)
+ candidate = stripped_code + f'\nreturn {{"output": {{{child_pairs}}}}}'
+ if candidate and required_paths <= _code_block_produced_output_paths(candidate):
+ return candidate, []
+ missing = sorted(required_paths - _code_block_produced_output_paths(candidate or stripped_code))
+ return stripped_code, [
+ "Unable to impose synthesized code block: selected output extraction does not return a keyed "
+ f"structure covering required output path(s): {', '.join(missing)}."
+ ]
+
+
+def _replace_direct_child_local_return(code: str, tree: ast.Module, required_paths: set[str]) -> str:
+ roots = {_output_path_root(path) for path in required_paths if _output_path_root(path)}
+ if roots != {"output"}:
+ return ""
+ child_names = sorted(
+ {
+ child
+ for path in required_paths
+ if (child := _output_path_direct_child(path, "output")) and _return_scaffold_name_is_safe(child)
+ }
+ )
+ if len(child_names) != 1:
+ return ""
+ child_name = child_names[0]
+ returns = [node for node in tree.body if isinstance(node, ast.Return)]
+ if len(returns) != 1:
+ return ""
+ return_node = returns[0]
+ if not isinstance(return_node.value, ast.Name) or return_node.value.id != child_name:
+ return ""
+ if return_node.end_lineno is None:
+ return ""
+ lines = code.splitlines()
+ if return_node.lineno < 1 or return_node.lineno > len(lines):
+ return ""
+ indent_match = re.match(r"\s*", lines[return_node.lineno - 1])
+ if indent_match is None:
+ return ""
+ indent = indent_match.group(0)
+ replacement = f'{indent}return {{"output": {{"{child_name}": {child_name}}}}}'
+ return "\n".join(
+ [
+ *lines[: return_node.lineno - 1],
+ replacement,
+ *lines[return_node.end_lineno :],
+ ]
+ )
def _apply_literal_dict_key_assignment(
dict_assignments: dict[str, set[str]],
- dynamic_dict_assignment_names: set[str],
- node: ast.Assign | ast.AnnAssign | ast.AugAssign,
+ dynamic_dict_assignment_names_or_node: set[str] | ast.Assign | ast.AnnAssign | ast.AugAssign,
+ node: ast.Assign | ast.AnnAssign | ast.AugAssign | None = None,
) -> None:
+ dynamic_dict_assignment_names: set[str] | None
+ if node is None:
+ dynamic_dict_assignment_names = None
+ node = cast(ast.Assign | ast.AnnAssign | ast.AugAssign, dynamic_dict_assignment_names_or_node)
+ else:
+ dynamic_dict_assignment_names = cast(set[str], dynamic_dict_assignment_names_or_node)
targets: list[ast.expr] = []
if isinstance(node, ast.Assign):
targets = list(node.targets)
@@ -1612,7 +3198,7 @@ def _apply_literal_dict_key_assignment(
assignment = _literal_dict_key_assignment(target)
target_name = _dict_subscript_target_name(target)
if assignment is None:
- if target_name in dict_assignments:
+ if dynamic_dict_assignment_names is not None and target_name in dict_assignments:
dynamic_dict_assignment_names.add(target_name)
continue
name, key = assignment
@@ -1660,6 +3246,20 @@ def _return_output_roots(
return _ProducedOutputRoots(set(), True)
+def _return_output_paths(
+ node: ast.expr,
+ dict_assignments: Mapping[str, set[str]],
+ helper_return_paths: Mapping[str, set[str]],
+) -> set[str]:
+ if isinstance(node, ast.Await):
+ return _return_output_paths(node.value, dict_assignments, helper_return_paths)
+ if isinstance(node, ast.Name):
+ return set(dict_assignments.get(node.id, set()))
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
+ return set(helper_return_paths.get(node.func.id, set()))
+ return _dict_literal_string_key_paths(node, dict_assignments)
+
+
def _list_literal_output_roots(node: ast.List) -> _ProducedOutputRoots:
if not node.elts:
return _ProducedOutputRoots(set(), False)
@@ -1724,6 +3324,39 @@ def _helper_function_literal_return_roots(statements: list[ast.stmt]) -> dict[st
return helpers
+def _helper_function_literal_return_paths(statements: list[ast.stmt]) -> dict[str, set[str]]:
+ helpers: dict[str, set[str]] = {}
+ for statement in statements:
+ if not isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ dict_assignments: dict[str, set[str]] = {}
+ paths: set[str] = set()
+ for helper_statement in _iter_top_level_scope(statement.body):
+ if (
+ isinstance(helper_statement, ast.Assign)
+ and len(helper_statement.targets) == 1
+ and isinstance(helper_statement.targets[0], ast.Name)
+ ):
+ dict_paths = _dict_literal_string_key_paths(helper_statement.value, dict_assignments)
+ if isinstance(helper_statement.value, ast.Dict):
+ dict_assignments[helper_statement.targets[0].id] = dict_paths
+ elif isinstance(helper_statement, ast.Assign):
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.AnnAssign):
+ if isinstance(helper_statement.target, ast.Name) and isinstance(helper_statement.value, ast.Dict):
+ dict_assignments[helper_statement.target.id] = _dict_literal_string_key_paths(
+ helper_statement.value, dict_assignments
+ )
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.AugAssign):
+ _apply_literal_dict_key_assignment(dict_assignments, helper_statement)
+ elif isinstance(helper_statement, ast.Return) and helper_statement.value is not None:
+ paths.update(_return_output_paths(helper_statement.value, dict_assignments, {}))
+ if paths:
+ helpers[statement.name] = paths
+ return helpers
+
+
def _dict_literal_output_roots(node: ast.Dict) -> _ProducedOutputRoots:
roots: set[str] = set()
abstained = False
@@ -1745,6 +3378,38 @@ def _dict_literal_string_key_roots(node: ast.expr) -> set[str]:
return roots
+def _dict_literal_string_key_paths(
+ node: ast.expr,
+ dict_assignments: Mapping[str, set[str]],
+ *,
+ prefix: str = "",
+) -> set[str]:
+ if isinstance(node, ast.List):
+ array_prefix = f"{prefix}[]" if prefix else "[]"
+ array_paths: set[str] = set()
+ for item in node.elts:
+ array_paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
+ return array_paths
+ if not isinstance(node, ast.Dict):
+ return set()
+ paths: set[str] = set()
+ for key_node, value_node in zip(node.keys, node.values):
+ if not (isinstance(key_node, ast.Constant) and isinstance(key_node.value, str) and key_node.value.strip()):
+ continue
+ path = f"{prefix}.{key_node.value.strip()}" if prefix else key_node.value.strip()
+ paths.add(path)
+ if isinstance(value_node, ast.Dict):
+ paths.update(_dict_literal_string_key_paths(value_node, dict_assignments, prefix=path))
+ elif isinstance(value_node, ast.List):
+ array_prefix = f"{path}[]"
+ for item in value_node.elts:
+ paths.update(_dict_literal_string_key_paths(item, dict_assignments, prefix=array_prefix))
+ elif isinstance(value_node, ast.Name):
+ for child_path in dict_assignments.get(value_node.id, set()):
+ paths.add(f"{path}{child_path}" if child_path.startswith("[]") else f"{path}.{child_path}")
+ return paths
+
+
def _output_empty_code_block_labels(workflow_yaml: str, code_artifact_metadata: object) -> list[str]:
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
if not code_blocks:
@@ -2161,6 +3826,9 @@ def _code_block_parameter_contract_error(workflow_yaml: str) -> str | None:
def _code_repair_progress_data(
repair_context: CodeAuthoringRepairContext | None = None,
+ *,
+ missing_requested_output_facts: list[dict[str, object]] | None = None,
+ metadata_repair_contract: dict[str, object] | None = None,
) -> dict[str, Any]:
"""Tag a code-authoring reject so the streaming adapter renders it as quiet de-duplicated progress."""
data: dict[str, Any] = {
@@ -2169,6 +3837,10 @@ def _code_repair_progress_data(
}
if repair_context is not None:
data["authoring_repair_context"] = repair_context.model_dump(mode="json")
+ if missing_requested_output_facts:
+ data["missing_requested_output_facts"] = missing_requested_output_facts
+ if metadata_repair_contract:
+ data["metadata_repair_contract"] = metadata_repair_contract
return data
@@ -2407,7 +4079,21 @@ def _submitted_code_block_changed(block: Mapping[str, Any], prior_yaml: str | No
def _should_impose_after_update_attempt(ctx: AgentContext) -> bool:
target = ctx.reached_download_target
- return isinstance(target, ReachedDownloadTarget) and not target.already_registered and bool(target.selector.strip())
+ return (
+ (isinstance(target, ReachedDownloadTarget) and not target.already_registered and bool(target.selector.strip()))
+ or synthesized_persistence_reopened_after_failed_run(ctx)
+ or _author_time_reject_reopens_synthesized_imposition(ctx)
+ )
+
+
+def _author_time_reject_reopens_synthesized_imposition(ctx: AgentContext) -> bool:
+ latest = ctx.latest_recorded_build_test_outcome
+ return (
+ isinstance(latest, RecordedBuildTestOutcome)
+ and latest.is_authoritative
+ and latest.phase == "author_time_reject"
+ and latest.reason_code in {"metadata_reject", "synthesized_parameter_binding_ambiguous"}
+ )
def _select_synthesized_imposition_code_block(
@@ -2786,7 +4472,10 @@ def _browser_surface_contains_full_action_spine(submitted_code: str, synthesized
submitted_mutations, _, submitted_ambiguous = _browser_surface_for_code(submitted_code)
if submitted_ambiguous:
return False
- return bool(synthesized_mutations) and submitted_mutations == synthesized_mutations
+ if not synthesized_mutations:
+ return False
+ submitted_iter = iter(submitted_mutations)
+ return all(any(candidate == synthesized for candidate in submitted_iter) for synthesized in synthesized_mutations)
_IDENTITY_QUALIFIER_BOUNDARY = ("[", "#", ".")
@@ -3495,6 +5184,91 @@ def _matching_string_parameter_key_by_default(
return matches[0] if len(matches) == 1 else None
+def _fill_type_argument_parameter_name(node: ast.AST) -> str:
+ if isinstance(node, ast.Name):
+ return node.id
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "str"
+ and len(node.args) == 1
+ and isinstance(node.args[0], ast.Name)
+ ):
+ return node.args[0].id
+ return ""
+
+
+def _selector_join_parameter_alias_by_authored_fill(
+ *,
+ parameters: list[Any],
+ submitted_code: str,
+ synthesized_key: str,
+ selector: str,
+) -> str | None:
+ selector = selector.strip()
+ if not selector:
+ return None
+ available = {
+ str(parameter.get("key") or "").strip()
+ for parameter in parameters
+ if isinstance(parameter, dict)
+ and str(parameter.get("key") or "").strip()
+ and not _is_credential_parameter(parameter)
+ }
+ available.discard(synthesized_key)
+ if not available:
+ return None
+ try:
+ tree = ast.parse(textwrap.dedent(submitted_code).strip() or "pass")
+ except SyntaxError:
+ return None
+ locator_aliases = _locator_alias_selectors(tree)
+ matches: set[str] = set()
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
+ continue
+ if node.func.attr not in {"fill", "type"} or not node.args:
+ continue
+ selectors = _locator_receiver_selectors(node.func.value, locator_aliases)
+ if selectors != {selector}:
+ continue
+ parameter_name = _fill_type_argument_parameter_name(node.args[0])
+ if parameter_name in available:
+ matches.add(parameter_name)
+ return next(iter(matches)) if len(matches) == 1 else None
+
+
+def _synthesized_parameter_binding_reject_count(ctx: AgentContext | None) -> int:
+ if ctx is None:
+ return 0
+ required_paths, _, _ = _output_contract_required_paths_source(ctx)
+ if required_paths:
+ contract_count = _output_contract_reject_count(
+ ctx,
+ _stable_output_contract_key(_output_contract_scope_key(ctx), required_paths),
+ )
+ if contract_count:
+ return contract_count
+ history = getattr(ctx, "recorded_build_test_outcome_history", None)
+ if isinstance(history, list):
+ count = 0
+ for item in history:
+ if (
+ isinstance(item, Mapping)
+ and item.get("reason_code") == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
+ ):
+ count += 1
+ if count:
+ return count
+ latest = getattr(ctx, "latest_recorded_build_test_outcome", None)
+ if (
+ isinstance(latest, RecordedBuildTestOutcome)
+ and latest.reason_code == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
+ ):
+ return 1
+ return 0
+
+
def _drop_parameter_key(parameters: list[Any], key: str) -> None:
parameters[:] = [
parameter
@@ -3503,6 +5277,11 @@ def _drop_parameter_key(parameters: list[Any], key: str) -> None:
]
+def _impose_required_string_parameter(parameters: list[Any], key: str) -> None:
+ _drop_parameter_key(parameters, key)
+ parameters.append(_required_string_parameter_row(key))
+
+
def _coerce_positive_int(value: Any) -> int | None:
try:
parsed = int(value)
@@ -3607,6 +5386,7 @@ def _scout_interaction_for_synthesized_parameter(
def _reconcile_synthesized_parameters(
*,
+ ctx: AgentContext | None = None,
parsed: dict[str, Any],
code_block: dict[str, Any],
submitted_code: str,
@@ -3634,6 +5414,7 @@ def _reconcile_synthesized_parameters(
parameter_keys: list[str] = []
violations: list[str] = []
aliases: dict[str, str] = {}
+ used_selector_join_aliases: set[str] = set()
repair_context: CodeAuthoringRepairContext | None = None
non_credential_synthesized = [param for param in synthesized_parameters if not param.get("credential_id")]
typed_lengths = [
@@ -3641,6 +5422,9 @@ def _reconcile_synthesized_parameters(
for interaction in scout_trajectory
if str(interaction.get("tool_name") or "") == "type_text"
]
+ binding_steering_exhausted = (
+ _synthesized_parameter_binding_reject_count(ctx) >= _MAX_OUTPUT_CONTRACT_STEERING_REJECTS
+ )
def add_binding_violation(key: str, message: str) -> None:
nonlocal repair_context
@@ -3674,6 +5458,27 @@ def _reconcile_synthesized_parameters(
scout_trajectory=scout_trajectory,
synthesized_parameters=synthesized_parameters,
)
+ matched_selector = str(matched_scout.get("selector") or "").strip() if matched_scout is not None else ""
+ selector_join_alias = _selector_join_parameter_alias_by_authored_fill(
+ parameters=parameters,
+ submitted_code=submitted_code,
+ synthesized_key=key,
+ selector=matched_selector,
+ )
+ if selector_join_alias is not None:
+ if selector_join_alias in used_selector_join_aliases:
+ if binding_steering_exhausted:
+ _impose_required_string_parameter(parameters, key)
+ else:
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: authored selector-join alias is reused by another synthesized input.",
+ )
+ continue
+ aliases[key] = selector_join_alias
+ used_selector_join_aliases.add(selector_join_alias)
+ parameter_keys[-1] = selector_join_alias
+ continue
scout_typed_length = (
_coerce_positive_int(matched_scout.get("typed_length")) if matched_scout is not None else None
)
@@ -3707,18 +5512,24 @@ def _reconcile_synthesized_parameters(
_drop_parameter_key(parameters, key)
continue
if _is_credential_parameter(existing):
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed.",
- )
- elif synthesized_default:
- existing_default = _string_parameter_default_value(existing)
- if existing_default != synthesized_default:
+ if binding_steering_exhausted:
+ _impose_required_string_parameter(parameters, key)
+ else:
add_binding_violation(
key,
- f"Unable to bind synthesized parameter `{key}`: "
- "submitted parameter default does not match the scout record.",
+ f"Unable to bind synthesized parameter `{key}`: submitted parameter is credential-typed.",
)
+ elif synthesized_default:
+ existing_default = _string_parameter_default_value(existing)
+ if existing_default != synthesized_default:
+ if binding_steering_exhausted:
+ _impose_required_string_parameter(parameters, key)
+ else:
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: "
+ "submitted parameter default does not match the scout record.",
+ )
continue
if synthesized_default:
@@ -3744,21 +5555,27 @@ def _reconcile_synthesized_parameters(
continue
if len(non_credential_synthesized) != 1:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.",
- )
+ if binding_steering_exhausted:
+ parameters.append(_required_string_parameter_row(key))
+ else:
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: missing submitted workflow parameter and literal binding is ambiguous.",
+ )
continue
typed_length = typed_length or scout_typed_length or (typed_lengths[0] if len(typed_lengths) == 1 else None)
direct_fill_usage = _submitted_direct_fill_type_usage(submitted_code, key)
if direct_fill_usage.matched and direct_fill_usage.mismatched:
- add_binding_violation(
- key,
- f"Unable to bind synthesized parameter `{key}`: submitted code mixes direct fills using `{key}` "
- "with other browser-locator fill/type values. Use the synthesized parameter for every scout-input "
- "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values.",
- )
+ if binding_steering_exhausted:
+ parameters.append(_required_string_parameter_row(key))
+ else:
+ add_binding_violation(
+ key,
+ f"Unable to bind synthesized parameter `{key}`: submitted code mixes direct fills using `{key}` "
+ "with other browser-locator fill/type values. Use the synthesized parameter for every scout-input "
+ "fill in the code block, or declare explicit workflow parameters/defaults for the other filled values.",
+ )
continue
if direct_fill_usage.matched:
parameters.append(_required_string_parameter_row(key))
@@ -3766,7 +5583,10 @@ def _reconcile_synthesized_parameters(
literal, error = _safe_singleton_literal_for_parameter(submitted_code, key, typed_length)
if error:
- add_binding_violation(key, error)
+ if binding_steering_exhausted:
+ parameters.append(_required_string_parameter_row(key))
+ else:
+ add_binding_violation(key, error)
continue
parameters.append(
{
@@ -3783,6 +5603,93 @@ def _reconcile_synthesized_parameters(
return _SynthesizedParameterReconciliation(parameter_keys, violations, aliases, repair_context)
+def _synthesized_durable_stage_codes(synthesized: SynthesizedCodeBlock, *, source_code: str | None = None) -> list[str]:
+ steps = getattr(synthesized, "steps", None)
+ if not isinstance(steps, list) or len(steps) < 2:
+ return []
+ lines = textwrap.dedent(source_code if source_code is not None else synthesized.code).strip("\n").splitlines()
+ if not lines:
+ return []
+ ranges: list[tuple[int, int]] = []
+ for step in steps:
+ if not isinstance(step, Mapping):
+ return []
+ start = _coerce_positive_int(step.get("line_start"))
+ end = _coerce_positive_int(step.get("line_end"))
+ if start is None or end is None or end < start or end > len(lines):
+ return []
+ ranges.append((start, end))
+ if ranges != sorted(ranges):
+ return []
+ stage_codes = ["\n".join(lines[start - 1 : end]).strip() for start, end in ranges]
+ return [code for code in stage_codes if code]
+
+
+def _top_level_block_list_for_selected_code_block(
+ parsed: Mapping[str, Any],
+ code_block: Mapping[str, Any],
+) -> tuple[list[Any], int] | None:
+ workflow_definition = parsed.get("workflow_definition")
+ if not isinstance(workflow_definition, dict):
+ return None
+ blocks = workflow_definition.get("blocks")
+ if not isinstance(blocks, list):
+ return None
+ matches = [index for index, block in enumerate(blocks) if block is code_block]
+ if len(matches) != 1:
+ return None
+ return blocks, matches[0]
+
+
+def _browser_stage_label(base_label: str, index: int) -> str:
+ safe_base = re.sub(r"[^0-9A-Za-z_]+", "_", base_label).strip("_") or "browser"
+ return f"{safe_base}_browser_stage_{index}"
+
+
+def _split_selected_output_owner_into_browser_stages(
+ *,
+ parsed: Mapping[str, Any],
+ code_block: dict[str, Any],
+ synthesized: SynthesizedCodeBlock,
+ synthesized_code: str,
+ extraction_code: str,
+ parameter_keys: list[str],
+) -> list[str]:
+ block_position = _top_level_block_list_for_selected_code_block(parsed, code_block)
+ if block_position is None:
+ return ["Unable to impose synthesized code block: selected output block insertion point is ambiguous."]
+ blocks, selected_index = block_position
+ stage_codes = _synthesized_durable_stage_codes(synthesized, source_code=synthesized_code)
+ if len(stage_codes) < 2:
+ return ["Unable to impose synthesized code block: synthesized browser stage boundaries are ambiguous."]
+ output_label = str(code_block.get("label") or "").strip()
+ if not output_label:
+ return ["Unable to impose synthesized code block: output owner block label is missing."]
+ parsed_blocks = _workflow_code_blocks(parsed) if isinstance(parsed, dict) else []
+ existing_labels = {
+ str(block.get("label") or "").strip()
+ for block in parsed_blocks
+ if isinstance(block, Mapping) and block is not code_block
+ }
+ stage_labels = [_browser_stage_label(output_label, index) for index in range(1, len(stage_codes) + 1)]
+ if len(set(stage_labels)) != len(stage_labels) or any(label in existing_labels for label in stage_labels):
+ return ["Unable to impose synthesized code block: synthesized browser stage label would collide."]
+ stage_blocks: list[dict[str, Any]] = []
+ for label, code in zip(stage_labels, stage_codes):
+ stage_block: dict[str, Any] = {
+ "block_type": "code",
+ "label": label,
+ "code": code.rstrip() + "\n",
+ }
+ if parameter_keys:
+ stage_block["parameter_keys"] = list(parameter_keys)
+ stage_blocks.append(stage_block)
+ code_block["code"] = textwrap.dedent(extraction_code).strip() + "\n"
+ code_block.pop("parameter_keys", None)
+ blocks[selected_index : selected_index + 1] = [*stage_blocks, code_block]
+ return []
+
+
def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext) -> _SynthesizedCodeImpositionResult:
if not getattr(ctx, "impose_synthesized_code_block", False):
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
@@ -3880,6 +5787,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
)
parameter_reconciliation = _reconcile_synthesized_parameters(
+ ctx=ctx,
parsed=parsed,
code_block=code_block,
submitted_code=submitted_code,
@@ -3944,13 +5852,6 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
and not append_selected_extraction
else None
)
- imposed_code = textwrap.dedent(submitted_code if preserve_submitted_extraction else synthesized.code).lstrip("\n")
- if extraction_suffix:
- imposed_code = imposed_code.rstrip() + "\n" + extraction_suffix.rstrip() + "\n"
- elif append_selected_extraction:
- imposed_code = imposed_code.rstrip() + "\n" + textwrap.dedent(submitted_code).strip() + "\n"
- imposed_code = _apply_parameter_reconciliation_to_code(imposed_code, parameter_reconciliation)
- code_block["code"] = imposed_code
credential_parameter_keys = [
str(param.get("key") or "") for param in synthesized.parameters if str(param.get("credential_id") or "").strip()
]
@@ -3962,6 +5863,72 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
"selector_provenance": diagnostics.locator_provenance,
"prior_source": prior_source,
}
+ reconciled_extraction_suffix = (
+ _submitted_suffix_after_synthesized_code(reconciled_submitted_code, reconciled_synthesized_code)
+ if preserve_submitted_extraction
+ else ""
+ )
+ split_extraction_code = ""
+ if extraction_suffix:
+ split_extraction_code = extraction_suffix
+ elif append_selected_extraction:
+ split_extraction_code = textwrap.dedent(submitted_code).strip()
+ elif preserve_submitted_extraction and reconciled_extraction_suffix:
+ split_extraction_code = reconciled_extraction_suffix
+ durable_stage_codes = _synthesized_durable_stage_codes(synthesized, source_code=reconciled_synthesized_code)
+ should_split_output_owner = (
+ metadata_declares_goal_values
+ and len(durable_stage_codes) >= 2
+ and (bool(extraction_suffix) or append_selected_extraction or preserve_submitted_extraction)
+ )
+ if should_split_output_owner:
+ if not split_extraction_code:
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=[
+ "Unable to impose synthesized code block: selected output extraction boundary is ambiguous."
+ ],
+ repair_context=repair_context,
+ )
+ split_extraction_code = _apply_parameter_reconciliation_to_code(split_extraction_code, parameter_reconciliation)
+ target_metadata = _metadata_item_for_block_label(raw_metadata, str(code_block.get("label") or ""))
+ required_split_paths = _metadata_item_goal_value_paths(target_metadata)
+ if required_split_paths:
+ split_extraction_code, static_return_violations = _extraction_code_with_required_static_return(
+ split_extraction_code,
+ required_paths=required_split_paths,
+ )
+ if static_return_violations:
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=static_return_violations,
+ repair_context=repair_context,
+ )
+ split_violations = _split_selected_output_owner_into_browser_stages(
+ parsed=parsed,
+ code_block=code_block,
+ synthesized=synthesized,
+ synthesized_code=reconciled_synthesized_code,
+ extraction_code=split_extraction_code,
+ parameter_keys=parameter_reconciliation.parameter_keys,
+ )
+ if split_violations:
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=split_violations,
+ repair_context=repair_context,
+ )
+ substitutions["separated_browser_stage_count"] = len(durable_stage_codes)
+ else:
+ imposed_code = textwrap.dedent(submitted_code if preserve_submitted_extraction else synthesized.code).lstrip(
+ "\n"
+ )
+ if extraction_suffix:
+ imposed_code = imposed_code.rstrip() + "\n" + extraction_suffix.rstrip() + "\n"
+ elif append_selected_extraction:
+ imposed_code = imposed_code.rstrip() + "\n" + textwrap.dedent(submitted_code).strip() + "\n"
+ imposed_code = _apply_parameter_reconciliation_to_code(imposed_code, parameter_reconciliation)
+ code_block["code"] = imposed_code
if parameter_reconciliation.aliases:
substitutions["parameter_aliases"] = parameter_reconciliation.aliases
if extraction_suffix:
@@ -4982,6 +6949,7 @@ async def _update_workflow(
ctx: AgentContext,
*,
allow_missing_credentials: bool | None = None,
+ allow_static_output_uncertainty: bool = False,
) -> dict[str, Any]:
def reject(
*,
@@ -4989,11 +6957,14 @@ async def _update_workflow(
user_facing_summary: str | None = None,
data: dict[str, Any] | None = None,
repair_context: CodeAuthoringRepairContext | None = None,
+ record_repair_context_outcome: bool = True,
) -> dict[str, Any]:
if repair_context is None:
_clear_code_authoring_repair_context(ctx)
- else:
+ elif record_repair_context_outcome:
_set_code_authoring_repair_context(ctx, repair_context)
+ else:
+ ctx.last_code_authoring_repair_context = repair_context
result: dict[str, Any] = {"ok": False, "error": error}
if user_facing_summary is not None:
result["user_facing_summary"] = user_facing_summary
@@ -5021,6 +6992,16 @@ async def _update_workflow(
# Imposition reconciles synthesized aliases/parameters before the persisted YAML contract is checked.
imposition = _maybe_impose_synthesized_code_block(workflow_yaml, ctx)
if imposition.violations:
+ if (
+ imposition.repair_context is not None
+ and imposition.repair_context.reason_code == _SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE
+ ):
+ required_paths, _, _ = _output_contract_required_paths_source(ctx)
+ _record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
+ )
return reject(
error="\n".join(imposition.violations),
user_facing_summary=_compiled_authoring_user_summary(),
@@ -5058,6 +7039,61 @@ async def _update_workflow(
user_facing_summary=_compiled_authoring_user_summary(),
data=_code_repair_progress_data(),
)
+ workflow_yaml, imposed_metadata, envelope_imposed = _impose_output_contract_envelope_after_steering(
+ ctx,
+ workflow_yaml,
+ params.get("code_artifact_metadata"),
+ )
+ if envelope_imposed:
+ params["workflow_yaml"] = workflow_yaml
+ params["code_artifact_metadata"] = imposed_metadata
+ params["raw_code_artifact_metadata"] = imposed_metadata
+ ctx.raw_code_artifact_metadata = imposed_metadata
+ scaffolded_metadata, scaffold_applied = _scaffold_metadata_contract_for_update(
+ ctx,
+ workflow_yaml,
+ params.get("code_artifact_metadata"),
+ )
+ if scaffold_applied:
+ params["code_artifact_metadata"] = scaffolded_metadata
+ params["raw_code_artifact_metadata"] = scaffolded_metadata
+ ctx.raw_code_artifact_metadata = scaffolded_metadata
+ (
+ required_child_output_paths,
+ output_path_coverage_source,
+ output_path_coverage_reason_code,
+ ) = _required_child_output_paths_for_authoring(ctx)
+ output_contract_evaluation = _evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ params.get("code_artifact_metadata"),
+ allow_static_return_advisory=allow_static_output_uncertainty,
+ )
+ output_contract_static_advisory_allowed = (
+ output_contract_evaluation is not None and output_contract_evaluation.can_attempt_run
+ )
+ if (
+ output_contract_evaluation is not None
+ and output_contract_evaluation.has_deficiencies
+ and not output_contract_static_advisory_allowed
+ ):
+ payload = _record_output_contract_reject(
+ ctx,
+ output_contract_evaluation,
+ summary="Submitted workflow does not satisfy the requested output contract.",
+ )
+ reject_result = _output_contract_reject_result(
+ output_contract_evaluation,
+ payload=payload,
+ tool_name="update_workflow",
+ )
+ return reject(
+ error=str(reject_result["error"]),
+ user_facing_summary=str(reject_result.get("user_facing_summary") or _compiled_authoring_user_summary()),
+ data=reject_result.get("data") if isinstance(reject_result.get("data"), dict) else None,
+ repair_context=output_contract_evaluation.repair_context,
+ record_repair_context_outcome=False,
+ )
missing_metadata_error = _missing_code_artifact_metadata_error(
workflow_yaml,
ctx,
@@ -5067,11 +7103,43 @@ async def _update_workflow(
missing_labels = _missing_code_artifact_metadata_labels(
workflow_yaml, ctx, params.get("code_artifact_metadata")
)
+ missing_metadata_output_facts = _missing_requested_output_facts(
+ required_child_output_paths,
+ reason_code=output_path_coverage_reason_code,
+ )
+ if required_child_output_paths:
+ missing_metadata_error = (
+ f"{missing_metadata_error}\nRequired requested output paths: "
+ f"{', '.join(sorted(required_child_output_paths))}"
+ )
+ metadata_repair_context = _metadata_output_repair_context(
+ block_labels=missing_labels,
+ required_paths=required_child_output_paths,
+ coverage_reason_code=output_path_coverage_reason_code,
+ source=output_path_coverage_source,
+ summary=missing_metadata_error,
+ )
+ metadata_repair_contract = _metadata_repair_contract(
+ block_labels=missing_labels,
+ required_paths=required_child_output_paths,
+ source=output_path_coverage_source,
+ reason_code=output_path_coverage_reason_code,
+ )
+ _record_output_contract_family_reject(
+ ctx,
+ required_child_output_paths,
+ reject_family="missing_code_artifact_metadata",
+ )
_record_author_time_reject_outcome(
ctx,
reason_code="metadata_reject",
summary=missing_metadata_error,
- structural_payload=_code_artifact_metadata_reject_payload(
+ structural_payload=_output_contract_author_time_structural_payload(
+ ctx,
+ required_child_output_paths,
+ block_label=missing_labels[0] if len(missing_labels) == 1 else "",
+ )
+ or _code_artifact_metadata_reject_payload(
workflow_yaml=workflow_yaml,
raw_metadata=params.get("code_artifact_metadata"),
offending_labels=[],
@@ -5079,11 +7147,18 @@ async def _update_workflow(
violation_categories=["missing_code_artifact_metadata"],
),
block_labels=missing_labels,
+ missing_requested_output_facts=missing_metadata_output_facts,
)
return reject(
error=missing_metadata_error,
user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
+ data=_code_repair_progress_data(
+ metadata_repair_context,
+ missing_requested_output_facts=missing_metadata_output_facts,
+ metadata_repair_contract=metadata_repair_contract,
+ ),
+ repair_context=metadata_repair_context,
+ record_repair_context_outcome=False,
)
scout_trajectory = getattr(ctx, "scout_trajectory", None)
normalization = _normalize_code_artifact_metadata_detailed(
@@ -5092,16 +7167,31 @@ async def _update_workflow(
impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,
scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,
verified_runtime_output_paths_by_label=_verified_runtime_output_contract_paths_by_label(ctx, workflow_yaml),
+ advisory_declared_output_return_shape_labels=(
+ {output_contract_evaluation.block_label}
+ if output_contract_static_advisory_allowed and output_contract_evaluation is not None
+ else None
+ ),
)
code_artifact_metadata = normalization.normalized
code_artifact_metadata_error = normalization.error
if code_artifact_metadata_error is not None:
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
+ _record_output_contract_family_reject(
+ ctx,
+ required_child_output_paths,
+ reject_family="metadata_normalization",
+ )
_record_author_time_reject_outcome(
ctx,
reason_code="metadata_reject",
summary=code_artifact_metadata_error,
- structural_payload=_code_artifact_metadata_reject_payload(
+ structural_payload=_output_contract_author_time_structural_payload(
+ ctx,
+ required_child_output_paths,
+ block_label=normalization.offending_labels[0] if len(normalization.offending_labels) == 1 else "",
+ )
+ or _code_artifact_metadata_reject_payload(
workflow_yaml=workflow_yaml,
raw_metadata=params.get("code_artifact_metadata"),
offending_labels=normalization.offending_labels,
@@ -5157,6 +7247,8 @@ async def _update_workflow(
ctx.workflow_verification_evidence.code_artifact_metadata = merged_metadata
_apply_code_artifact_requested_output_evidence_sources(ctx, merged_metadata)
params["code_artifact_metadata"] = merged_metadata
+ workflow_yaml = _apply_metadata_contract_schema_to_workflow_yaml(ctx, workflow_yaml, merged_metadata)
+ params["workflow_yaml"] = workflow_yaml
if (
credential_scout_errors
and code_safety_errors
@@ -5340,7 +7432,7 @@ async def _update_workflow(
data=_code_repair_progress_data(),
)
- recorded_output_paths_required = _recorded_outcome_requires_output_candidate(ctx)
+ recorded_output_paths_required = output_path_coverage_source == "recorded_outcome"
recorded_missing_output_paths = (
_recorded_outcome_missing_output_paths(ctx) if recorded_output_paths_required else set()
)
@@ -5479,7 +7571,14 @@ async def _update_workflow(
# prepare_workflow and the runtime parameter-row read consume canonical
# values; terminal handlers roll back on non-auto-accept.
prior_workflow = await _get_prior_workflow(ctx)
- requires_canonical_persist = _workflow_requires_canonical_persist(prior_workflow, workflow)
+ requires_canonical_persist = _workflow_requires_canonical_persist(
+ prior_workflow, workflow
+ ) or _workflow_needs_contract_readback_persist(
+ ctx,
+ prior_workflow,
+ workflow,
+ allow_static_output_uncertainty=allow_static_output_uncertainty,
+ )
if requires_canonical_persist:
created_by_stamp = await resolve_copilot_created_by_stamp(ctx.workflow_id, ctx.organization_id)
await app.WORKFLOW_SERVICE.update_workflow_definition(
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index a08bb55a1..ca1873190 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -646,6 +646,9 @@ def _build_proposed_workflow_data(updated_workflow: Workflow, agent_result: Agen
proposed_data = dict(updated_workflow.model_dump(mode="json"))
if agent_result.workflow_yaml:
proposed_data["_copilot_yaml"] = agent_result.workflow_yaml
+ code_artifact_metadata = getattr(agent_result, "code_artifact_metadata", None)
+ if code_artifact_metadata:
+ proposed_data["_copilot_code_artifact_metadata"] = code_artifact_metadata
if _proposal_disposition(agent_result) == "review_untested":
proposed_data["_copilot_unvalidated"] = True
return proposed_data
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 2741f525a..6d0127a84 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -15,6 +15,7 @@ from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.agent import (
_VERIFIED_WORKFLOW_SUCCESS_REPLY,
_build_built_unverified_exit_result,
@@ -71,7 +72,12 @@ from skyvern.forge.sdk.copilot.request_policy import (
redact_raw_secrets_for_prompt,
)
from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
-from skyvern.forge.sdk.copilot.tools.completion import _completion_verification_criteria
+from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
+from skyvern.forge.sdk.copilot.tools.completion import (
+ _authored_output_contract_criteria,
+ _completion_verification_criteria,
+ _maybe_run_completion_verification,
+)
from skyvern.forge.sdk.copilot.turn_context import TranscriptContext, TurnContextOmission, TurnContextPacket
from skyvern.forge.sdk.copilot.turn_halt import (
CopilotTurnHalt,
@@ -886,6 +892,53 @@ workflow_definition:
assert "adapt the next code block to the observed page state" in prompt
assert "do not re-emit the same failing selector or name path" in prompt
+ def test_metadata_repair_context_prompt_includes_failure_and_contract_guidance(self) -> None:
+ long_reason = "missing requested output child paths " + ("x" * 220)
+ repair_context = CodeAuthoringRepairContext(
+ block_label="lookup_status",
+ reason_code="metadata_reject",
+ runtime_failure_reason=long_reason,
+ runtime_failure_class="requested_output_contract_missing_output_coverage",
+ required_goal_value_paths=["output.record_id", "output.flags"],
+ required_extraction_schema_paths=["output.record_id", "output.flags"],
+ required_code_return_paths=["output.record_id", "output.flags"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ repair_instruction=(
+ "Declare code_artifact_metadata goal_value_paths and extraction_schema for required output paths."
+ ),
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+ standard_ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.STANDARD,
+ last_code_authoring_repair_context=repair_context,
+ )
+
+ prompt = agent_module._code_authoring_repair_context_prompt(ctx)
+
+ assert agent_module._code_authoring_repair_context_prompt(standard_ctx) == ""
+ assert "reason_code: metadata_reject" in prompt
+ assert "block_label: lookup_status" in prompt
+ assert "runtime_failure_reason: missing requested output child paths " in prompt
+ assert "x" * 180 not in prompt
+ assert "runtime_failure_class: requested_output_contract_missing_output_coverage" in prompt
+ assert "metadata_contract_source: requested_output_contract" in prompt
+ assert "metadata_contract_reason_code: requested_output_contract_missing_output_coverage" in prompt
+ assert "required_goal_value_paths: output.record_id, output.flags" in prompt
+ assert "required_extraction_schema_paths: output.record_id, output.flags" in prompt
+ assert "required_code_return_paths: output.record_id, output.flags" in prompt
+ assert "code_artifact_metadata" in prompt
+ assert "goal_value_paths" in prompt
+ assert "valid extraction_schema" in prompt
+ assert "code return paths" in prompt
+ assert "required requested output child paths" in prompt
+ assert "rerun update_and_run_blocks" in prompt
+ assert "Declare code_artifact_metadata goal_value_paths" in prompt
+ assert "Coastal" not in prompt
+
def test_recorded_build_test_outcome_prompt_renders_structural_grounding(self) -> None:
ctx = _ctx(
block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
@@ -1891,9 +1944,219 @@ class TestBlockGoalMainGoal:
class TestRuntimeBlockGoalPersistenceBoundary:
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_metadata_repair_context_preflight_blocks_before_update(
+ self, monkeypatch
+ ) -> None:
+ workflow_yaml = """
+title: Test workflow
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+"""
+ repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id", "output.flags"],
+ required_extraction_schema_paths=["output.record_id", "output.flags"],
+ required_code_return_paths=["output.record_id", "output.flags"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(
+ tools_module,
+ "_tool_loop_error",
+ lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("loop check should not run")),
+ )
+ monkeypatch.setattr(
+ tools_module,
+ "_update_workflow",
+ AsyncMock(side_effect=AssertionError("_update_workflow should not run")),
+ )
+ record_calls: list[tuple[str, dict[str, object]]] = []
+ monkeypatch.setattr(
+ tools_module,
+ "record_tool_step_result_for_ctx",
+ lambda _ctx, tool, _args, result: record_calls.append((tool, result)),
+ )
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["extract_entry_output"]}),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is False
+ assert "cannot attempt a run" in parsed["error"]
+ assert parsed["data"]["reason_code"] == "metadata_contract_required_before_run"
+ assert parsed["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
+ "output.flags",
+ "output.record_id",
+ ]
+ assert parsed["data"]["metadata_repair_contract"]["block_label"] == "extract_entry_output"
+ assert record_calls
+ assert record_calls[0][0] == "update_and_run_blocks"
+ assert record_calls[0][1]["data"]["reason_code"] == "metadata_contract_required_before_run"
+
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_metadata_contract_scaffold_reaches_update_workflow(self, monkeypatch) -> None:
+ workflow_yaml = """
+title: Test workflow
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
+"""
+ repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id", "output.flags"],
+ required_extraction_schema_paths=["output.record_id", "output.flags"],
+ required_code_return_paths=["output.record_id", "output.flags"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+ captured_metadata: list[dict[str, object]] = []
+
+ async def fake_update_workflow(payload, _ctx, **_kwargs):
+ captured_metadata.extend(payload["code_artifact_metadata"])
+ return {"ok": False, "error": "sentinel update reached", "data": {"from_update": True}}
+
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps({"workflow_yaml": workflow_yaml, "block_labels": ["extract_entry_output"]}),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is False
+ assert parsed["error"] == "sentinel update reached"
+ assert captured_metadata
+ assert captured_metadata[0]["block_label"] == "extract_entry_output"
+ assert captured_metadata[0]["artifact_id"] == "code_artifact:extract_entry_output"
+ assert captured_metadata[0]["claimed_outcomes"][0]["goal_value_paths"] == [
+ "output.flags",
+ "output.record_id",
+ ]
+
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_post_steering_static_return_gap_reaches_run(self, monkeypatch) -> None:
+ workflow_yaml = """
+title: Test workflow
+workflow_definition:
+ parameters: []
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return "not a structured output"
+"""
+ required_paths = {"output.record_id"}
+ schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
+ repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ ctx = _ctx(
+ block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ last_code_authoring_repair_context=repair_context,
+ )
+ ctx.turn_id = "metadata-contract-run-preflight"
+ signature = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ required_paths=required_paths,
+ )
+ ctx.output_contract_reject_count_by_signature = {signature: 2}
+ captured: dict[str, object] = {}
+
+ async def fake_update_workflow(payload, update_ctx, **_kwargs):
+ captured["update_called"] = True
+ captured["metadata"] = payload["code_artifact_metadata"]
+ update_ctx.workflow_yaml = payload["workflow_yaml"]
+ update_ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[]))
+ return {"ok": True, "data": {"block_count": 1}}
+
+ async def fake_run_blocks(params, _ctx, **_kwargs):
+ captured["run_called"] = True
+ captured["run_params"] = params
+ return {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr-1",
+ "overall_status": "completed",
+ "blocks": [{"label": "extract_entry_output", "status": "completed"}],
+ },
+ }
+
+ monkeypatch.setattr(tools_module, "_request_policy_allows_update_and_skip_run", lambda *args: False)
+ monkeypatch.setattr(tools_module, "_authority_tool_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_tool_loop_error", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "_update_and_run_blocks_composition_evidence_precheck", lambda *args: None)
+ monkeypatch.setattr(tools_module, "_get_prior_workflow_definition", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_update_workflow", fake_update_workflow)
+ monkeypatch.setattr(
+ tools_module, "_plan_frontier", lambda *args: (["extract_entry_output"], {}, "extract_entry_output")
+ )
+ monkeypatch.setattr(tools_module, "_frontier_run_size_error", lambda *args: None)
+ monkeypatch.setattr(tools_module, "_run_blocks_and_collect_debug", fake_run_blocks)
+ monkeypatch.setattr(tools_module, "_verify_and_record_run_blocks_result", AsyncMock(return_value=None))
+ monkeypatch.setattr(tools_module, "_record_diagnosis_repair_contract", lambda *args, **kwargs: None)
+ monkeypatch.setattr(tools_module, "enqueue_screenshot_from_result", lambda *args, **kwargs: None)
+
+ result = await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ SimpleNamespace(context=ctx, tool_name="update_and_run_blocks"),
+ json.dumps(
+ {
+ "workflow_yaml": workflow_yaml,
+ "block_labels": ["extract_entry_output"],
+ }
+ ),
+ )
+
+ parsed = json.loads(result)
+ assert parsed["ok"] is True
+ assert captured["update_called"] is True
+ assert captured["run_called"] is True
+ update_metadata = captured["metadata"]
+ assert isinstance(update_metadata, list)
+ assert update_metadata[0]["block_label"] == "extract_entry_output"
+ assert update_metadata[0]["claimed_outcomes"][0]["goal_value_paths"] == ["output.record_id"]
+ assert update_metadata[0]["claimed_outcomes"][0]["extraction_schema"] == schema
+
@pytest.mark.asyncio
async def test_update_and_run_blocks_persists_clean_yaml(self, monkeypatch) -> None:
- from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
clean_yaml = """
@@ -1907,7 +2170,9 @@ workflow_definition:
"""
captured: dict[str, str | bool] = {}
- async def fake_update_workflow(payload, ctx, allow_missing_credentials=False):
+ async def fake_update_workflow(
+ payload, ctx, allow_missing_credentials=False, allow_static_output_uncertainty=False
+ ):
captured["workflow_yaml"] = payload["workflow_yaml"]
ctx.workflow_yaml = payload["workflow_yaml"]
workflow = await _process_workflow_yaml(
@@ -5190,7 +5455,6 @@ class TestRunBlocksCredentialApproval:
@pytest.mark.asyncio
async def test_update_and_run_blocks_rejects_unapproved_credential_at_shared_run_seam(self, monkeypatch) -> None:
- from skyvern.forge.sdk.copilot import tools as tools_module
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
workflow = self._workflow("cred_unapproved")
@@ -6172,3 +6436,331 @@ class TestDeclaredEqualsGradedCompletionCriteria:
graded = policy.graded_completion_criteria()
assert all(not criterion.method_mandated for criterion in graded)
assert {criterion.id for criterion in graded} == {"c2", "c3", "c4", "c5"}
+
+ @pytest.mark.asyncio
+ async def test_fallback_floor_uses_authored_output_contract_paths_for_completion(self) -> None:
+ label = "validate_public_path"
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ code_artifact_metadata={
+ label: {
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": [
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ]
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "goal_value_paths": [
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ]
+ }
+ ],
+ }
+ },
+ workflow_yaml=(
+ "title: Utility path\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ f" label: {label}\n"
+ " code: |\n"
+ " return {}\n"
+ ),
+ last_workflow_yaml=None,
+ completion_verification_result=None,
+ copilot_total_timeout_exceeded=False,
+ reached_download_target=None,
+ workflow_verification_evidence=SimpleNamespace(block_verified=[]),
+ verified_prefix_labels=[],
+ verified_block_outputs={},
+ post_run_page_observation_after_failed_test=False,
+ completion_criteria_turn_state=None,
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ ]
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_public_path",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": label,
+ "status": "completed",
+ "extracted_data": {
+ "public_form_exists": True,
+ "visible_page_path_label": "sign in",
+ "recommended_next_action": "authenticate",
+ "evidence_text": "diagnostic only",
+ },
+ }
+ ],
+ "executed_block_labels": [label],
+ },
+ },
+ 0,
+ )
+
+ assert verification is not None
+ assert "__copilot_fallback_floor__run" not in verification.criterion_ids
+ assert {verdict.output_path for verdict in verification.verdicts} == {
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ }
+ assert not any(verdict.satisfied for verdict in verification.verdicts)
+ assert all(verdict.reason_code != "evidence_confirms" for verdict in verification.verdicts)
+
+ def test_fallback_floor_uses_repair_context_output_contract_paths_when_metadata_missing(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ code_artifact_metadata={},
+ workflow_verification_evidence=SimpleNamespace(code_artifact_metadata={}),
+ last_code_authoring_repair_context=CodeAuthoringRepairContext(
+ block_label="validate_public_path",
+ reason_code="metadata_reject",
+ required_goal_value_paths=[
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ],
+ ),
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ ]
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ def test_fallback_floor_prefers_repair_context_paths_over_stale_metadata(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ code_artifact_metadata={
+ "stale_output": {
+ "claimed_outcomes": [{"goal_value_paths": ["output.old_path"]}],
+ }
+ },
+ workflow_verification_evidence=SimpleNamespace(code_artifact_metadata={}),
+ last_code_authoring_repair_context=CodeAuthoringRepairContext(
+ block_label="validate_public_path",
+ reason_code="metadata_reject",
+ required_goal_value_paths=[
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ],
+ ),
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ ]
+ assert "output.old_path" not in {criterion.output_path for criterion in criteria}
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ def test_staged_contract_uses_durable_metadata_before_repair_context(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ has_staged_proposal=True,
+ staged_workflow=object(),
+ code_artifact_metadata={
+ "stale_output": {
+ "claimed_outcomes": [{"goal_value_paths": ["output.old_path"]}],
+ }
+ },
+ workflow_verification_evidence=SimpleNamespace(
+ code_artifact_metadata={
+ "validate_public_path": {
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": [
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ]
+ }
+ ]
+ }
+ }
+ ),
+ last_code_authoring_repair_context=CodeAuthoringRepairContext(
+ block_label="validate_public_path",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.repair_context_only"],
+ ),
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ ]
+ assert "output.old_path" not in {criterion.output_path for criterion in criteria}
+ assert "output.repair_context_only" not in {criterion.output_path for criterion in criteria}
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ def test_staged_contract_uses_ctx_metadata_when_evidence_metadata_empty(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ has_staged_proposal=True,
+ staged_workflow=object(),
+ workflow_verification_evidence=SimpleNamespace(code_artifact_metadata={}),
+ code_artifact_metadata={
+ "validate_public_path": {
+ "claimed_outcomes": [{"goal_value_paths": ["output.public_form_exists"]}],
+ }
+ },
+ last_code_authoring_repair_context=None,
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+
+ assert [(criterion.id, criterion.output_path) for criterion in criteria] == [
+ ("__copilot_authored_output__output_public_form_exists", "output.public_form_exists")
+ ]
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ def test_staged_contract_canonicalizes_block_local_metadata_paths(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ has_staged_proposal=True,
+ staged_workflow=object(),
+ workflow_verification_evidence=SimpleNamespace(
+ code_artifact_metadata={
+ "validate_public_path": {
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": [
+ "public_form_exists",
+ "visible_page_path_label",
+ "recommended_next_action",
+ ]
+ }
+ ]
+ }
+ }
+ ),
+ code_artifact_metadata={},
+ last_code_authoring_repair_context=None,
+ )
+
+ criteria = _authored_output_contract_criteria(ctx)
+
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ ]
+
+ @pytest.mark.asyncio
+ async def test_staged_contract_missing_durable_metadata_fails_closed_without_fallback_floor(self) -> None:
+ ctx = SimpleNamespace(
+ request_policy=RequestPolicy(
+ completion_criteria=build_classifier_fallback_floor([]),
+ classifier_status="fallback",
+ ),
+ has_staged_proposal=True,
+ staged_workflow=object(),
+ code_artifact_metadata={},
+ workflow_verification_evidence=SimpleNamespace(code_artifact_metadata={}),
+ last_code_authoring_repair_context=CodeAuthoringRepairContext(
+ block_label="validate_public_path",
+ reason_code="metadata_reject",
+ required_goal_value_paths=[
+ "output.public_form_exists",
+ "output.visible_page_path_label",
+ "output.recommended_next_action",
+ ],
+ ),
+ workflow_yaml="title: Utility path\nworkflow_definition:\n blocks: []\n",
+ last_workflow_yaml=None,
+ completion_verification_result=None,
+ copilot_total_timeout_exceeded=False,
+ reached_download_target=None,
+ verified_prefix_labels=[],
+ verified_block_outputs={},
+ post_run_page_observation_after_failed_test=False,
+ completion_criteria_turn_state=None,
+ )
+
+ criteria = _completion_verification_criteria(ctx)
+
+ assert [criterion.id for criterion in criteria] == ["__copilot_authored_output_contract_missing"]
+ assert [criterion.output_path for criterion in criteria] == [
+ "output.__copilot_missing_authored_output_contract__"
+ ]
+ assert not any(is_fallback_floor_criterion(criterion) for criterion in criteria)
+
+ verification = await _maybe_run_completion_verification(
+ ctx,
+ {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_missing_contract",
+ "overall_status": "completed",
+ "blocks": [
+ {
+ "label": "validate_public_path",
+ "status": "completed",
+ "extracted_data": {
+ "public_form_exists": True,
+ "visible_page_path_label": "sign in",
+ "recommended_next_action": "authenticate",
+ "evidence_text": "diagnostic only",
+ },
+ }
+ ],
+ "executed_block_labels": ["validate_public_path"],
+ },
+ },
+ 0,
+ )
+
+ assert verification is not None
+ assert "__copilot_fallback_floor__run" not in verification.criterion_ids
+ assert verification.criterion_ids == ["__copilot_authored_output_contract_missing"]
+ assert not verification.is_fully_satisfied()
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index a7bf25aa4..ae1e9e4bc 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -6,8 +6,10 @@ OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.
from __future__ import annotations
import ast
+import json
import textwrap
from types import SimpleNamespace
+from unittest.mock import AsyncMock
import pytest
import yaml
@@ -19,8 +21,12 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
assert_clean_user_facing_text,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ BuildTestOutcomeReasonCode,
RecordedBuildTestOutcome,
authored_structure_signature_from_workflow,
+ latest_recorded_build_test_outcome_repeated,
+ recorded_outcome_from_author_time_reject,
+ recorded_outcome_from_run_blocks_result,
)
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
@@ -41,7 +47,7 @@ from skyvern.forge.sdk.copilot.output_contracts import code_block_available_bind
from skyvern.forge.sdk.copilot.output_utils import sanitize_tool_result_for_llm
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
-from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE
+from skyvern.forge.sdk.copilot.run_outcome import TERMINAL_CHALLENGE_BLOCKER_REASON_CODE, RecordedRunOutcome
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tools import (
_code_block_safety_errors,
@@ -1426,7 +1432,8 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
@pytest.mark.asyncio
async def test_authoritative_outcome_not_demonstrated_rejects_candidate_missing_requested_output_roots(
@@ -1528,7 +1535,9 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
+ assert ctx.workflow_yaml is not None
+ assert "confirmation_number" in ctx.workflow_yaml
+ assert "extraction_schema" in ctx.workflow_yaml
assert ctx.latest_recorded_build_test_outcome is not None
assert ctx.latest_recorded_build_test_outcome.reason_code == "outcome_not_demonstrated"
@@ -1578,8 +1587,8 @@ class TestCodeRepairProgressClassification:
assert result["ok"] is False
assert "output.confirmation_number" in result["error"]
- assert "Declare those exact output_path values" in result["error"]
- assert "output_root is diagnostic only" in result["error"]
+ assert "satisfies the requested output contract" in result["error"]
+ assert "canonical required child paths" in result["error"]
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert outcome.reason_code == "metadata_reject"
@@ -1725,7 +1734,8 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is True
- assert ctx.workflow_yaml == candidate_yaml
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert outcome.reason_code == "outcome_not_demonstrated"
@@ -2397,12 +2407,2571 @@ class TestCodeRepairProgressClassification:
assert outcome.missing_requested_output_facts == []
@pytest.mark.asyncio
- async def test_metadata_reject_persists_missing_roots_for_next_author_context(
+ async def test_metadata_reject_persists_missing_roots_for_next_author_context(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_provider_and_extract_credentials"
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "value_status": "no_typed_value",
+ },
+ ],
+ )
+ partial_yaml = _yaml(
+ f"""
+ title: Provider lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.locator("#locInput").wait_for(state="visible", timeout=15000)
+ return {{"search_options": []}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "address",
+ "output_root": "address",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "credentialing_status",
+ "output_root": "credentialing_status",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ rendered = agent_module._recorded_build_test_outcome_prompt(ctx)
+ assert "missing_requested_output_facts:" in rendered
+ assert "output_root=address" in rendered
+ assert "output_root=credentialing_status" in rendered
+ assert "value_status=no_typed_value" in rendered
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_broad_output_for_missing_child_paths(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["output"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.verified_block_outputs[label] = {
+ "output": {"npi": "1234567890", "address": "Example location", "statuses": ["active"]}
+ }
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
+ {"output_path": "output.address", "output_root": "output", "value_status": "no_typed_value"},
+ {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"output": {{"summary": "found"}}}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "requested output contract" in result["error"]
+ assert result["data"]["canonical_required_child_paths"] == [
+ "output.address",
+ "output.npi",
+ "output.statuses",
+ ]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "output.address",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.statuses",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_allows_exact_missing_array_child_paths(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ child_paths = ["output.npi", "output.locations[].address", "output.locations[].status", "output.statuses"]
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"locations":{"type":"array","items":{"type":"object","properties":{'
+ '"address":{"type":"string"},"status":{"type":"string"}}}},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
+ {
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.locations[].status",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{
+ "output": {{
+ "npi": "1234567890",
+ "locations": [{{"address": "Example location", "status": "active"}}],
+ "statuses": ["active"],
+ }}
+ }}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
+
+ @pytest.mark.asyncio
+ async def test_authoritative_outcome_not_demonstrated_rejects_extraction_schema_wrapper_mismatch(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ child_paths = ["output.npi", "output.locations[].address", "output.statuses"]
+ claimed_schema = (
+ '{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}'
+ )
+ terminal_schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": claimed_schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": child_paths, "extraction_schema": terminal_schema}
+ ],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:unsatisfied-output",
+ authored_structure_signature="authored:previous-failed-candidate",
+ missing_requested_output_facts=[
+ {"output_path": "output.npi", "output_root": "output", "value_status": "no_typed_value"},
+ {
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "value_status": "no_typed_value",
+ },
+ {"output_path": "output.statuses", "output_root": "output", "value_status": "no_typed_value"},
+ ],
+ )
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{
+ "output": {{
+ "npi": "1234567890",
+ "locations": [{{"address": "Example location"}}],
+ "statuses": ["active"],
+ }}
+ }}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "extraction_schema" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.statuses",
+ "output_root": "output",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "metadata_reject"
+ assert repair_context["block_label"] == label
+ assert repair_context["runtime_failure_class"] == "recorded_outcome_missing_output_coverage"
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+
+ @pytest.mark.asyncio
+ async def test_requested_output_contract_rejects_collapsed_candidate_before_recorded_outcome(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ SimpleNamespace(
+ id="requested_locations",
+ output_path="output.locations",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ SimpleNamespace(
+ id="requested_statuses",
+ output_path="output.statuses",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": ["output"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{"output": {{"summary": "found"}}}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "requested output contract" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "output.locations",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.statuses",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "metadata_reject"
+ assert repair_context["block_label"] == label
+ assert repair_context["runtime_failure_class"] == "requested_output_contract_missing_output_coverage"
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+
+ def test_single_output_contract_evaluator_reports_all_deficiency_classes(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: value
+ blocks:
+ - block_type: code
+ label: extract_record
+ parameter_keys:
+ - value
+ code: |
+ _scout_entry_target = page.locator("#filter")
+ try:
+ await _scout_entry_target.wait_for(state="visible", timeout=1000)
+ except Exception:
+ await page.goto("https://example.com/records", wait_until="domcontentloaded")
+ await _scout_entry_target.wait_for(state="visible")
+ await page.locator("#filter").fill(str(value))
+ await page.locator("#choose").click()
+ await page.wait_for_load_state("domcontentloaded")
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is True
+ assert evaluation.block_label == "extract_record"
+ assert evaluation.missing_metadata_paths == ["output.record_id"]
+ assert evaluation.missing_schema_paths == ["output.record_id"]
+ assert evaluation.missing_return_paths == ["output.record_id"]
+ assert evaluation.shape_violations == ["separated_spine_shape_required"]
+ assert evaluation.payload["satisfying_templates"]["code_artifact_metadata"]["block_label"] == "extract_record"
+ assert evaluation.payload["satisfying_templates"]["return_skeleton"] == (
+ 'return {"output": {"record_id": record_id}}'
+ )
+
+ def test_schema_template_derives_nested_array_paths_without_semantic_values(self) -> None:
+ schema = workflow_update_module._schema_template_for_required_paths(
+ {"output.npi", "output.locations[].address", "output.locations[].status"}
+ )
+
+ output = schema["properties"]["output"]
+ assert output["type"] == "object"
+ assert output["required"] == ["locations", "npi"]
+ locations = output["properties"]["locations"]
+ assert locations["type"] == "array"
+ assert locations["items"]["required"] == ["address", "status"]
+ assert locations["items"]["properties"]["address"] == {}
+ assert locations["items"]["properties"]["status"] == {}
+ assert "description" not in json.dumps(schema)
+ assert "example" not in json.dumps(schema)
+
+ @pytest.mark.asyncio
+ async def test_update_and_run_preflight_and_update_workflow_share_contract_signature(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+ repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ update_ctx = _code_only_ctx()
+ update_ctx.turn_id = "shared-contract-turn"
+ update_ctx.last_code_authoring_repair_context = repair_context
+ run_ctx = _code_only_ctx()
+ run_ctx.turn_id = "shared-contract-turn"
+ run_ctx.last_code_authoring_repair_context = repair_context
+
+ update_result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml}, update_ctx, allow_missing_credentials=True
+ )
+ run_result = workflow_update_module._metadata_contract_run_preflight_reject(run_ctx, workflow_yaml, [])
+
+ assert run_result is not None
+ assert update_result["ok"] is False
+ assert run_result["ok"] is False
+ assert (
+ update_result["data"]["canonical_output_contract_signature"]
+ == (run_result["data"]["canonical_output_contract_signature"])
+ )
+ assert (
+ update_result["data"]["canonical_required_child_paths"]
+ == (run_result["data"]["canonical_required_child_paths"])
+ )
+ assert update_result["data"]["satisfying_templates"] == run_result["data"]["satisfying_templates"]
+
+ def test_static_return_uncertainty_is_run_preflight_advisory_only(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ schema = workflow_update_module._schema_template_text_for_required_paths({"output.record_id"})
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return build_output()
+ """
+ )
+
+ update_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=False
+ )
+ run_eval = workflow_update_module._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, allow_static_return_advisory=True
+ )
+
+ assert update_eval is not None
+ assert update_eval.missing_return_paths == ["output.record_id"]
+ assert update_eval.can_attempt_run is False
+ assert run_eval is not None
+ assert run_eval.missing_return_paths == []
+ assert run_eval.payload["static_return_advisory_paths"] == ["output.record_id"]
+ assert run_eval.can_attempt_run is True
+
+ @pytest.mark.asyncio
+ async def test_save_only_update_rejects_post_steering_static_return_gap(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return "not a structured output"
+ """
+ )
+ ctx.turn_id = "save-only-static-return-gap"
+ signature = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ required_paths={"output.record_id"},
+ )
+ ctx.output_contract_reject_count_by_signature = {signature: 2}
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "output_contract_required"
+ assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
+ assert result["data"]["static_return_advisory_paths"] == []
+
+ @pytest.mark.asyncio
+ async def test_run_path_allows_post_steering_declared_output_return_shape_gap(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return "not a structured output"
+ """
+ )
+ ctx.turn_id = "run-static-return-gap"
+ signature = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=workflow_yaml,
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ required_paths={"output.record_id"},
+ )
+ ctx.output_contract_reject_count_by_signature = {signature: 2}
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
+ "output.record_id"
+ ]
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["extraction_schema"]
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ block = _single_code_block(parsed)
+ schema = json.loads(block["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["record_id"] == {}
+
+ @pytest.mark.asyncio
+ async def test_run_path_persists_effective_output_contract_for_readback(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ required_paths = {"output.record_id"}
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ schema = workflow_update_module._schema_template_text_for_required_paths(required_paths)
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+
+ def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
+ return SimpleNamespace(
+ title="Entry lookup",
+ description=None,
+ workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
+ proxy_location=None,
+ webhook_callback_url=None,
+ totp_verification_url=None,
+ totp_identifier=None,
+ persist_browser_session=False,
+ browser_profile_id=None,
+ browser_profile_key=None,
+ model=None,
+ max_screenshot_scrolls=None,
+ extra_http_headers=None,
+ cdp_connect_headers=None,
+ run_with=None,
+ ai_fallback=None,
+ cache_key=None,
+ adaptive_caching=None,
+ enable_self_healing=None,
+ code_version=None,
+ run_sequentially=False,
+ sequential_key=None,
+ )
+
+ persisted: dict[str, object] = {}
+
+ async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
+ return fake_workflow(blocks=[])
+
+ async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return fake_workflow(blocks=[SimpleNamespace(label="extract_entry_output")])
+
+ async def fake_update_workflow_definition(**kwargs: object) -> None:
+ persisted.update(kwargs)
+
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
+ monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(
+ workflow_update_module.app.WORKFLOW_SERVICE,
+ "update_workflow_definition",
+ fake_update_workflow_definition,
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "resolve_copilot_created_by_stamp",
+ AsyncMock(return_value="copilot"),
+ )
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert persisted["workflow_id"] == "w"
+ definition = persisted["workflow_definition"]
+ assert definition.blocks == [SimpleNamespace(label="extract_entry_output")]
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ block = _single_code_block(parsed)
+ assert json.loads(block["extraction_schema"])["properties"]["output"]["properties"]["record_id"] == {}
+
+ @pytest.mark.asyncio
+ async def test_run_path_persists_blocks_for_api_readback_when_metadata_missing(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ ctx = _code_only_ctx()
+ workflow_yaml = _yaml(
+ """
+ title: Public path validation
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: validate_public_path
+ code: |
+ return {"public_form_exists": False}
+ """
+ )
+
+ def fake_workflow(*, blocks: list[object]) -> SimpleNamespace:
+ return SimpleNamespace(
+ title="Public path validation",
+ description=None,
+ workflow_definition=SimpleNamespace(blocks=blocks, parameters=[]),
+ proxy_location=None,
+ webhook_callback_url=None,
+ totp_verification_url=None,
+ totp_identifier=None,
+ persist_browser_session=False,
+ browser_profile_id=None,
+ browser_profile_key=None,
+ model=None,
+ max_screenshot_scrolls=None,
+ extra_http_headers=None,
+ cdp_connect_headers=None,
+ run_with=None,
+ ai_fallback=None,
+ cache_key=None,
+ adaptive_caching=None,
+ enable_self_healing=None,
+ code_version=None,
+ run_sequentially=False,
+ sequential_key=None,
+ )
+
+ persisted: dict[str, object] = {}
+
+ async def fake_get_prior_workflow(_ctx: CopilotContext) -> SimpleNamespace:
+ return fake_workflow(blocks=[])
+
+ async def fake_process_workflow_yaml(**_kwargs: object) -> SimpleNamespace:
+ return fake_workflow(blocks=[SimpleNamespace(label="validate_public_path")])
+
+ async def fake_update_workflow_definition(**kwargs: object) -> None:
+ persisted.update(kwargs)
+
+ monkeypatch.setattr(workflow_update_module, "_get_prior_workflow", fake_get_prior_workflow)
+ monkeypatch.setattr(workflow_update_module, "_process_workflow_yaml", fake_process_workflow_yaml)
+ monkeypatch.setattr(workflow_update_module, "composition_page_evidence_error", lambda *_args, **_kwargs: None)
+ monkeypatch.setattr(
+ workflow_update_module.app.WORKFLOW_SERVICE,
+ "update_workflow_definition",
+ fake_update_workflow_definition,
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "resolve_copilot_created_by_stamp",
+ AsyncMock(return_value="copilot"),
+ )
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
+ )
+
+ assert result["ok"] is True
+ assert persisted["workflow_id"] == "w"
+ assert persisted["workflow_definition"].blocks == [SimpleNamespace(label="validate_public_path")]
+
+ @pytest.mark.asyncio
+ async def test_output_contract_reject_budget_counts_same_signature_across_classes(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ result = None
+ for _ in range(4):
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result is not None
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "output_contract_reject_budget_exhausted"
+ assert result["data"]["output_contract_reject_count"] == 4
+ assert result["data"]["canonical_required_child_paths"] == ["output.record_id"]
+
+ def test_output_contract_signature_uses_stable_scope_and_required_path_identity(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "contract-scope-a"
+ first = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=_yaml(
+ """
+ title: First draft title
+ workflow_definition:
+ blocks: []
+ """
+ ),
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ required_paths={"output.locations[].address", "output.statuses"},
+ )
+ second = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml=_yaml(
+ """
+ title: Changed draft title
+ workflow_definition:
+ blocks: []
+ """
+ ),
+ source="runtime_output_repair",
+ reason_code="runtime_output_repair_required",
+ required_paths={"output.statuses", "output.locations[].address"},
+ )
+ other_ctx = _code_only_ctx()
+ other_ctx.turn_id = "contract-scope-b"
+ other_scope = workflow_update_module._output_contract_signature(
+ ctx=other_ctx,
+ workflow_yaml=_yaml(
+ """
+ title: Changed draft title
+ workflow_definition:
+ blocks: []
+ """
+ ),
+ source="runtime_output_repair",
+ reason_code="runtime_output_repair_required",
+ required_paths={"output.statuses", "output.locations[].address"},
+ )
+
+ assert first == second
+ assert other_scope != first
+ assert (
+ workflow_update_module._output_contract_pin_key(
+ ctx,
+ "title: First\nworkflow_definition:\n blocks: []\n",
+ {"output.locations[].address", "output.statuses"},
+ )
+ == first
+ )
+
+ def test_output_contract_family_counter_aggregates_reject_classes(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "contract-budget-scope"
+ required_paths = {"output.record_id"}
+ first = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="metadata_reject",
+ )
+ second = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="synthesized_parameter_binding_ambiguous",
+ )
+ third = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="declared_output_return_shape",
+ )
+
+ signature = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml="title: Any\nworkflow_definition:\n blocks: []\n",
+ source="different",
+ reason_code="different",
+ required_paths=required_paths,
+ )
+ assert (first, second, third) == (1, 2, 3)
+ assert ctx.output_contract_reject_count_by_signature[signature] == 3
+
+ def test_output_contract_family_counter_is_scoped(self) -> None:
+ ctx = _code_only_ctx()
+ required_paths = {"output.status"}
+
+ ctx.turn_id = "first-build-goal"
+ first_scope_count = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="metadata_reject",
+ )
+ ctx.turn_id = "second-build-goal"
+ second_scope_count = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="metadata_reject",
+ )
+ ctx.turn_id = "first-build-goal"
+ first_scope_again = workflow_update_module._record_output_contract_family_reject(
+ ctx,
+ required_paths,
+ reject_family="output_contract_required",
+ )
+
+ assert (first_scope_count, second_scope_count, first_scope_again) == (1, 1, 2)
+ assert len(ctx.output_contract_reject_count_by_signature) == 2
+
+ def test_output_contract_owner_pin_is_scoped(self) -> None:
+ ctx = _code_only_ctx()
+ workflow_yaml = _yaml(
+ """
+ title: Shared output shape
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: first_goal_output
+ code: |
+ return {"output": {"status": "one"}}
+ - block_type: code
+ label: second_goal_output
+ code: |
+ return {"output": {"status": "two"}}
+ """
+ )
+ required_paths = {"output.status"}
+
+ ctx.turn_id = "first-build-goal"
+ workflow_update_module._pin_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ required_paths,
+ "first_goal_output",
+ )
+ ctx.turn_id = "second-build-goal"
+ workflow_update_module._pin_output_contract_block_label(
+ ctx,
+ workflow_yaml,
+ required_paths,
+ "second_goal_output",
+ )
+
+ assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
+ "second_goal_output"
+ )
+ ctx.turn_id = "first-build-goal"
+ assert workflow_update_module._pinned_output_contract_block_label(ctx, workflow_yaml, required_paths) == (
+ "first_goal_output"
+ )
+
+ def test_metadata_contract_recorded_key_stable_across_reject_count_and_reason_drift(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "stable-contract-key"
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: First candidate title
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ assert evaluation is not None
+
+ workflow_update_module._record_output_contract_reject(
+ ctx,
+ evaluation,
+ summary="First reject wording.",
+ )
+ first_key = ctx.latest_recorded_build_test_outcome.structural_key
+ assert first_key is not None
+
+ evaluation.payload["reason_code"] = "declared_output_return_shape"
+ workflow_update_module._record_output_contract_reject(
+ ctx,
+ evaluation,
+ summary="Changed reject wording and reason site.",
+ )
+
+ assert ctx.latest_recorded_build_test_outcome.structural_key == first_key
+ assert latest_recorded_build_test_outcome_repeated(ctx) is True
+ assert ctx.latest_recorded_build_test_outcome.missing_requested_output_facts
+
+ def test_metadata_contract_recorded_key_keeps_distinct_scope_and_required_paths(self) -> None:
+ workflow_yaml = _yaml(
+ """
+ title: Shared shape
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ def recorded_key(turn_id: str, output_path: str) -> str:
+ ctx = _code_only_ctx()
+ ctx.turn_id = turn_id
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path=output_path,
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+ assert evaluation is not None
+ workflow_update_module._record_output_contract_reject(ctx, evaluation, summary="Rejected.")
+ key = ctx.latest_recorded_build_test_outcome.structural_key
+ assert key is not None
+ return key
+
+ first_scope_key = recorded_key("first-build-goal", "output.status")
+ second_scope_key = recorded_key("second-build-goal", "output.status")
+ other_path_key = recorded_key("first-build-goal", "output.npi")
+
+ assert first_scope_key != second_scope_key
+ assert first_scope_key != other_path_key
+
+ @pytest.mark.asyncio
+ async def test_output_contract_after_two_steering_cycles_imposes_keyed_return_envelope(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ SimpleNamespace(
+ id="requested_flags",
+ output_path="output.flags",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ record_id = "ABC123"
+ flags = ["enabled"]
+ """
+ )
+
+ first = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+ second = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+ imposed = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert imposed["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert 'return {"output": {"flags": flags, "record_id": record_id}}' in code
+ metadata = ctx.code_artifact_metadata["extract_entry_output"]
+ assert metadata["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
+ assert (
+ json.loads(metadata["claimed_outcomes"][0]["extraction_schema"])["properties"]["output"]["properties"][
+ "record_id"
+ ]
+ == {}
+ )
+
+ @pytest.mark.asyncio
+ async def test_requested_output_contract_omits_repair_context_when_output_target_is_ambiguous(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ ctx.code_artifact_metadata = {
+ "lookup_entry": {
+ "block_label": "lookup_entry",
+ "claimed_outcomes": [{"goal_value_paths": ["output"]}],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ candidate_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: open_entry
+ code: |
+ await page.goto("https://example.com/search")
+ - block_type: code
+ label: lookup_entry
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "requested output contract" in result["error"]
+ assert "authoring_repair_context" not in result["data"]
+ assert ctx.last_code_authoring_repair_context is None
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
+ assert result["data"]["metadata_repair_contract"] is None
+ assert result["data"]["block_label"] == ""
+ assert result["data"]["shape_violations"] == ["missing_output_owner"]
+
+ @pytest.mark.asyncio
+ async def test_requested_output_contract_omits_repair_context_when_schema_target_is_ambiguous(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ child_paths = ["output.npi"]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ ctx.code_artifact_metadata = {
+ "lookup_entry": {
+ "block_label": "lookup_entry",
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": child_paths,
+ "extraction_schema": '{"type":"object","properties":{"npi":{"type":"string"}}}',
+ }
+ ],
+ "terminal_verifier_expectations": [{"goal_value_paths": child_paths}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ candidate_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: open_entry
+ code: |
+ await page.goto("https://example.com/search")
+ - block_type: code
+ label: lookup_entry
+ code: |
+ return {"output": {"npi": "1234567890"}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "extraction_schema" in result["error"]
+ assert result["data"]["authoring_repair_context"]["block_label"] == "lookup_entry"
+ assert ctx.last_code_authoring_repair_context is not None
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == [
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ }
+ ]
+ assert result["data"]["metadata_repair_contract"]["block_label"] == "lookup_entry"
+
+ def test_metadata_contract_run_preflight_blocks_missing_metadata_before_run(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id", "output.flags"],
+ required_extraction_schema_paths=["output.record_id", "output.flags"],
+ required_code_return_paths=["output.record_id", "output.flags"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, [])
+
+ assert result is not None
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "metadata_contract_required_before_run"
+ assert result["data"]["output_contract_reason_code"] == "output_contract_required"
+ assert result["data"]["block_label"] == "extract_entry_output"
+ assert result["data"]["canonical_required_child_paths"] == ["output.flags", "output.record_id"]
+ assert result["data"]["missing_goal_value_paths"] == ["output.flags", "output.record_id"]
+ assert result["data"]["missing_extraction_schema_paths"] == ["output.flags", "output.record_id"]
+ assert result["data"]["missing_code_return_paths"] == ["output.flags", "output.record_id"]
+ assert result["data"]["metadata_contract_source"] == "requested_output_contract"
+ assert result["data"]["metadata_contract_reason_code"] == ("requested_output_contract_missing_output_coverage")
+ assert result["data"]["metadata_repair_contract"] == {
+ "block_label": "extract_entry_output",
+ "required_goal_value_paths": ["output.flags", "output.record_id"],
+ "required_extraction_schema_paths": ["output.flags", "output.record_id"],
+ "required_code_return_paths": ["output.flags", "output.record_id"],
+ "source": "requested_output_contract",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ }
+ assert result["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
+ "output.flags",
+ "output.record_id",
+ ]
+ assert result["data"]["missing_requested_output_facts"] == [
+ {
+ "output_path": "output.flags",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.record_id",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", result)
+ assert sanitized["data"]["authoring_repair_context"]["metadata_contract_source"] == (
+ "requested_output_contract"
+ )
+ assert sanitized["data"]["metadata_repair_contract"] == result["data"]["metadata_repair_contract"]
+
+ def test_metadata_contract_run_preflight_budget_remains_repair_before_run(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.turn_id = "preflight-no-run-contract"
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ results = [
+ workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, []) for _ in range(4)
+ ]
+
+ assert results[0] is not None
+ assert results[0]["ok"] is False
+ assert results[0]["data"]["reason_code"] == "metadata_contract_required_before_run"
+ assert results[0]["data"]["output_contract_reason_code"] == "output_contract_required"
+ assert results[0]["data"]["output_contract_reject_count"] == 1
+ assert results[-1] is None
+ assert all(
+ result is None or result["data"]["reason_code"] != "output_contract_reject_budget_exhausted"
+ for result in results
+ )
+
+ def test_metadata_contract_run_preflight_accepts_complete_submitted_metadata(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id", "output.flags"],
+ required_extraction_schema_paths=["output.record_id", "output.flags"],
+ required_code_return_paths=["output.record_id", "output.flags"],
+ metadata_contract_source="recorded_outcome",
+ metadata_contract_reason_code="recorded_outcome_missing_output_coverage",
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"record_id":{"type":"string"},'
+ '"flags":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [
+ {"goal_value_paths": ["output.record_id", "output.flags"], "extraction_schema": schema}
+ ],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id", "output.flags"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+
+ assert result is None
+
+ def test_metadata_contract_run_preflight_rejects_root_only_output(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.last_code_authoring_repair_context = CodeAuthoringRepairContext(
+ block_label="extract_entry_output",
+ reason_code="metadata_reject",
+ required_goal_value_paths=["output.record_id"],
+ required_extraction_schema_paths=["output.record_id"],
+ required_code_return_paths=["output.record_id"],
+ metadata_contract_source="requested_output_contract",
+ metadata_contract_reason_code="requested_output_contract_missing_output_coverage",
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"summary": "found"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [
+ {
+ "goal_value_paths": ["output"],
+ "extraction_schema": '{"type":"object","properties":{"output":{"type":"object"}}}',
+ }
+ ],
+ "terminal_verifier_expectations": [{"goal_value_paths": ["output"]}],
+ }
+ ]
+
+ result = workflow_update_module._metadata_contract_run_preflight_reject(ctx, workflow_yaml, metadata)
+
+ assert result is not None
+ assert result["ok"] is False
+ assert result["data"]["missing_goal_value_paths"] == ["output.record_id"]
+ assert result["data"]["missing_extraction_schema_paths"] == ["output.record_id"]
+ assert result["data"]["missing_code_return_paths"] == ["output.record_id"]
+
+ def test_metadata_contract_scaffold_uses_recorded_paths_before_request_policy(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.requested_value",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:typed-output",
+ missing_requested_output_facts=[
+ {"output_path": "output.recorded_value", "output_root": "output", "value_status": "no_typed_value"},
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"recorded_value": "ABC123"}}
+ """
+ )
+ required_paths, source, reason_code = workflow_update_module._required_child_output_paths_for_authoring(ctx)
+
+ scaffolded = workflow_update_module._apply_metadata_contract_scaffold(
+ ctx,
+ workflow_yaml,
+ [],
+ required_paths=required_paths,
+ source=source,
+ reason_code=reason_code,
+ )
+
+ assert scaffolded[0]["artifact_id"] == "code_artifact:extract_entry_output"
+ assert scaffolded[0]["block_label"] == "extract_entry_output"
+ assert scaffolded[0]["claimed_outcomes"][0]["goal_value_paths"] == ["output.recorded_value"]
+ schema = json.loads(scaffolded[0]["claimed_outcomes"][0]["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["recorded_value"] == {}
+
+ def test_runtime_output_facts_record_same_run_null_without_evidence_text_backfill(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "extract_entry_output",
+ "status": "completed",
+ "extracted_data": {"output": {"npi": None}, "evidence_text": "Value 1234567890"},
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="evidence_contradicts",
+ output_path="output.npi",
+ grounding_mode="exact_value",
+ expected_output_shape="string",
+ has_exact_value=False,
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.reason_code == "outcome_not_demonstrated"
+ assert outcome.runtime_output_repair_facts == [
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
+ "output_root": "output",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "grounding_mode": "exact_value",
+ "expected_output_shape": "string",
+ "value_status": "null",
+ "evidence_refs": ["registered_output:extract_entry_output:npi", "output:extract_entry_output"],
+ }
+ ]
+ assert outcome.is_authoritative is True
+
+ def test_runtime_output_facts_record_empty_typed_output_paths_for_next_contract(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [
+ {
+ "label": "summarize_access_output",
+ "status": "completed",
+ "extracted_data": {
+ "extracted_information": [],
+ "summarize_access_output": {},
+ "evidence_text": "diagnostic page text",
+ },
+ }
+ ],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=[
+ "requested_form_exists",
+ "requested_path_label",
+ "requested_next_action",
+ ],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_form_exists",
+ state="unsatisfied",
+ reason_code="structurally_abstained",
+ output_path="output.public_form_exists",
+ grounding_mode="missing",
+ expected_output_shape="boolean",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_path_label",
+ state="unsatisfied",
+ reason_code="missing_exact_field",
+ output_path="output.visible_page_path_label",
+ grounding_mode="missing",
+ expected_output_shape="string",
+ ),
+ CriterionVerdict(
+ criterion_id="requested_next_action",
+ state="unsatisfied",
+ reason_code="missing_exact_field",
+ output_path="output.recommended_next_action",
+ grounding_mode="missing",
+ expected_output_shape="string",
+ ),
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+ assert outcome is not None
+ assert {fact["output_path"]: fact["value_status"] for fact in outcome.runtime_output_repair_facts} == {
+ "output.public_form_exists": "structural_abstained",
+ "output.recommended_next_action": "no_typed_value",
+ "output.visible_page_path_label": "no_typed_value",
+ }
+ assert all(
+ "evidence_text" not in ref
+ for fact in outcome.runtime_output_repair_facts
+ for ref in fact.get("evidence_refs", [])
+ )
+
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = outcome
+ required_paths, source, reason_code = workflow_update_module._output_contract_required_paths_source(ctx)
+ assert required_paths == {
+ "output.public_form_exists",
+ "output.recommended_next_action",
+ "output.visible_page_path_label",
+ }
+ assert source == "runtime_output_repair"
+ assert reason_code == "runtime_output_repair_required"
+
+ def test_runtime_output_facts_ignore_other_run_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "workflow_run_id": "wr_previous",
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
+ )
+
+ def test_runtime_output_facts_ignore_unscoped_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ registered_output_parameter_payloads=[
+ {
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "block_label" not in outcome.runtime_output_repair_facts[0]
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
+ )
+
+ def test_runtime_output_facts_ignore_unscoped_fallback_registered_values(self) -> None:
+ result = {
+ "ok": True,
+ "data": {
+ "workflow_run_id": "wr_current",
+ "registered_output_parameter_values": [
+ {
+ "block_label": "extract_entry_output",
+ "output_parameter_key": "npi",
+ "value": None,
+ }
+ ],
+ "blocks": [{"label": "extract_entry_output", "status": "completed", "extracted_data": {}}],
+ },
+ }
+ verification = CompletionVerificationResult(
+ status="evaluated",
+ criterion_ids=["requested_npi"],
+ verdicts=[
+ CriterionVerdict(
+ criterion_id="requested_npi",
+ state="unsatisfied",
+ reason_code="no_evidence",
+ output_path="output.npi",
+ grounding_mode="missing",
+ )
+ ],
+ )
+
+ outcome = recorded_outcome_from_run_blocks_result(
+ result,
+ recorded_run_outcome=RecordedRunOutcome(
+ verdict="not_demonstrated",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ ),
+ completion_verification=verification,
+ )
+
+ assert outcome is not None
+ assert outcome.runtime_output_repair_facts[0]["value_status"] == "no_typed_value"
+ assert "block_label" not in outcome.runtime_output_repair_facts[0]
+ assert "registered_output:extract_entry_output:npi" not in outcome.runtime_output_repair_facts[0].get(
+ "evidence_refs", []
+ )
+
+ def test_runtime_output_facts_override_request_policy_contract_paths(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.requested_value",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
+ "output_root": "output",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "value_status": "null",
+ },
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.locations[].address",
+ "output_root": "output",
+ "criterion_id": "requested_location",
+ "reason_code": "structurally_abstained",
+ "grounding_mode": "missing",
+ "value_status": "structural_abstained",
+ },
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"requested_value": "wrong"}}
+ """
+ )
+
+ evaluation = workflow_update_module._evaluate_output_contract_for_code_block(ctx, workflow_yaml, [])
+
+ assert evaluation is not None
+ assert evaluation.source == "runtime_output_repair"
+ assert evaluation.reason_code == "runtime_output_repair_required"
+ assert evaluation.required_paths == {"output.locations[].address", "output.npi"}
+ assert sorted(
+ evaluation.payload["runtime_output_repair_facts"],
+ key=lambda item: str(item.get("output_path") or ""),
+ ) == sorted(
+ ctx.latest_recorded_build_test_outcome.runtime_output_repair_facts,
+ key=lambda item: str(item.get("output_path") or ""),
+ )
+
+ @pytest.mark.asyncio
+ async def test_runtime_output_repair_facts_trigger_one_envelope_attempt_before_budget(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ workflow_run_id="wr_current",
+ structural_failure_identity="completion:runtime-output",
+ runtime_output_repair_facts=[
+ {
+ "workflow_run_id": "wr_current",
+ "block_label": "extract_entry_output",
+ "output_path": "output.npi",
+ "output_root": "output",
+ "criterion_id": "requested_npi",
+ "reason_code": "evidence_contradicts",
+ "value_status": "null",
+ }
+ ],
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ npi = "1234567890"
+ return npi
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ code = _single_code_block(parsed)["code"]
+ assert 'return {"output": {"npi": npi}}' in code
+ assert ctx.code_artifact_metadata["extract_entry_output"]["claimed_outcomes"][0]["goal_value_paths"] == [
+ "output.npi"
+ ]
+ assert ctx.runtime_output_repair_attempt_by_signature
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_applies_metadata_contract_scaffold_for_unambiguous_owner(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ SimpleNamespace(
+ id="requested_flags",
+ output_path="output.flags",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ code: |
+ return {"output": {"record_id": "ABC123", "flags": ["enabled"]}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ stored = ctx.code_artifact_metadata["extract_entry_output"]
+ assert stored["artifact_id"] == "code_artifact:extract_entry_output"
+ assert stored["claimed_outcomes"][0]["goal_value_paths"] == ["output.flags", "output.record_id"]
+ assert stored["terminal_verifier_expectations"][0]["goal_value_paths"] == [
+ "output.flags",
+ "output.record_id",
+ ]
+ schema = json.loads(stored["claimed_outcomes"][0]["extraction_schema"])
+ assert schema["properties"]["output"]["properties"]["record_id"] == {}
+ assert schema["properties"]["output"]["properties"]["flags"] == {}
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_allows_neutral_multi_block_output_owner(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: enter_filters
+ code: |
+ await page.locator("#filter").fill("ABC123")
+ - block_type: code
+ label: choose_record
+ code: |
+ await page.locator("#choose").click()
+ - block_type: code
+ label: extract_record
+ code: |
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == ["enter_filters", "choose_record", "extract_record"]
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_rejects_collapsed_output_owner(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: value
+ blocks:
+ - block_type: code
+ label: extract_record
+ parameter_keys:
+ - value
+ code: |
+ _scout_entry_target = page.locator("#filter")
+ try:
+ await _scout_entry_target.wait_for(state="visible", timeout=1000)
+ except Exception:
+ await page.goto("https://example.com/records", wait_until="domcontentloaded")
+ await _scout_entry_target.wait_for(state="visible")
+ await page.locator("#filter").fill(str(value))
+ await page.locator("#choose").click()
+ await page.wait_for_load_state("domcontentloaded")
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "output_contract_required"
+ assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_rejects_collapsed_output_owner_with_sibling_block(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: prepare_session
+ code: |
+ context["ready"] = True
+ - block_type: code
+ label: extract_record
+ code: |
+ _scout_entry_target = page.locator("#filter")
+ try:
+ await _scout_entry_target.wait_for(state="visible", timeout=1000)
+ except Exception:
+ await page.goto("https://example.com/records", wait_until="domcontentloaded")
+ await _scout_entry_target.wait_for(state="visible")
+ await page.locator("#filter").fill(str(value))
+ await page.locator("#choose").click()
+ await page.wait_for_load_state("domcontentloaded")
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "output_contract_required"
+ assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
+ assert result["data"]["block_label"] == "extract_record"
+ assert result["data"]["output_owner_labels"] == ["extract_record"]
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_rejects_collapsed_output_owner_with_extra_browser_action(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "type_text", "selector": "#filter", "source_url": "https://example.com/records"},
+ {"tool_name": "click", "selector": "#choose", "source_url": "https://example.com/records"},
+ ]
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: value
+ blocks:
+ - block_type: code
+ label: extract_record
+ parameter_keys:
+ - value
+ code: |
+ _scout_entry_target = page.locator("#filter")
+ try:
+ await _scout_entry_target.wait_for(state="visible", timeout=1000)
+ except Exception:
+ await page.goto("https://example.com/records", wait_until="domcontentloaded")
+ await _scout_entry_target.wait_for(state="visible")
+ await page.locator("#filter").fill(str(value))
+ await page.locator("#choose").click()
+ await page.wait_for_load_state("domcontentloaded")
+ await page.locator("#extra").click()
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "output_contract_required"
+ assert result["data"]["shape_violations"] == ["separated_spine_shape_required"]
+ assert result["data"]["block_label"] == "extract_record"
+
+ @pytest.mark.asyncio
+ async def test_separated_spine_shape_ignores_one_block_without_multi_stage_spine(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{"record_id":{"type":"string"}}}}}'
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: extract_record
+ code: |
+ return {"output": {"record_id": "ABC123"}}
+ """
+ )
+ metadata = [
+ {
+ "block_label": "extract_record",
+ "claimed_outcomes": [{"goal_value_paths": ["output.record_id"], "extraction_schema": schema}],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.record_id"], "extraction_schema": schema}
+ ],
+ }
+ ]
+
+ result = await _update_workflow(
+ {"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert result["ok"] is True
+
+ @pytest.mark.asyncio
+ async def test_requested_output_contract_allows_aligned_initial_candidate(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ label = "lookup_entry"
+ child_paths = ["output.npi", "output.locations[].address", "output.statuses"]
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"locations":{"type":"array","items":{"type":"object","properties":{"address":{"type":"string"}}}},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id=f"requested_{index}",
+ output_path=path,
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ for index, path in enumerate(child_paths)
+ ]
+ )
+ ctx.code_artifact_metadata = {
+ label: {
+ "block_label": label,
+ "claimed_outcomes": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ "terminal_verifier_expectations": [{"goal_value_paths": child_paths, "extraction_schema": schema}],
+ }
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ candidate_yaml = _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ return {{
+ "output": {{
+ "npi": "1234567890",
+ "locations": [{{"address": "Example location"}}],
+ "statuses": ["active"],
+ }}
+ }}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is True
+ labels = [block.get("label") for block in workflow_blocks(parse_workflow_yaml(ctx.workflow_yaml))]
+ assert labels == [label]
+
+ @pytest.mark.asyncio
+ async def test_requested_output_contract_missing_metadata_records_child_path_facts(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.npi",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ SimpleNamespace(
+ id="requested_statuses",
+ output_path="output.statuses",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
+ candidate_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: output, key: lookup_result}
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ prompt: Extract structured fields from the entry.
+ code: |
+ await page.goto("https://example.com/search")
+ return {"output": {"summary": "found"}}
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
+
+ assert result["ok"] is False
+ assert "Required requested output paths: output.npi, output.statuses" in result["error"]
+ expected_facts = [
+ {
+ "output_path": "output.npi",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ {
+ "output_path": "output.statuses",
+ "output_root": "output",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ "value_status": "no_typed_value",
+ },
+ ]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert outcome is not None
+ assert outcome.reason_code == "metadata_reject"
+ assert outcome.missing_requested_output_facts == expected_facts
+ assert result["data"]["missing_requested_output_facts"] == expected_facts
+ assert result["data"]["metadata_repair_contract"] == {
+ "block_label": "extract_entry_output",
+ "required_goal_value_paths": ["output.npi", "output.statuses"],
+ "required_extraction_schema_paths": ["output.npi", "output.statuses"],
+ "required_code_return_paths": ["output.npi", "output.statuses"],
+ "source": "requested_output_contract",
+ "reason_code": "requested_output_contract_missing_output_coverage",
+ }
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "metadata_reject"
+ assert repair_context["block_label"] == "extract_entry_output"
+ assert repair_context["runtime_failure_class"] == "requested_output_contract_missing_output_coverage"
+ assert repair_context["required_goal_value_paths"] == ["output.npi", "output.statuses"]
+ assert repair_context["required_extraction_schema_paths"] == ["output.npi", "output.statuses"]
+ assert repair_context["required_code_return_paths"] == ["output.npi", "output.statuses"]
+ assert repair_context["metadata_contract_source"] == "requested_output_contract"
+ assert repair_context["metadata_contract_reason_code"] == ("requested_output_contract_missing_output_coverage")
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
+ sanitized = sanitize_tool_result_for_llm("update_workflow", result)
+ assert sanitized["data"]["authoring_repair_context"]["required_goal_value_paths"] == [
+ "output.npi",
+ "output.statuses",
+ ]
+ assert sanitized["data"]["authoring_repair_context"]["required_extraction_schema_paths"] == [
+ "output.npi",
+ "output.statuses",
+ ]
+ assert sanitized["data"]["authoring_repair_context"]["required_code_return_paths"] == [
+ "output.npi",
+ "output.statuses",
+ ]
+ assert sanitized["data"]["authoring_repair_context"]["metadata_contract_source"] == "requested_output_contract"
+ assert sanitized["data"]["metadata_repair_contract"] == result["data"]["metadata_repair_contract"]
+
+ schema = (
+ '{"type":"object","properties":{"output":{"type":"object","properties":{'
+ '"npi":{"type":"string"},'
+ '"statuses":{"type":"array","items":{"type":"string"}}}}}}'
+ )
+ corrected_metadata = {
+ "extract_entry_output": {
+ "block_label": "extract_entry_output",
+ "claimed_outcomes": [
+ {"goal_value_paths": ["output.npi", "output.statuses"], "extraction_schema": schema}
+ ],
+ "terminal_verifier_expectations": [
+ {"goal_value_paths": ["output.npi", "output.statuses"], "extraction_schema": schema}
+ ],
+ }
+ }
+ corrected_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: output, key: lookup_result}
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ prompt: Extract structured fields from the entry.
+ code: |
+ await page.goto("https://example.com/search")
+ return {"output": {"npi": "1234567890", "statuses": ["active"]}}
+ """
+ )
+
+ corrected = await _update_workflow(
+ {"workflow_yaml": corrected_yaml, "code_artifact_metadata": corrected_metadata},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert corrected["ok"] is True
+ accepted_yaml = parse_workflow_yaml(ctx.workflow_yaml)
+ accepted_block = _single_code_block(accepted_yaml)
+ assert accepted_block["label"] == "extract_entry_output"
+ assert (
+ accepted_block["code"].strip().endswith('return {"output": {"npi": "1234567890", "statuses": ["active"]}}')
+ )
+ assert accepted_block["extraction_schema"] == schema
+ stored_metadata = ctx.code_artifact_metadata["extract_entry_output"]
+ assert stored_metadata["block_label"] == corrected_metadata["extract_entry_output"]["block_label"]
+ assert stored_metadata["claimed_outcomes"][0]["goal_value_paths"] == ["output.npi", "output.statuses"]
+ assert stored_metadata["claimed_outcomes"][0]["extraction_schema"] == schema
+ assert stored_metadata["terminal_verifier_expectations"][0]["goal_value_paths"] == [
+ "output.npi",
+ "output.statuses",
+ ]
+ assert stored_metadata["terminal_verifier_expectations"][0]["extraction_schema"] == schema
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+
+ @pytest.mark.asyncio
+ async def test_recorded_outcome_missing_metadata_records_recorded_child_path_facts(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
- label = "lookup_provider_and_extract_credentials"
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.requested_only",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ ),
+ ]
+ )
ctx.latest_recorded_build_test_outcome = RecordedBuildTestOutcome(
phase="persisted_block_run",
attempted_tool="update_and_run_blocks",
@@ -2411,52 +4980,60 @@ class TestCodeRepairProgressClassification:
structural_failure_identity="completion:unsatisfied-output",
authored_structure_signature="authored:previous-failed-candidate",
missing_requested_output_facts=[
- {"output_path": "address", "output_root": "address", "value_status": "no_typed_value"},
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
- "value_status": "no_typed_value",
- },
+ {"output_path": "output.recorded", "output_root": "output", "value_status": "no_typed_value"},
],
)
- partial_yaml = _yaml(
- f"""
- title: Provider lookup
+ candidate_yaml = _yaml(
+ """
+ title: Entry lookup
workflow_definition:
+ parameters:
+ - {parameter_type: output, key: lookup_result}
blocks:
- block_type: code
- label: {label}
+ label: extract_entry_output
+ prompt: Extract structured fields from the entry.
code: |
- await page.locator("#locInput").wait_for(state="visible", timeout=15000)
- return {{"search_options": []}}
+ await page.goto("https://example.com/search")
+ return {"output": {"summary": "found"}}
"""
)
- result = await _update_workflow({"workflow_yaml": partial_yaml}, ctx, allow_missing_credentials=True)
+ result = await _update_workflow({"workflow_yaml": candidate_yaml}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
+ assert "Required requested output paths: output.recorded" in result["error"]
outcome = ctx.latest_recorded_build_test_outcome
assert outcome is not None
assert outcome.reason_code == "metadata_reject"
assert outcome.missing_requested_output_facts == [
{
- "output_path": "address",
- "output_root": "address",
- "reason_code": "recorded_outcome_missing_output_coverage",
- "value_status": "no_typed_value",
- },
- {
- "output_path": "credentialing_status",
- "output_root": "credentialing_status",
+ "output_path": "output.recorded",
+ "output_root": "output",
"reason_code": "recorded_outcome_missing_output_coverage",
"value_status": "no_typed_value",
- },
+ }
]
- rendered = agent_module._recorded_build_test_outcome_prompt(ctx)
- assert "missing_requested_output_facts:" in rendered
- assert "output_root=address" in rendered
- assert "output_root=credentialing_status" in rendered
- assert "value_status=no_typed_value" in rendered
+ assert result["data"]["missing_requested_output_facts"] == outcome.missing_requested_output_facts
+ assert result["data"]["metadata_repair_contract"] == {
+ "block_label": "extract_entry_output",
+ "required_goal_value_paths": ["output.recorded"],
+ "required_extraction_schema_paths": ["output.recorded"],
+ "required_code_return_paths": ["output.recorded"],
+ "source": "recorded_outcome",
+ "reason_code": "recorded_outcome_missing_output_coverage",
+ }
+ repair_context = result["data"]["authoring_repair_context"]
+ assert repair_context["reason_code"] == "metadata_reject"
+ assert repair_context["block_label"] == "extract_entry_output"
+ assert repair_context["runtime_failure_class"] == "recorded_outcome_missing_output_coverage"
+ assert repair_context["required_goal_value_paths"] == ["output.recorded"]
+ assert repair_context["required_extraction_schema_paths"] == ["output.recorded"]
+ assert repair_context["required_code_return_paths"] == ["output.recorded"]
+ assert repair_context["metadata_contract_source"] == "recorded_outcome"
+ assert repair_context["metadata_contract_reason_code"] == "recorded_outcome_missing_output_coverage"
+ assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
+ assert ctx.last_code_authoring_repair_context.model_dump(mode="json") == repair_context
@pytest.mark.asyncio
async def test_authoritative_outcome_not_demonstrated_rejects_when_only_last_run_produced_required_roots(
@@ -3714,11 +6291,16 @@ class TestCompiledAuthoringImposition:
assert result["data"]["imposed_substitutions"]["preserved_extraction_suffix"] is True
parsed = parse_workflow_yaml(ctx.workflow_yaml)
assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert "<fill" not in block["code"]
- assert "<fill: captured value>" not in block["code"]
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
- assert 'records = [{"number": "REC-001", "status": "credentialed"}]' in block["code"]
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ output_block = _code_blocks(parsed)["search_registry"]
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ output_code = str(output_block["code"])
+ assert "<fill" not in browser_code
+ assert "<fill: captured value>" not in browser_code
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in browser_code
+ assert 'records = [{"number": "REC-001", "status": "credentialed"}]' in output_code
+ assert 'return {"records": records}' in output_code
@pytest.mark.asyncio
async def test_imposition_preserves_custom_extraction_code_with_goal_metadata(
@@ -3761,14 +6343,8 @@ class TestCompiledAuthoringImposition:
result = await _update_workflow({"workflow_yaml": submitted, "code_artifact_metadata": metadata}, ctx)
- assert result["ok"] is True
- assert result["data"]["imposed_substitutions"]["preserved_submitted_extraction_code"] is True
- parsed = parse_workflow_yaml(ctx.workflow_yaml)
- assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["code"].startswith("records = []")
- assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
- assert 'records.append({"number": "REC-001", "status": "credentialed"})' in block["code"]
+ assert result["ok"] is False
+ assert "selected output extraction boundary is ambiguous" in result["error"]
def _download_ctx(self) -> CopilotContext:
from skyvern.forge.sdk.copilot.reached_download_target import ReachedDownloadTarget
@@ -4157,6 +6733,38 @@ class TestCompiledAuthoringImposition:
assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
assert ctx.workflow_yaml == ""
+ @pytest.mark.asyncio
+ async def test_selector_join_aliases_synthesized_parameter_to_authored_fill_parameter(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ submitted = _yaml(
+ """
+ title: Provider lookup
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: string
+ key: search_term
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.locator("#provInput").fill(search_term)
+ """
+ )
+
+ result = await _update_workflow({"workflow_yaml": submitted}, ctx)
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["search_term"]
+ assert 'await page.locator("#provInput").fill(str(search_term))' in block["code"]
+ assert "provider_name" not in block["code"]
+
@pytest.mark.asyncio
async def test_single_synthesized_key_without_bindable_fill_records_repair_context(
self, monkeypatch: pytest.MonkeyPatch
@@ -4202,6 +6810,98 @@ class TestCompiledAuthoringImposition:
assert ctx.latest_recorded_build_test_outcome.is_authoritative is True
assert ctx.workflow_yaml == ""
+ @pytest.mark.asyncio
+ async def test_parameter_binding_after_two_steering_cycles_imposes_synthesized_binding(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+ first = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+ second = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+ imposed = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert imposed["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_name"]
+ assert 'await page.locator("#provInput").fill(str(provider_name))' in block["code"]
+
+ @pytest.mark.asyncio
+ async def test_parameter_binding_after_two_steering_cycles_imposes_for_mixed_fill(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = self._provider_search_ctx()
+
+ first = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
+ second = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
+ imposed = await _update_workflow({"workflow_yaml": _SUBMITTED_MIXED_LOCATOR_FILL_COMPUTED_LITERAL_YAML}, ctx)
+
+ assert first["ok"] is False
+ assert second["ok"] is False
+ assert imposed["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ assert block["parameter_keys"] == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ }
+ ]
+
+ def test_parameter_binding_uses_canonical_output_contract_budget(self) -> None:
+ ctx = self._provider_search_ctx()
+ ctx.turn_id = "parameter-binding-contract-budget"
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ SimpleNamespace(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ signature = workflow_update_module._output_contract_signature(
+ ctx=ctx,
+ workflow_yaml="title: First\nworkflow_definition:\n blocks: []\n",
+ source="metadata_reject",
+ reason_code="metadata_reject",
+ required_paths={"output.record_id"},
+ )
+ ctx.output_contract_reject_count_by_signature = {signature: 2}
+ parsed = parse_workflow_yaml(_SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML)
+ assert isinstance(parsed, dict)
+ block = _single_code_block(parsed)
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+
+ reconciliation = workflow_update_module._reconcile_synthesized_parameters(
+ ctx=ctx,
+ parsed=parsed,
+ code_block=block,
+ submitted_code=str(block.get("code") or ""),
+ synthesized_parameters=synthesized.parameters,
+ scout_trajectory=ctx.scout_trajectory,
+ )
+
+ assert reconciliation.violations == []
+ assert reconciliation.parameter_keys == ["provider_name"]
+ assert parsed["workflow_definition"]["parameters"] == [
+ {
+ "parameter_type": "workflow",
+ "workflow_parameter_type": "string",
+ "key": "provider_name",
+ }
+ ]
+
@pytest.mark.asyncio
async def test_synthesized_parameter_repair_context_uses_safe_selector_atom(
self, monkeypatch: pytest.MonkeyPatch
@@ -4289,14 +6989,14 @@ class TestCompiledAuthoringImposition:
result = await _update_workflow({"workflow_yaml": submitted}, ctx)
assert result["ok"] is False
- assert "missing submitted workflow parameter and literal binding is ambiguous" in result["error"]
+ assert "authored selector-join alias is reused by another synthesized input" in result["error"]
assert ctx.workflow_yaml == ""
repair_context = result["data"]["authoring_repair_context"]
assert repair_context["reason_code"] == "synthesized_parameter_binding_ambiguous"
assert repair_context["block_label"] == "order_status"
- assert repair_context["unresolved_names"] == ["enter_confirmation"]
+ assert repair_context["unresolved_names"] == ["zip_code"]
assert repair_context["available_parameter_keys"] == ["order_lookup"]
- assert repair_context["binding_candidates"] == ["enter_confirmation", "order_lookup"]
+ assert repair_context["binding_candidates"] == ["zip_code", "order_lookup"]
assert "update_and_run_blocks" in repair_context["repair_instruction"]
assert "secret" not in str(repair_context).lower()
assert isinstance(ctx.last_code_authoring_repair_context, CodeAuthoringRepairContext)
@@ -6127,6 +8827,19 @@ def _quote_ctx() -> CopilotContext:
return ctx
+def _author_time_reject_outcome(reason_code: BuildTestOutcomeReasonCode) -> RecordedBuildTestOutcome:
+ return recorded_outcome_from_author_time_reject(
+ reason_code=reason_code,
+ attempted_block_label=workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
+ block_labels=[workflow_update_module._SYNTHESIZED_BLOCK_LABEL],
+ structural_payload={
+ "reason_code": reason_code,
+ "block_label": workflow_update_module._SYNTHESIZED_BLOCK_LABEL,
+ },
+ observed_evidence_summary=reason_code,
+ )
+
+
def _code_blocks(parsed: dict[str, object]) -> dict[str, dict[str, object]]:
blocks = [block for block in workflow_blocks(parsed) if str(block.get("block_type") or "").lower() == "code"]
return {str(block.get("label") or ""): block for block in blocks}
@@ -6581,6 +9294,11 @@ class TestWholeTrajectoryImposition:
def test_preserves_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
ctx = _quote_ctx()
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ metadata = _terminal_metadata(label, "quote")
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
+ ctx.raw_code_artifact_metadata = [metadata]
synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
assert synthesized is not None
submitted_code = (
@@ -6593,7 +9311,7 @@ class TestWholeTrajectoryImposition:
workflow_definition:
blocks:
- block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ label: {label}
code: |
{textwrap.indent(submitted_code, " " * 18)}
"""
@@ -6604,15 +9322,23 @@ class TestWholeTrajectoryImposition:
assert result.violations == []
assert result.substitutions is not None
assert result.substitutions["preserved_extraction_suffix"] is True
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert 'await page.locator("#coverage-next").click()' in code
- assert 'heading = await page.locator("h1").inner_text()' in code
- assert code.index('await page.locator("#coverage-next").click()') < code.index(
- 'heading = await page.locator("h1").inner_text()'
- )
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ assert code_blocks[-1]["label"] == label
+ output_code = str(code_blocks[-1]["code"]).strip()
+ assert output_code == 'heading = await page.locator("h1").inner_text()\nreturn {"heading": heading}'
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
def test_preserves_page_read_only_selected_extraction_suffix_after_exact_spine(self) -> None:
ctx = _quote_ctx()
+ label = workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ metadata = _terminal_metadata(label, "quote")
+ metadata["claimed_outcomes"][0]["goal_value_paths"] = ["heading"]
+ metadata["terminal_verifier_expectations"][0]["goal_value_paths"] = ["heading"]
+ ctx.raw_code_artifact_metadata = [metadata]
synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
assert synthesized is not None
submitted_code = (
@@ -6625,7 +9351,7 @@ class TestWholeTrajectoryImposition:
workflow_definition:
blocks:
- block_type: code
- label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ label: {label}
code: |
{textwrap.indent(submitted_code, " " * 18)}
"""
@@ -6636,12 +9362,15 @@ class TestWholeTrajectoryImposition:
assert result.violations == []
assert result.substitutions is not None
assert result.substitutions["preserved_extraction_suffix"] is True
- code = str(_single_code_block(parse_workflow_yaml(result.workflow_yaml))["code"])
- assert 'await page.locator("#coverage-next").click()' in code
- assert 'heading = await page.inner_text("h1")' in code
- assert code.index('await page.locator("#coverage-next").click()') < code.index(
- 'heading = await page.inner_text("h1")'
- )
+ parsed = parse_workflow_yaml(result.workflow_yaml)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ assert code_blocks[-1]["label"] == label
+ output_code = str(code_blocks[-1]["code"]).strip()
+ assert output_code == 'heading = await page.inner_text("h1")\nreturn {"heading": heading}'
+ browser_code = "\n".join(str(block.get("code") or "") for block in code_blocks[:-1])
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert all("heading" not in str(block.get("code") or "") for block in code_blocks[:-1])
def test_rejects_selected_extraction_suffix_browser_mutation(self) -> None:
ctx = _quote_ctx()
@@ -6946,7 +9675,23 @@ class TestWholeTrajectoryImposition:
result = await _update_workflow(
{
"workflow_yaml": submitted,
- "code_artifact_metadata": [_terminal_metadata(label, "quote")],
+ "code_artifact_metadata": [
+ {
+ **_terminal_metadata(label, "quote"),
+ "claimed_outcomes": [
+ {
+ **_terminal_metadata(label, "quote")["claimed_outcomes"][0],
+ "goal_value_paths": ["quote"],
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ **_terminal_metadata(label, "quote")["terminal_verifier_expectations"][0],
+ "goal_value_paths": ["quote"],
+ }
+ ],
+ }
+ ],
},
ctx,
)
@@ -6958,12 +9703,16 @@ class TestWholeTrajectoryImposition:
assert "scrubbed_stale_selected_goal_value_paths" not in substitutions
parsed = parse_workflow_yaml(ctx.workflow_yaml)
assert isinstance(parsed, dict)
- block = _single_code_block(parsed)
- assert block["parameter_keys"] == ["postal_code"]
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ block = _code_blocks(parsed)[label]
+ browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ assert all(stage.get("parameter_keys") == ["postal_code"] for stage in code_blocks[:-1])
+ assert "postal_code" in browser_code
+ assert "zip_code" not in browser_code
code = str(block["code"])
- assert "postal_code" in code
- assert "zip_code" not in code
assert f"return {label}_output" in code
+ assert "postal_code" not in code
artifact = ctx.code_artifact_metadata[label]
assert workflow_update_module._artifact_declares_goal_values(artifact)
@@ -6979,7 +9728,7 @@ class TestWholeTrajectoryImposition:
- block_type: code
label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
code: |
- records = [{{"quote": "pending"}}]
+ records = [{{"number": "Q-001"}}]
"""
)
@@ -6996,17 +9745,142 @@ class TestWholeTrajectoryImposition:
assert result["ok"] is True
parsed = parse_workflow_yaml(ctx.workflow_yaml)
assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
- code = str(block["code"])
+ code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ output_code = str(block["code"])
+ assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
assert 'await page.locator("#zip").fill(str(zip_code))' in code
assert 'await page.locator("#continue").click()' in code
assert 'await page.locator("#coverage-next").click()' in code
- assert 'records = [{"quote": "pending"}]' in code
- assert code.index('await page.locator("#coverage-next").click()') < code.index(
- 'records = [{"quote": "pending"}]'
+ assert 'records = [{"number": "Q-001"}]' in output_code
+ assert 'return {"records": records}' in output_code
+ assert all("records" not in str(stage.get("code") or "") for stage in code_blocks[:-1])
+
+ @pytest.mark.asyncio
+ async def test_author_metadata_reject_reopens_changed_collapsed_code_block(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _quote_ctx()
+ ctx.update_workflow_called = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("metadata_reject")
+ ctx.workflow_yaml = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "old"}}]
+ """
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"number": "Q-001"}}]
+ """
+ )
+
+ result = await _update_workflow(
+ {
+ "workflow_yaml": submitted,
+ "code_artifact_metadata": [
+ _terminal_metadata(workflow_update_module._SYNTHESIZED_BLOCK_LABEL, "quote result")
+ ],
+ },
+ ctx,
+ )
+
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ block = _code_blocks(parsed)[workflow_update_module._SYNTHESIZED_BLOCK_LABEL]
+ browser_code = "\n".join(str(stage.get("code") or "") for stage in code_blocks[:-1])
+ code = str(block["code"])
+ assert result["data"]["imposed_substitutions"]["source_trajectory_count"] == 3
+ assert result["data"]["imposed_substitutions"]["separated_browser_stage_count"] > 1
+ assert code_blocks[-1]["label"] == workflow_update_module._SYNTHESIZED_BLOCK_LABEL
+ assert 'await page.locator("#zip").fill(str(zip_code))' in browser_code
+ assert 'await page.locator("#continue").click()' in browser_code
+ assert 'await page.locator("#coverage-next").click()' in browser_code
+ assert 'records = [{"number": "Q-001"}]' not in browser_code
+ assert 'records = [{"number": "Q-001"}]' in code
+ assert 'return {"records": records}' in code
+
+ @pytest.mark.asyncio
+ async def test_author_parameter_reject_reopens_and_preserves_typed_violation(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "type_text",
+ "selector": "#provInput",
+ "source_url": "https://example.com/find-care",
+ "typed_length": 13,
+ "role": "textbox",
+ "accessible_name": "Provider Name",
+ "trajectory_index": 0,
+ }
+ ]
+ ctx.update_workflow_called = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("synthesized_parameter_binding_ambiguous")
+ ctx.workflow_yaml = _SUBMITTED_LITERAL_YAML
+
+ result = await _update_workflow({"workflow_yaml": _SUBMITTED_UNKNOWN_COMPUTED_LITERAL_YAML}, ctx)
+
+ assert result["ok"] is False
+ assert "Unable to bind synthesized parameter `provider_name`" in result["error"]
+ repair = result["data"]["authoring_repair_context"]
+ assert repair["reason_code"] == "synthesized_parameter_binding_ambiguous"
+ assert repair["unresolved_names"] == ["provider_name"]
+ assert ctx.workflow_yaml == _SUBMITTED_LITERAL_YAML
+
+ def test_author_schema_incompatibility_does_not_reopen_collapsed_code_block(self) -> None:
+ ctx = _quote_ctx()
+ ctx.update_workflow_called = True
+ ctx.latest_recorded_build_test_outcome = _author_time_reject_outcome("schema_incompatibility")
+ ctx.workflow_yaml = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "old"}}]
+ """
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ records = [{{"quote": "pending"}}]
+ """
)
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert result.violations == []
+ assert result.substitutions is None
+ assert result.workflow_yaml == submitted
+
@pytest.mark.asyncio
async def test_stale_selected_goal_paths_do_not_block_imposed_scout_spine(
self, monkeypatch: pytest.MonkeyPatch
@@ -7161,8 +10035,13 @@ class TestWholeTrajectoryImposition:
assert artifact["claimed_outcomes"][0]["goal_value_paths"] == ["premium", "eligible"]
assert artifact["terminal_verifier_expectations"][0]["goal_value_paths"] == ["premium", "eligible"]
assert artifact["claimed_outcomes"][0]["extraction_schema"] == schema
- code = str(_single_code_block(parse_workflow_yaml(ctx.workflow_yaml))["code"])
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code_blocks = [block for block in workflow_blocks(parsed) if block.get("block_type") == "code"]
+ assert len(code_blocks) > 1
+ code = str(_code_blocks(parsed)[label]["code"])
assert f"return {label}_output" in code
+ assert all(label + "_output" not in str(block.get("code") or "") for block in code_blocks[:-1])
@pytest.mark.asyncio
async def test_invalid_selected_extraction_suffix_keeps_goal_path_rejection(
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index d71955100..7db5221f9 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -435,6 +435,11 @@ class TestSynthesizedOfferPersistenceGate:
assert signal.blocked_tool == "click"
assert signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
assert "last recorded test outcome" in signal.agent_steering_text
+ update_signal = synthesized_block_persistence_signal(ctx, "update_workflow")
+ assert isinstance(update_signal, CopilotToolBlockerSignal)
+ assert update_signal.internal_reason_code == SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE
+ assert update_signal.blocked_tool == "update_workflow"
+ assert update_signal.cleared_by_tools == frozenset({"update_and_run_blocks"})
assert synthesized_block_persistence_signal(ctx, "update_and_run_blocks") is None
assert synthesized_block_persistence_signal(ctx, "evaluate") is None