Skip to content

Automate browser based workflows with AI

feat(SKY-11842): positive independent-evidence binding for requested-output completion (#7090)

Andrew Neilson31c333af5de5
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 454fa610b..b069fa764 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -47,6 +47,7 @@ from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
from skyvern.forge.sdk.copilot.build_test_outcome import (
_VALUE_EXCERPT_MAX,
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
observed_value_extraction_scaffold_lines,
)
@@ -58,6 +59,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
)
from skyvern.forge.sdk.copilot.completion_criteria_store import (
StoredCriteriaSnapshot,
+ apply_requested_output_producer_floor,
build_turn_state,
reconcile_completion_criteria,
)
@@ -410,28 +412,45 @@ def _stored_active_completion_criteria(
return list(snapshot.active.criteria)
+def _log_requested_output_producer_floor(rekeyed_paths: tuple[str, ...]) -> None:
+ if not rekeyed_paths:
+ return
+ LOG.info(
+ "copilot requested-output producer floor",
+ requested_output_floor_rekeyed_paths=list(rekeyed_paths),
+ requested_output_floor_rekeyed_count=len(rekeyed_paths),
+ )
+
+
def _reconcile_completion_criteria_on_context(
ctx: CopilotContext,
policy: RequestPolicy,
policy_inputs: RequestPolicyGuardrailInputs,
) -> None:
+ fresh_criteria = list(policy.completion_criteria)
+ floored_fresh, fresh_floor_rekeyed_paths = apply_requested_output_producer_floor(fresh_criteria)
+ if fresh_floor_rekeyed_paths:
+ policy.completion_criteria = list(floored_fresh)
snapshot = policy_inputs.stored_completion_criteria
if snapshot is None:
+ _log_requested_output_producer_floor(fresh_floor_rekeyed_paths)
return
requested_output_path_aliases = (
ctx.copilot_config.requested_output_path_aliases if ctx.copilot_config is not None else None
)
decision = reconcile_completion_criteria(
snapshot,
- list(policy.completion_criteria),
+ fresh_criteria,
actionable=policy.user_response_policy != "ask_clarification",
requested_output_path_aliases=requested_output_path_aliases,
)
- if decision.action == "adopt_stored":
- policy.completion_criteria = list(decision.criteria)
ctx.completion_criteria_turn_state = build_turn_state(snapshot, decision)
record_criteria_lifecycle(ctx, decision.to_trace_data())
LOG.info("copilot completion criteria reconciled", **decision.to_trace_data())
+ floored_criteria, floor_rekeyed_paths = apply_requested_output_producer_floor(decision.criteria)
+ if decision.action == "adopt_stored" or floor_rekeyed_paths:
+ policy.completion_criteria = list(floored_criteria)
+ _log_requested_output_producer_floor(floor_rekeyed_paths)
def _store_request_policy_on_context(
@@ -892,10 +911,23 @@ def _recorded_build_test_outcome_prompt(ctx: CopilotContext | None) -> str:
lines.append(
f"challenge_controls: {_render_authoring_repair_prompt_list(payload.challenge_control_summaries)}"
)
- lines.append(
- "Before saving or rerunning, change the next authored step, selector, extraction, or binding based on this "
- "recorded structure. Do not re-emit the same plan against the same structural key."
- )
+ binding = ctx.recorded_outcome_binding_constraint
+ if isinstance(binding, RecordedOutcomeBindingConstraint):
+ lines.extend(
+ [
+ "RECORDED OUTCOME BINDING CONSTRAINT:",
+ f"frontier_facet: {_clean_authoring_repair_prompt_atom(binding.frontier_facet)}",
+ f"owning_block_labels: {_render_authoring_repair_prompt_list(binding.owning_block_labels)}",
+ f"diagnostic_reason: {_clean_authoring_repair_prompt_atom(binding.diagnostic_reason)}",
+ "The next authored change must move the named frontier facet on the owning block(s); an unchanged "
+ "frontier is rejected before rerun.",
+ ]
+ )
+ else:
+ lines.append(
+ "Before saving or rerunning, change the next authored step, selector, extraction, or binding based on "
+ "this recorded structure."
+ )
return "\n\n" + "\n".join(line for line in lines if line)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index e55358f1a..7eb389f88 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -167,10 +167,46 @@ class RecordedOutcomeGroundingRequirement(BaseModel):
payload: RecordedOutcomeGroundingPayload | None = None
+BindingFrontierFacet = Literal[
+ "unexecuted_submit",
+ "value_shape",
+ "amend_in_place",
+ "selector_frontier",
+]
+_UNCROSSABLE_DIAGNOSTIC_REASONS = frozenset({"empty_page", "challenge_gated", "capture_degraded"})
+
+
+class RecordedOutcomeBindingConstraint(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ repeated_structural_key: str
+ phase: BuildTestOutcomePhase
+ reason_code: BuildTestOutcomeReasonCode
+ frontier_facet: BindingFrontierFacet
+ owning_block_labels: list[str] = Field(default_factory=list)
+ diagnostic_reason: Literal["none", "empty_page", "challenge_gated", "capture_degraded"] = "none"
+ workflow_run_id: str | None = None
+ recorded_block_signatures: dict[str, str] = Field(default_factory=dict)
+
+ @property
+ def frontier_uncrossable(self) -> bool:
+ return self.diagnostic_reason in _UNCROSSABLE_DIAGNOSTIC_REASONS
+
+ def owning_block_frontier_moved(self, candidate_block_signatures: Mapping[str, str]) -> bool:
+ if not self.owning_block_labels:
+ return True
+ for label in self.owning_block_labels:
+ recorded = self.recorded_block_signatures.get(label)
+ if recorded is None or candidate_block_signatures.get(label) != recorded:
+ return True
+ return False
+
+
class _RecordedBuildTestOutcomeContext(Protocol):
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None
recorded_build_test_outcome_history: list[dict[str, object]]
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None
+ recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None
def record_build_test_outcome(ctx: _RecordedBuildTestOutcomeContext, outcome: RecordedBuildTestOutcome | None) -> None:
@@ -216,6 +252,83 @@ def authored_structure_signature_from_workflow(
return _stable_hash(payload)
+def authored_block_signatures_from_workflow(
+ workflow_yaml: str | None,
+ code_artifact_metadata: object = None,
+) -> dict[str, str]:
+ payload = _authored_structure_payload_from_workflow(workflow_yaml, code_artifact_metadata)
+ if payload is None:
+ return {}
+ signatures: dict[str, str] = {}
+ code_blocks = payload.get("code_blocks")
+ if not isinstance(code_blocks, list):
+ return signatures
+ for block in code_blocks:
+ if not isinstance(block, Mapping):
+ continue
+ label = _safe_str(block.get("label"))
+ if not label:
+ continue
+ signatures[label] = _stable_hash(
+ {
+ "code_hash": block.get("code_hash"),
+ "parameter_keys": block.get("parameter_keys"),
+ "output_metadata": block.get("output_metadata"),
+ }
+ )
+ return signatures
+
+
+def _binding_frontier_facet(outcome: RecordedBuildTestOutcome) -> BindingFrontierFacet:
+ if outcome.reason_code == "scout_act_observe_hollow_after_interaction":
+ return "unexecuted_submit"
+ if outcome.runtime_output_repair_facts:
+ return "amend_in_place"
+ if outcome.missing_requested_output_facts:
+ return "value_shape"
+ if outcome.reason_code in {"sandbox_unresolved_name", "synthesized_parameter_binding_ambiguous"}:
+ return "amend_in_place"
+ if outcome.reason_code in {"outcome_not_demonstrated", "no_meaningful_output", "runtime_missing_output_dependency"}:
+ return "value_shape"
+ return "selector_frontier"
+
+
+def _bind_recorded_outcome_constraint(ctx: object, requirement: RecordedOutcomeGroundingRequirement) -> None:
+ outcome = getattr(ctx, "latest_recorded_build_test_outcome", None)
+ payload = requirement.payload
+ if not isinstance(outcome, RecordedBuildTestOutcome) or payload is None:
+ return
+ owning_labels = _clean_list(
+ outcome.block_labels or ([outcome.attempted_block_label] if outcome.attempted_block_label else [])
+ )
+ recorded_signatures = authored_block_signatures_from_workflow(
+ getattr(ctx, "workflow_yaml", None),
+ getattr(ctx, "code_artifact_metadata", None),
+ )
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=requirement.structural_key,
+ phase=outcome.phase,
+ reason_code=outcome.reason_code,
+ frontier_facet=_binding_frontier_facet(outcome),
+ owning_block_labels=owning_labels,
+ diagnostic_reason=payload.diagnostic_reason,
+ workflow_run_id=requirement.workflow_run_id,
+ recorded_block_signatures={
+ label: recorded_signatures[label] for label in owning_labels if label in recorded_signatures
+ },
+ )
+ ctx.recorded_outcome_binding_constraint = constraint # type: ignore[attr-defined]
+ LOG.info(
+ "copilot recorded outcome binding bound",
+ repeated_structural_key=constraint.repeated_structural_key,
+ frontier_facet=constraint.frontier_facet,
+ owning_block_labels=constraint.owning_block_labels,
+ diagnostic_reason=constraint.diagnostic_reason,
+ frontier_uncrossable=constraint.frontier_uncrossable,
+ workflow_run_id=constraint.workflow_run_id,
+ )
+
+
def latest_recorded_build_test_outcome_repeated(ctx: object) -> bool | None:
history = getattr(ctx, "recorded_build_test_outcome_history", None)
if not isinstance(history, list) or not history:
@@ -290,6 +403,7 @@ def arm_recorded_outcome_grounding_requirement(ctx: object) -> RecordedOutcomeGr
def clear_recorded_outcome_grounding_requirement(ctx: object) -> None:
ctx.recorded_outcome_grounding_requirement = None # type: ignore[attr-defined]
+ ctx.recorded_outcome_binding_constraint = None # type: ignore[attr-defined]
def recorded_outcome_grounding_requires_current_page(ctx: object) -> bool:
@@ -314,9 +428,8 @@ def maybe_satisfy_recorded_outcome_grounding_requirement(ctx: object) -> bool:
if payload is None:
_log_grounding_rejection(requirement, evidence)
return False
- ctx.recorded_outcome_grounding_requirement = requirement.model_copy( # type: ignore[attr-defined]
- update={"satisfied": True, "payload": payload}
- )
+ satisfied_requirement = requirement.model_copy(update={"satisfied": True, "payload": payload})
+ ctx.recorded_outcome_grounding_requirement = satisfied_requirement # type: ignore[attr-defined]
LOG.info(
"copilot recorded outcome grounding satisfied",
structural_key=requirement.structural_key,
@@ -325,6 +438,7 @@ def maybe_satisfy_recorded_outcome_grounding_requirement(ctx: object) -> bool:
observed_after_workflow_run=payload.observed_after_workflow_run,
source_tool=payload.source_tool,
)
+ _bind_recorded_outcome_constraint(ctx, satisfied_requirement)
return True
diff --git a/skyvern/forge/sdk/copilot/completion_criteria_store.py b/skyvern/forge/sdk/copilot/completion_criteria_store.py
index 2b0d9f197..70a22fa04 100644
--- a/skyvern/forge/sdk/copilot/completion_criteria_store.py
+++ b/skyvern/forge/sdk/copilot/completion_criteria_store.py
@@ -8,9 +8,11 @@ stays at the route/repository seam; everything here is side-effect free.
from __future__ import annotations
-from dataclasses import dataclass, field
+from collections.abc import Iterable
+from dataclasses import dataclass, field, replace
from typing import Any, Literal, cast
+from skyvern.forge.sdk.copilot.completion_output_grounding import split_requested_output_criteria
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
run_plane_all_no_evidence,
@@ -363,6 +365,34 @@ def _fresh_generic_rephrase_lacks_stored_requested_outputs(
return bool(stored_requested_tokens - fresh_tokens)
+def apply_requested_output_producer_floor(
+ criteria: Iterable[CompletionCriterion],
+) -> tuple[tuple[CompletionCriterion, ...], tuple[str, ...]]:
+ """Re-key presence-only requested-output criteria (no expected value, shape, or deliverable) to a
+ run-plane outcome so the observed-end-state judge grades them instead of failing closed. Typed
+ value/shape criteria, judgment booleans, and typed deliverables are untouched; the transform is idempotent."""
+ criteria = tuple(criteria)
+ requested, _remaining = split_requested_output_criteria(list(criteria))
+ presence_only_ids = {
+ criterion.id
+ for criterion in requested
+ if criterion.expected_output_value is None
+ and criterion.expected_output_shape is None
+ and criterion.deliverable_kind is None
+ }
+ if not presence_only_ids:
+ return criteria, ()
+ floored: list[CompletionCriterion] = []
+ rekeyed_paths: list[str] = []
+ for criterion in criteria:
+ if criterion.id in presence_only_ids:
+ rekeyed_paths.append(criterion.output_path or "")
+ floored.append(replace(criterion, output_path=None, level="run", kind="outcome"))
+ else:
+ floored.append(criterion)
+ return tuple(floored), tuple(rekeyed_paths)
+
+
def reconcile_completion_criteria(
snapshot: StoredCriteriaSnapshot | None,
fresh: list[CompletionCriterion],
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index ca3525687..a0d69b511 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -1593,6 +1593,15 @@ def _is_hidden_modal_candidate(node: Any) -> bool:
return False
+def _is_css_hidden_node(node: Any) -> bool:
+ if not hasattr(node, "has_attr"):
+ return False
+ if node.has_attr("hidden"):
+ return True
+ style = _attr_value(node, "style").replace(" ", "").lower()
+ return "display:none" in style or "visibility:hidden" in style
+
+
def _modal_dismiss_controls(node: Any) -> list[dict[str, Any]]:
controls: list[dict[str, Any]] = []
seen_selectors: set[str] = set()
@@ -1714,6 +1723,11 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
for node in soup.find_all(["script", "style", "noscript"]):
node.decompose()
+ for node in soup.find_all(True):
+ if node.decomposed:
+ continue
+ if _is_css_hidden_node(node):
+ node.decompose()
visible_text = _node_text(soup.body if getattr(soup, "body", None) is not None else soup)
all_nodes = soup.find_all(True)
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index bb2b4809b..d401f6a2e 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -99,6 +99,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
@@ -636,6 +637,7 @@ class CopilotContext(AgentContext):
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
+ recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = None
# Turn-scoped monotonic marks of verified forward progress: the union of
# completion criteria the judge confirmed satisfied so far this turn, and the
# high-water length of the verified block prefix. A repair that grows either
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 731b9038c..b2c98c5b6 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -37,6 +37,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import CompletionCriteriaTurnState
@@ -252,6 +253,8 @@ class AgentContext:
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
+ recorded_outcome_binding_constraint: RecordedOutcomeBindingConstraint | None = None
+ consecutive_non_converging_repair_count: int = 0
completion_verification_result: CompletionVerificationResult | None = None
completion_criteria_turn_state: CompletionCriteriaTurnState | None = None
verified_terminal_proposal_ready: bool = False
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index 456adab64..a960fd911 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -1488,6 +1488,13 @@ def _tool_loop_error(ctx: AgentContext, tool_name: str, arguments: dict[str, Any
persistence_signal = synthesized_block_persistence_signal(ctx, tool_name)
if persistence_signal is not None:
+ grounding_signal = _recorded_outcome_grounding_signal(ctx, tool_name)
+ if grounding_signal is not None:
+ LOG.info(
+ "copilot recorded outcome grounding enforced over persistence force",
+ tool_name=tool_name,
+ )
+ return _emit_tool_blocker_signal(ctx, grounding_signal)
return _emit_tool_blocker_signal(ctx, persistence_signal)
detected = detect_failed_tool_step_loop_for_ctx(ctx, tool_name, arguments or {})
diff --git a/skyvern/forge/sdk/copilot/tools/completion.py b/skyvern/forge/sdk/copilot/tools/completion.py
index 266a89c96..401a795b6 100644
--- a/skyvern/forge/sdk/copilot/tools/completion.py
+++ b/skyvern/forge/sdk/copilot/tools/completion.py
@@ -1,6 +1,6 @@
import time
from collections.abc import Mapping
-from typing import Any
+from typing import Any, Protocol
import structlog
@@ -73,6 +73,10 @@ from .blockers import (
LOG = structlog.get_logger()
_TYPED_DOWNLOAD_KINDS = frozenset({DOWNLOAD_KIND_REGISTERED, DOWNLOAD_KIND_ATTRIBUTE, DOWNLOAD_KIND_EXTENSION})
+_POST_RUN_PAGE_OBSERVATION_LABEL = "post_run_page_observation"
+# Stamp keys the same-run gate reads; they are dropped from the graded payload so the run id
+# and observation flag cannot be traversed as observed page content.
+_POST_RUN_PAGE_EVIDENCE_STAMP_KEYS = frozenset({"workflow_run_id", "observed_after_workflow_run"})
_REGISTERED_DOWNLOAD_REQUESTED_OUTPUT_PATHS = frozenset(f"output.{key}" for key in REGISTERED_DOWNLOAD_OUTPUT_KEYS)
_AUTHORED_OUTPUT_CONTRACT_CRITERION_ID_PREFIX = "__copilot_authored_output__"
_AUTHORED_OUTPUT_CONTRACT_MISSING_CRITERION_ID = "__copilot_authored_output_contract_missing"
@@ -729,6 +733,44 @@ def _artifact_health_blocker_from_result(
return reason, failed_labels, failure_classes
+class _PostRunPageEvidenceCtx(Protocol):
+ composition_page_evidence: Mapping[str, Any] | None
+
+
+def _same_run_post_run_page_evidence(
+ copilot_ctx: _PostRunPageEvidenceCtx, run_id: str | None
+) -> Mapping[str, Any] | None:
+ """Post-run page evidence stamped for the graded run, admitted only when its own
+ ``workflow_run_id`` matches and it was observed after the run, so a stale pre-run page cannot certify."""
+ if not isinstance(run_id, str) or not run_id:
+ return None
+ evidence = copilot_ctx.composition_page_evidence
+ if not isinstance(evidence, Mapping):
+ return None
+ if evidence.get("observed_after_workflow_run") is not True:
+ return None
+ if evidence.get("workflow_run_id") != run_id:
+ return None
+ return evidence
+
+
+def _bind_independent_post_run_page_evidence(
+ copilot_ctx: _PostRunPageEvidenceCtx,
+ run_id: str | None,
+ block_outputs: dict[str, Any],
+ block_output_sources: dict[str, EvidenceSourceKind],
+) -> None:
+ if _POST_RUN_PAGE_OBSERVATION_LABEL in block_outputs:
+ return
+ evidence = _same_run_post_run_page_evidence(copilot_ctx, run_id)
+ if evidence is None:
+ return
+ block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL] = {
+ key: value for key, value in evidence.items() if key not in _POST_RUN_PAGE_EVIDENCE_STAMP_KEYS
+ }
+ block_output_sources[_POST_RUN_PAGE_OBSERVATION_LABEL] = "independent_page_evidence"
+
+
def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> RunEvidenceSnapshot:
data = result.get("data")
data = data if isinstance(data, dict) else {}
@@ -774,10 +816,13 @@ def _build_run_evidence_snapshot(copilot_ctx: Any, result: dict[str, Any]) -> Ru
else:
block_outputs[registered_block_label] = registered_output_value
block_output_sources[registered_block_label] = "registered_output_parameter"
+ run_id = data.get("workflow_run_id")
+ _bind_independent_post_run_page_evidence(
+ copilot_ctx, run_id if isinstance(run_id, str) else None, block_outputs, block_output_sources
+ )
executed = data.get("executed_block_labels")
executed_block_labels = [str(label) for label in executed] if isinstance(executed, list) else []
page_title = data.get("page_title")
- run_id = data.get("workflow_run_id")
run_terminal_status = data.get("overall_status")
failure_reasons = [" ".join(reason.split()) for reason in iter_failure_reasons(result)]
_artifact_reason, artifact_failed_labels, artifact_failure_classes = _artifact_health_blocker_from_result(
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 2573d6168..c5a0bda0b 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -23,6 +23,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
+ RecordedOutcomeGroundingRequirement,
arm_recorded_outcome_grounding_requirement,
authored_structure_signature_from_workflow,
clear_recorded_outcome_grounding_requirement,
@@ -2834,10 +2835,15 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRe
prior_count = copilot_ctx.consecutive_non_converging_repair_count
count = prior_count + 1 if signature == prior_signature else 1
requirement = copilot_ctx.recorded_outcome_grounding_requirement
- if isinstance(copilot_ctx.latest_recorded_build_test_outcome, RecordedBuildTestOutcome) and not signature.endswith(
- requirement.structural_key if requirement is not None else ""
- ):
- clear_recorded_outcome_grounding_requirement(copilot_ctx)
+ if isinstance(requirement, RecordedOutcomeGroundingRequirement):
+ latest = copilot_ctx.latest_recorded_build_test_outcome
+ current_key = latest.structural_key if isinstance(latest, RecordedBuildTestOutcome) else None
+ if (
+ current_key is None
+ or signature != f"recorded_build_test_outcome:{current_key}"
+ or requirement.structural_key != current_key
+ ):
+ clear_recorded_outcome_grounding_requirement(copilot_ctx)
copilot_ctx.consecutive_non_converging_repair_count = count
copilot_ctx.last_repair_non_convergence_signature = signature
contract.repair_loop_state = RepairLoopState(
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 2f58a68ea..76384a762 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -32,6 +32,8 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestOutcomeReasonCode,
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
+ authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
record_build_test_outcome,
@@ -111,6 +113,7 @@ from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, may
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import (
blocker_signal_is_genuinely_terminal,
+ stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
@@ -1174,10 +1177,14 @@ def _is_unresolved_symbol_repair_context(repair_context: CodeAuthoringRepairCont
_CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_priority_authoring_churn"})
-def _record_code_authoring_guardrail_reject(ctx: AgentContext, *, defer_churn_stop: bool = False) -> None:
+def _record_code_authoring_guardrail_reject(
+ ctx: AgentContext, *, defer_churn_stop: bool = False, frontier_unchanged: bool = False
+) -> None:
# Callers record the current build-test outcome first so repeat detection compares that key to history.
repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
- if repeated_outcome is False:
+ # A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each
+ # turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.
+ if repeated_outcome is False and not frontier_unchanged:
ctx.code_authoring_guardrail_reject_count = 0
ctx.code_authoring_guardrail_reject_count += 1
ctx.last_code_authoring_reject_was_credential_priority = defer_churn_stop
@@ -1513,19 +1520,63 @@ def _metadata_violation_categories(violations: list[str]) -> list[str]:
return categories
-def _unchanged_after_recorded_outcome_signature(
+class _ConvergenceReject(NamedTuple):
+ authored_structure_signature: str
+ reason: Literal["identical_authored_structure", "frontier_unchanged"]
+ commit_early_terminal: bool
+
+
+def _recorded_outcome_convergence_reject(
ctx: AgentContext,
*,
workflow_yaml: str,
code_artifact_metadata: object,
-) -> str | None:
+) -> _ConvergenceReject | None:
latest = ctx.latest_recorded_build_test_outcome
if not isinstance(latest, RecordedBuildTestOutcome) or not latest.is_authoritative:
return None
candidate_signature = authored_structure_signature_from_workflow(workflow_yaml, code_artifact_metadata)
- if candidate_signature is None or candidate_signature != latest.authored_structure_signature:
+ if candidate_signature is None:
+ return None
+ if candidate_signature == latest.authored_structure_signature:
+ return _ConvergenceReject(candidate_signature, "identical_authored_structure", False)
+ constraint = ctx.recorded_outcome_binding_constraint
+ if not isinstance(constraint, RecordedOutcomeBindingConstraint):
+ return None
+ # An author-time reject re-keys `latest` without an executed run, so keep the binding
+ # anchored to its run-outcome key across consecutive frontier-unchanged rejects until a
+ # real run legitimately re-keys it.
+ if latest.phase != "author_time_reject" and constraint.repeated_structural_key != latest.structural_key:
return None
- return candidate_signature
+ candidate_block_signatures = authored_block_signatures_from_workflow(workflow_yaml, code_artifact_metadata)
+ if constraint.owning_block_frontier_moved(candidate_block_signatures):
+ return None
+ return _ConvergenceReject(candidate_signature, "frontier_unchanged", constraint.frontier_uncrossable)
+
+
+def _commit_recorded_outcome_early_terminal(ctx: AgentContext) -> None:
+ constraint = ctx.recorded_outcome_binding_constraint
+ diagnostic_reason = (
+ constraint.diagnostic_reason if isinstance(constraint, RecordedOutcomeBindingConstraint) else "none"
+ )
+ signal = CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text=(
+ f"The recorded build-test outcome names an uncrossable page frontier ({diagnostic_reason}) and the "
+ "frontier block is unchanged; stop retrying and report the recorded blocker from the preserved draft."
+ ),
+ user_facing_reason="I can't get past this page, so I'll stop here and report what I found.",
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code="repair_ceiling_reached",
+ blocked_tool="update_workflow",
+ )
+ stash_blocker_signal(ctx, signal)
+ stash_repair_ceiling_turn_halt(
+ ctx, signal, consecutive_identical_repair_count=ctx.consecutive_non_converging_repair_count
+ )
def _recorded_outcome_requires_output_candidate(ctx: AgentContext) -> bool:
@@ -7408,12 +7459,12 @@ async def _update_workflow(
# New data-write blocks default to surfacing failures rather than swallowing them.
workflow_yaml = default_data_write_continue_on_failure(workflow_yaml, ctx.workflow_yaml)
- authored_structure_signature = _unchanged_after_recorded_outcome_signature(
+ convergence_reject = _recorded_outcome_convergence_reject(
ctx,
workflow_yaml=workflow_yaml,
code_artifact_metadata=getattr(ctx, "code_artifact_metadata", None),
)
- if authored_structure_signature is not None:
+ if convergence_reject is not None:
block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
_record_author_time_reject_outcome(
ctx,
@@ -7421,17 +7472,27 @@ async def _update_workflow(
summary="The authored code and output structure are unchanged after the last recorded test outcome.",
structural_payload={
"reason_code": "unchanged_after_recorded_outcome",
- "authored_structure_signature": authored_structure_signature,
+ "authored_structure_signature": convergence_reject.authored_structure_signature,
"block_labels": block_labels,
},
- authored_structure_signature=authored_structure_signature,
+ authored_structure_signature=convergence_reject.authored_structure_signature,
block_labels=block_labels,
)
- _record_code_authoring_guardrail_reject(ctx)
+ _record_code_authoring_guardrail_reject(
+ ctx, frontier_unchanged=convergence_reject.reason == "frontier_unchanged"
+ )
+ LOG.info(
+ "copilot recorded outcome convergence behavior",
+ convergence_reason=convergence_reject.reason,
+ commit_early_terminal=convergence_reject.commit_early_terminal,
+ block_labels=block_labels,
+ )
+ if convergence_reject.commit_early_terminal:
+ _commit_recorded_outcome_early_terminal(ctx)
return reject(
error=(
- "Submitted workflow is unchanged after the last recorded test outcome. "
- "Revise the code block or output metadata before testing again."
+ "Submitted workflow left the frontier the last recorded test outcome named unchanged. "
+ "Revise the code block or output metadata that owns that frontier before testing again."
),
user_facing_summary=_compiled_authoring_user_summary(),
data=_code_repair_progress_data(),
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 0ad257806..b80c1b4f6 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -937,7 +937,7 @@ workflow_definition:
assert "phase: persisted_block_run" in prompt
assert "reason_code: runtime_block_failure" in prompt
assert "page_evidence_refs: form:Search #search, result:#results rows=unknown" in prompt
- assert "Do not re-emit the same plan against the same structural key" in prompt
+ assert "change the next authored step, selector, extraction, or binding based on" in prompt
def test_recorded_build_test_outcome_prompt_renders_exact_missing_output_paths(self) -> None:
ctx = _ctx(
@@ -6498,6 +6498,7 @@ class TestDeclaredEqualsGradedCompletionCriteria:
verified_prefix_labels=[],
verified_block_outputs={},
post_run_page_observation_after_failed_test=False,
+ composition_page_evidence=None,
completion_criteria_turn_state=None,
)
@@ -6738,6 +6739,7 @@ class TestDeclaredEqualsGradedCompletionCriteria:
verified_prefix_labels=[],
verified_block_outputs={},
post_run_page_observation_after_failed_test=False,
+ composition_page_evidence=None,
completion_criteria_turn_state=None,
)
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index a274bc855..7535e8268 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -4,6 +4,9 @@ from types import SimpleNamespace
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
+ _binding_frontier_facet,
+ authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
observed_value_extraction_scaffold_lines,
@@ -399,6 +402,119 @@ def test_authored_structure_signature_is_stable_and_excludes_raw_code_or_prose()
assert "Find the exact provider" not in str(dumped)
+def test_binding_frontier_facet_derivation_per_reason_code() -> None:
+ def _outcome(**updates: object) -> RecordedBuildTestOutcome:
+ base: dict[str, object] = {
+ "phase": "persisted_block_run",
+ "verdict": "repairable_failure",
+ "reason_code": "runtime_block_failure",
+ "structural_failure_identity": "runtime:x",
+ }
+ base.update(updates)
+ return RecordedBuildTestOutcome(**base) # type: ignore[arg-type]
+
+ assert _binding_frontier_facet(_outcome()) == "selector_frontier"
+ assert _binding_frontier_facet(_outcome(reason_code="sandbox_unresolved_name")) == "amend_in_place"
+ assert _binding_frontier_facet(_outcome(reason_code="synthesized_parameter_binding_ambiguous")) == "amend_in_place"
+ assert _binding_frontier_facet(_outcome(reason_code="outcome_not_demonstrated")) == "value_shape"
+ assert (
+ _binding_frontier_facet(_outcome(reason_code="scout_act_observe_hollow_after_interaction"))
+ == "unexecuted_submit"
+ )
+ assert (
+ _binding_frontier_facet(_outcome(missing_requested_output_facts=[{"output_path": "records[].npi"}]))
+ == "value_shape"
+ )
+ assert (
+ _binding_frontier_facet(_outcome(runtime_output_repair_facts=[{"output_path": "records[].npi"}]))
+ == "amend_in_place"
+ )
+
+
+def test_authored_block_signatures_track_only_owning_block_frontier_movement() -> None:
+ base = """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ parameter_keys:
+ - provider_query
+ code: |
+ return {"records": [{"npi": "123"}]}
+ - block_type: code
+ label: format_output
+ code: |
+ return {"formatted": True}
+ """
+ changed_owning = base.replace('"123"', '"456"')
+ changed_other = base.replace('"formatted": True', '"formatted": False')
+
+ baseline = authored_block_signatures_from_workflow(base, None)
+ assert set(baseline) == {"search_registry", "format_output"}
+
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key="k",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_registry"],
+ recorded_block_signatures={"search_registry": baseline["search_registry"]},
+ )
+
+ assert constraint.owning_block_frontier_moved(baseline) is False
+ assert constraint.owning_block_frontier_moved(authored_block_signatures_from_workflow(changed_other, None)) is False
+ assert constraint.owning_block_frontier_moved(authored_block_signatures_from_workflow(changed_owning, None)) is True
+ assert constraint.owning_block_frontier_moved({}) is True
+
+
+def test_authored_block_signatures_ignore_cosmetic_block_fields() -> None:
+ base = """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ parameter_keys:
+ - provider_query
+ code: |
+ return {"records": [{"npi": "123"}]}
+ """
+ described = base.replace(
+ " label: search_registry\n",
+ " label: search_registry\n description: look up the provider by name\n",
+ )
+ continue_on_failure = base.replace(
+ " label: search_registry\n",
+ " label: search_registry\n continue_on_failure: true\n",
+ )
+ renamed = base.replace("search_registry", "lookup_registry")
+
+ baseline = authored_block_signatures_from_workflow(base, None)
+ assert authored_block_signatures_from_workflow(described, None) == baseline
+ assert authored_block_signatures_from_workflow(continue_on_failure, None) == baseline
+
+ renamed_signatures = authored_block_signatures_from_workflow(renamed, None)
+ assert set(renamed_signatures) == {"lookup_registry"}
+ assert renamed_signatures["lookup_registry"] == baseline["search_registry"]
+
+
+def test_binding_constraint_uncrossable_reflects_diagnostic_reason() -> None:
+ def _constraint(reason: str) -> RecordedOutcomeBindingConstraint:
+ return RecordedOutcomeBindingConstraint(
+ repeated_structural_key="k",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ diagnostic_reason=reason, # type: ignore[arg-type]
+ )
+
+ assert _constraint("none").frontier_uncrossable is False
+ assert _constraint("empty_page").frontier_uncrossable is True
+ assert _constraint("challenge_gated").frontier_uncrossable is True
+ assert _constraint("capture_degraded").frontier_uncrossable is True
+
+
def test_authored_structure_signature_changes_on_code_parameter_or_output_structure() -> None:
base = """
title: Registry lookup
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 3698141a2..5eb9f8180 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -1155,7 +1155,7 @@ class TestCodeRepairProgressClassification:
result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
- assert "unchanged after the last recorded test outcome" in result["error"]
+ assert "left the frontier the last recorded test outcome named unchanged" in result["error"]
assert ctx.workflow_yaml == _code_yaml('await page.goto("https://example.com/old")')
assert ctx.has_staged_proposal is False
assert ctx.latest_recorded_build_test_outcome is not None
diff --git a/tests/unit/test_copilot_completion_criteria_lifecycle.py b/tests/unit/test_copilot_completion_criteria_lifecycle.py
index 6dc16fac7..a4224a726 100644
--- a/tests/unit/test_copilot_completion_criteria_lifecycle.py
+++ b/tests/unit/test_copilot_completion_criteria_lifecycle.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
note_adjudication_on_turn_state,
plan_persistence,
reconcile_completion_criteria,
+ split_requested_output_criteria,
)
from skyvern.forge.sdk.copilot.completion_verification import (
CompletionVerificationResult,
@@ -1088,6 +1089,17 @@ def test_reconcile_on_context_skips_without_snapshot() -> None:
assert ctx.completion_criteria_turn_state is None
+def test_reconcile_on_context_floors_presence_only_requested_output_on_degraded_snapshot() -> None:
+ policy = RequestPolicy(completion_criteria=[_presence_only_requested_output()])
+ ctx = _ctx()
+ _reconcile_completion_criteria_on_context(ctx, policy, _policy_inputs(None))
+ assert ctx.completion_criteria_turn_state is None
+ assert [c.output_path for c in policy.completion_criteria] == [None]
+ assert [c.kind for c in policy.completion_criteria] == ["outcome"]
+ requested, _remaining = split_requested_output_criteria(list(policy.completion_criteria))
+ assert requested == []
+
+
def test_stored_active_criteria_forwarded_only_with_active_set() -> None:
stored = _stored("the item is in the cart")
snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=2)
@@ -1096,6 +1108,83 @@ def test_stored_active_criteria_forwarded_only_with_active_set() -> None:
assert _stored_active_completion_criteria(_policy_inputs(None)) is None
+def _presence_only_requested_output() -> CompletionCriterion:
+ return _criterion(
+ "c_conf",
+ "the confirmation number is returned",
+ output_path="output.confirmation_number",
+ kind="terminal_action",
+ terminal_action_family="request",
+ )
+
+
+def test_reconcile_floors_grading_plane_but_persists_typed_originals() -> None:
+ policy = RequestPolicy(completion_criteria=[_presence_only_requested_output()])
+ ctx = _ctx()
+
+ _reconcile_completion_criteria_on_context(
+ ctx, policy, _policy_inputs(StoredCriteriaSnapshot(active=None, next_epoch=1))
+ )
+
+ assert [c.output_path for c in policy.completion_criteria] == [None]
+ assert [c.kind for c in policy.completion_criteria] == ["outcome"]
+ turn_state = ctx.completion_criteria_turn_state
+ assert turn_state is not None and turn_state.decision is not None
+ assert [c.output_path for c in turn_state.decision.criteria] == ["output.confirmation_number"]
+ assert [c.kind for c in turn_state.decision.criteria] == ["terminal_action"]
+ plan = plan_persistence(turn_state)
+ assert plan is not None and plan.creates_set is True
+ assert [c.output_path for c in plan.create_criteria] == ["output.confirmation_number"]
+ assert [c.kind for c in plan.create_criteria] == ["terminal_action"]
+
+
+def test_reconcile_presence_only_floor_does_not_churn_across_turns() -> None:
+ policy1 = RequestPolicy(completion_criteria=[_presence_only_requested_output()])
+ ctx1 = _ctx()
+ _reconcile_completion_criteria_on_context(
+ ctx1, policy1, _policy_inputs(StoredCriteriaSnapshot(active=None, next_epoch=1))
+ )
+ plan1 = plan_persistence(ctx1.completion_criteria_turn_state)
+ assert plan1 is not None and plan1.create_epoch == 1
+
+ stored = StoredCriteriaSet(set_id="wccs_1", goal_epoch=1, criteria=plan1.create_criteria)
+ policy2 = RequestPolicy(completion_criteria=[_presence_only_requested_output()])
+ ctx2 = _ctx()
+ _reconcile_completion_criteria_on_context(
+ ctx2, policy2, _policy_inputs(StoredCriteriaSnapshot(active=stored, next_epoch=2))
+ )
+
+ turn_state2 = ctx2.completion_criteria_turn_state
+ assert turn_state2 is not None and turn_state2.decision is not None
+ assert turn_state2.decision.action == "adopt_stored"
+ assert turn_state2.decision.reason == "kept"
+ assert plan_persistence(turn_state2) is None
+ assert [c.output_path for c in policy2.completion_criteria] == [None]
+ assert [c.kind for c in policy2.completion_criteria] == ["outcome"]
+
+
+def test_reconcile_clarification_turn_floors_grading_but_never_persists() -> None:
+ presence_only = _presence_only_requested_output()
+ stored = StoredCriteriaSet(set_id="wccs_1", goal_epoch=1, criteria=(presence_only,))
+ policy = RequestPolicy(
+ completion_criteria=[presence_only],
+ user_response_policy="ask_clarification",
+ )
+ ctx = _ctx()
+
+ _reconcile_completion_criteria_on_context(
+ ctx, policy, _policy_inputs(StoredCriteriaSnapshot(active=stored, next_epoch=2))
+ )
+
+ turn_state = ctx.completion_criteria_turn_state
+ assert turn_state is not None and turn_state.decision is not None
+ assert turn_state.decision.action == "adopt_stored"
+ assert [c.output_path for c in policy.completion_criteria] == [None]
+ assert [c.kind for c in policy.completion_criteria] == ["outcome"]
+ assert [c.output_path for c in turn_state.decision.criteria] == ["output.confirmation_number"]
+ assert plan_persistence(turn_state) is None
+
+
def test_plan_persistence_for_supersede_creates_new_epoch_and_points_back() -> None:
stored = _stored("the item is in the cart", set_id="wccs_old", epoch=2)
snapshot = StoredCriteriaSnapshot(active=stored, next_epoch=3)
diff --git a/tests/unit/test_copilot_completion_verification.py b/tests/unit/test_copilot_completion_verification.py
index 6985be0e0..8ed8a5c20 100644
--- a/tests/unit/test_copilot_completion_verification.py
+++ b/tests/unit/test_copilot_completion_verification.py
@@ -16,7 +16,11 @@ from skyvern.forge.sdk.copilot.agent import (
_rewrite_failed_test_response,
_verified_workflow_or_none,
)
-from skyvern.forge.sdk.copilot.completion_criteria_store import criteria_from_json, criteria_to_json
+from skyvern.forge.sdk.copilot.completion_criteria_store import (
+ apply_requested_output_producer_floor,
+ criteria_from_json,
+ criteria_to_json,
+)
from skyvern.forge.sdk.copilot.completion_output_grounding import (
_schema_boolean_output_paths,
_value_matches_expected,
@@ -95,6 +99,7 @@ from skyvern.forge.sdk.copilot.tools import (
_watchdog_exit_allows_terminal_promotion,
)
from skyvern.forge.sdk.copilot.tools.completion import (
+ _POST_RUN_PAGE_OBSERVATION_LABEL,
_artifact_health_blocker_from_result,
_reconcile_download_completion_criterion,
)
@@ -9467,3 +9472,481 @@ def test_requested_output_corroborator_same_record_not_flipped_when_source_is_in
corroborator = _verdict_for(result, "c1__requested_output_corroborator")
assert corroborator.state == "satisfied"
assert corroborator.reason_code == "evidence_confirms"
+
+
+def _post_run_page_evidence(*, run_id: str, visible_text_excerpt: str, **structured: object) -> dict[str, Any]:
+ return {
+ "workflow_run_id": run_id,
+ "observed_after_workflow_run": True,
+ "current_url": "https://example.test/confirmation",
+ "visible_text_excerpt": visible_text_excerpt,
+ **structured,
+ }
+
+
+def test_same_run_post_run_page_evidence_binds_structured_under_reserved_independent_label() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.composition_page_evidence = _post_run_page_evidence(
+ run_id="wr_requested_output",
+ visible_text_excerpt="Confirmation Number WTR-1842-DEMO",
+ confirmation_number="WTR-1842-DEMO",
+ )
+
+ snapshot = _build_run_evidence_snapshot(ctx, _requested_output_result({"note": "ok"}))
+
+ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]
+ assert snapshot.block_output_sources[_POST_RUN_PAGE_OBSERVATION_LABEL] == "independent_page_evidence"
+ assert bound["confirmation_number"] == "WTR-1842-DEMO"
+ assert bound["visible_text_excerpt"] == "Confirmation Number WTR-1842-DEMO"
+ assert "workflow_run_id" not in bound
+ assert "observed_after_workflow_run" not in bound
+ assert snapshot.block_output_sources["extract_profile"] == "runtime_output"
+
+
+def test_same_run_post_run_page_evidence_positive_arm_confirms_reached_value() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ ctx.composition_page_evidence = _post_run_page_evidence(
+ run_id="wr_requested_output",
+ visible_text_excerpt="Confirmation Number WTR-1842-DEMO",
+ confirmation_number="WTR-1842-DEMO",
+ )
+
+ snapshot = _build_run_evidence_snapshot(ctx, _requested_output_result({"note": "ok"}))
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ],
+ snapshot,
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_source == "independent_page_evidence"
+ result = CompletionVerificationResult(
+ status="evaluated", criterion_ids=["c_confirmation_number"], verdicts=verdicts
+ )
+ assert result.is_fully_satisfied() is True
+
+
+def test_post_run_page_evidence_from_a_different_run_is_not_bound() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.composition_page_evidence = _post_run_page_evidence(
+ run_id="wr_stale_prior_run",
+ visible_text_excerpt="Confirmation Number WTR-1842-DEMO",
+ )
+
+ snapshot = _build_run_evidence_snapshot(ctx, _requested_output_result({"note": "ok"}))
+
+ assert _POST_RUN_PAGE_OBSERVATION_LABEL not in snapshot.block_outputs
+ assert _POST_RUN_PAGE_OBSERVATION_LABEL not in snapshot.block_output_sources
+
+
+def test_pre_run_page_evidence_without_post_run_stamp_is_not_bound() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_requested_output",
+ "visible_text_excerpt": "Confirmation Number WTR-1842-DEMO",
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, _requested_output_result({"note": "ok"}))
+
+ assert _POST_RUN_PAGE_OBSERVATION_LABEL not in snapshot.block_outputs
+
+
+def test_independent_page_text_substring_alone_never_confirms_reached_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "visible_text_excerpt": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_source is None
+
+
+def test_generic_static_page_chrome_phrase_never_confirms_on_non_reached_goal() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
+ criteria = [
+ _criterion(
+ "c_status",
+ "The submission status is returned.",
+ output_path="output.status",
+ expected_output_value="Application submitted successfully",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "visible_text_excerpt": "Home Application submitted successfully Contact us",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_source is None
+
+
+def test_independent_page_evidence_text_alone_still_does_not_confirm_authored_prose() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "evidence_text": "Water Service Request Submitted. Confirmation Number WTR-1842-DEMO.",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+
+
+def test_independent_page_text_never_confirms_a_boolean_via_prose() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("passed_validation")
+ criteria = [
+ replace(
+ _criterion(
+ "c_passed_validation",
+ "Validation passed.",
+ output_path="output.passed_validation",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ expected_output_value=True,
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {"visible_text_excerpt": "Status: true. Everything looks true."}
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code != "evidence_confirms"
+
+
+def test_structured_contradiction_masks_the_text_confirmation_door() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "confirmation_number": "WTR-9999-OTHER",
+ "visible_text_excerpt": "Confirmation Number WTR-1842-DEMO",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "evidence_contradicts"
+
+
+def test_registered_artifact_content_confirms_structured_value() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("confirmation_number")
+ criteria = [
+ _criterion(
+ "c_confirmation_number",
+ "The returned record includes confirmation number.",
+ output_path="output.confirmation_number",
+ expected_output_value="WTR-1842-DEMO",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"artifact_content": {"confirmation_number": "WTR-1842-DEMO"}},
+ block_output_sources={"artifact_content": "registered_artifact_content"},
+ ),
+ )
+
+ assert verdicts[0].state == "satisfied"
+ assert verdicts[0].reason_code == "evidence_confirms"
+ assert verdicts[0].evidence_source == "registered_artifact_content"
+
+
+def test_self_emitted_judgment_without_independent_corroborator_still_vetoes() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("selected_highest_priority")
+ criteria = [
+ _criterion(
+ "c_selected",
+ "The highest-priority document was correctly selected.",
+ output_path="output.selected_highest_priority",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={"extract_profile": {"selected_highest_priority": True}},
+ block_output_sources={"extract_profile": "runtime_output"},
+ ),
+ )
+
+ assert verdicts[0].self_emitted_judgment_not_independent is True
+ result = CompletionVerificationResult(status="evaluated", criterion_ids=["c_selected"], verdicts=verdicts)
+ assert result.is_fully_satisfied() is False
+
+
+def test_producer_floor_rekeys_presence_only_requested_output_to_run_outcome() -> None:
+ criteria = [
+ _criterion(
+ "c_presence_only",
+ "The confirmation number is returned.",
+ output_path="output.confirmation_number",
+ )
+ ]
+
+ floored, rekeyed_paths = apply_requested_output_producer_floor(criteria)
+
+ assert rekeyed_paths == ("output.confirmation_number",)
+ assert floored[0].output_path is None
+ assert floored[0].level == "run"
+ assert floored[0].kind == "outcome"
+ assert floored[0].outcome == "The confirmation number is returned."
+ requested, _remaining = split_requested_output_criteria(list(floored))
+ assert requested == []
+
+
+def test_producer_floor_leaves_typed_value_shape_and_judgment_booleans_untouched() -> None:
+ criteria = [
+ _criterion(
+ "c_value",
+ "The NPI is returned.",
+ output_path="output.npi",
+ expected_output_value="1234567890",
+ ),
+ _criterion(
+ "c_shape",
+ "The confirmation number is returned.",
+ output_path="output.confirmation_number",
+ expected_output_shape="reference_code",
+ ),
+ _criterion(
+ "c_bool",
+ "Validation passed.",
+ output_path="output.passed_validation",
+ expected_output_shape="goal_judgment_boolean",
+ requested_output_evidence_source="independent_run_evidence",
+ ),
+ ]
+
+ floored, rekeyed_paths = apply_requested_output_producer_floor(criteria)
+
+ assert rekeyed_paths == ()
+ assert floored == tuple(criteria)
+
+
+def test_producer_floor_is_idempotent() -> None:
+ criteria = [
+ _criterion(
+ "c_presence_only",
+ "The confirmation number is returned.",
+ output_path="output.confirmation_number",
+ )
+ ]
+
+ once, _rekeyed = apply_requested_output_producer_floor(criteria)
+ twice, rekeyed_twice = apply_requested_output_producer_floor(once)
+
+ assert rekeyed_twice == ()
+ assert twice == once
+
+
[email protected]("generic_expected", ["Yes", "Submitted", "Approved", "7", "in progress", "not found"])
+def test_short_generic_expected_value_does_not_confirm_via_page_chrome(generic_expected: str) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("status")
+ criteria = [
+ _criterion(
+ "c_status",
+ "The returned record includes status.",
+ output_path="output.status",
+ expected_output_value=generic_expected,
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "visible_text_excerpt": f"Home Submitted Approved Yes No 7 items {generic_expected}",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code != "evidence_confirms"
+
+
+ "distinctive_expected, page_text",
+ [
+ ("Order 84213 confirmed", "Your Order 84213 confirmed. Thank you for your purchase."),
+ ("INV-2024-001", "Invoice INV-2024-001 has been generated and emailed."),
+ ],
+)
+def test_distinctive_expected_value_in_page_text_alone_no_longer_confirms(
+ distinctive_expected: str, page_text: str
+) -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_for_requested_paths("reference")
+ criteria = [
+ _criterion(
+ "c_reference",
+ "The returned record includes reference.",
+ output_path="output.reference",
+ expected_output_value=distinctive_expected,
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={_POST_RUN_PAGE_OBSERVATION_LABEL: {"visible_text_excerpt": page_text}},
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code == "missing_exact_field"
+ assert verdicts[0].evidence_source is None
+
+
+def test_judgment_declared_string_value_does_not_confirm_via_page_text_door() -> None:
+ ctx = _run_ctx()
+ ctx.code_artifact_metadata = _metadata_with_declared_independent_criterion("select_plan", "output.selected_plan")
+ criteria = [
+ _criterion(
+ "c_selected_plan",
+ "The cheapest plan was selected.",
+ output_path="output.selected_plan",
+ expected_output_value="Plan-Gold-4000",
+ requested_output_evidence_source="runtime_output",
+ )
+ ]
+
+ verdicts = grade_requested_output_criteria(
+ ctx,
+ criteria,
+ RunEvidenceSnapshot(
+ block_outputs={
+ _POST_RUN_PAGE_OBSERVATION_LABEL: {
+ "visible_text_excerpt": "Compare plans: Plan-Silver-2000, Plan-Gold-4000, Plan-Bronze-1000.",
+ }
+ },
+ block_output_sources={_POST_RUN_PAGE_OBSERVATION_LABEL: "independent_page_evidence"},
+ ),
+ )
+
+ assert verdicts[0].state == "unsatisfied"
+ assert verdicts[0].reason_code != "evidence_confirms"
+
+
+def test_bound_post_run_page_evidence_drops_only_stamp_keys() -> None:
+ ctx = _run_ctx()
+ _set_workflow_labels(ctx, "extract_profile")
+ ctx.composition_page_evidence = {
+ "workflow_run_id": "wr_requested_output",
+ "observed_after_workflow_run": True,
+ "current_url": "https://example.test/confirmation",
+ "visible_text_excerpt": "Confirmation Number WTR-1842-DEMO",
+ "status": "shipped",
+ }
+
+ snapshot = _build_run_evidence_snapshot(ctx, _requested_output_result({"note": "ok"}))
+
+ bound = snapshot.block_outputs[_POST_RUN_PAGE_OBSERVATION_LABEL]
+ assert bound == {
+ "current_url": "https://example.test/confirmation",
+ "visible_text_excerpt": "Confirmation Number WTR-1842-DEMO",
+ "status": "shipped",
+ }
+ assert "workflow_run_id" not in bound
+ assert "observed_after_workflow_run" not in bound
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index 5b57fbfa3..bd1d4691f 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -155,6 +155,27 @@ def test_composition_parse_html_extracts_labeled_fields_and_submit_controls() ->
assert parsed["source_tool"] == "inspect_page_for_composition"
+def test_composition_parse_html_excludes_css_hidden_text_from_visible_excerpt() -> None:
+ parsed = parse_composition_html(
+ """
+ <html><body>
+ <p>Visible submission summary</p>
+ <div style="display: none;">Application submitted successfully</div>
+ <span style="visibility:hidden">Hidden confirmation token XYZ</span>
+ <p hidden>Hidden via attribute</p>
+ </body></html>
+ """,
+ inspected_url="https://example.com/results",
+ current_url="https://example.com/results",
+ )
+
+ excerpt = parsed["visible_text_excerpt"]
+ assert "Visible submission summary" in excerpt
+ assert "Application submitted successfully" not in excerpt
+ assert "Hidden confirmation token XYZ" not in excerpt
+ assert "Hidden via attribute" not in excerpt
+
+
def test_composition_parse_html_extracts_modal_overlay_controls() -> None:
parsed = parse_composition_html(
"""
diff --git a/tests/unit/test_copilot_recorded_outcome_grounding.py b/tests/unit/test_copilot_recorded_outcome_grounding.py
index de9fabc86..cb219eb5c 100644
--- a/tests/unit/test_copilot_recorded_outcome_grounding.py
+++ b/tests/unit/test_copilot_recorded_outcome_grounding.py
@@ -9,7 +9,9 @@ from skyvern.forge.sdk.copilot.agent import _recorded_build_test_outcome_prompt
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
+ RecordedOutcomeBindingConstraint,
arm_recorded_outcome_grounding_requirement,
+ authored_block_signatures_from_workflow,
maybe_satisfy_recorded_outcome_grounding_requirement,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
@@ -22,12 +24,64 @@ from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
RepairNextAction,
VerificationResult,
)
+from skyvern.forge.sdk.copilot.enforcement import MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
from skyvern.forge.sdk.copilot.failure_tracking import ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE
from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_module
from skyvern.forge.sdk.copilot.tools.blockers import _tool_loop_error
from skyvern.forge.sdk.copilot.tools.run_execution import _update_repair_loop_state
+from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ _commit_recorded_outcome_early_terminal,
+ _record_author_time_reject_outcome,
+ _record_code_authoring_guardrail_reject,
+ _recorded_outcome_convergence_reject,
+)
+from skyvern.forge.sdk.copilot.turn_halt import TurnHaltKind
from skyvern.forge.sdk.copilot.turn_intent import TurnIntent, TurnIntentAuthority, TurnIntentMode
+_OWNING_BLOCK_WORKFLOW = """
+title: Registry lookup
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_records
+ code: |
+ return {"records": [{"npi": "123"}]}
+"""
+
+_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT = """
+title: Registry lookup
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_records
+ description: Human-facing note adjusted; steps identical
+ code: |
+ return {"records": [{"npi": "123"}]}
+"""
+
+_OWNING_BLOCK_WORKFLOW_RENAMED = """
+title: Registry lookup
+workflow_definition:
+ blocks:
+ - block_type: code
+ label: renamed_records
+ code: |
+ return {"records": [{"npi": "123"}]}
+"""
+
+_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE = """
+title: Registry lookup
+workflow_definition:
+ parameters:
+ - key: added_param
+ parameter_type: workflow
+ blocks:
+ - block_type: code
+ label: search_records
+ code: |
+ return {"records": [{"npi": "123"}]}
+"""
+
def _outcome(**updates: object) -> RecordedBuildTestOutcome:
base = {
@@ -502,3 +556,304 @@ def test_grounding_payload_reaches_recorded_outcome_prompt() -> None:
assert f"repeated_structural_key: {outcome.structural_key}" in prompt
assert "source_tool: inspect_page_for_composition" in prompt
assert "observed_after_workflow_run: false" in prompt
+
+
+def _binding_ctx() -> SimpleNamespace:
+ outcome = _outcome()
+ ctx = _ctx(outcome)
+ ctx.workflow_yaml = _OWNING_BLOCK_WORKFLOW
+ ctx.code_artifact_metadata = None
+ ctx.recorded_outcome_binding_constraint = None
+ return ctx
+
+
+def test_satisfy_binds_typed_constraint_with_frontier_facet_and_owning_blocks() -> None:
+ ctx = _binding_ctx()
+ arm_recorded_outcome_grounding_requirement(ctx)
+ ctx.composition_page_evidence = _bounded_inspect_evidence()
+
+ assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
+
+ constraint = ctx.recorded_outcome_binding_constraint
+ assert isinstance(constraint, RecordedOutcomeBindingConstraint)
+ assert constraint.repeated_structural_key == ctx.latest_recorded_build_test_outcome.structural_key
+ assert constraint.frontier_facet == "selector_frontier"
+ assert constraint.owning_block_labels == ["search_records"]
+ assert constraint.diagnostic_reason == "none"
+ assert constraint.frontier_uncrossable is False
+ assert constraint.recorded_block_signatures == {
+ "search_records": authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
+ }
+
+ prompt = _recorded_build_test_outcome_prompt(ctx) # type: ignore[arg-type]
+ assert "RECORDED OUTCOME BINDING CONSTRAINT:" in prompt
+ assert "frontier_facet: selector_frontier" in prompt
+
+
+def test_degraded_capture_binds_uncrossable_constraint() -> None:
+ ctx = _binding_ctx()
+ arm_recorded_outcome_grounding_requirement(ctx)
+ ctx.composition_page_evidence = _bounded_inspect_evidence(
+ forms=[], result_containers=[], navigation_targets=[], challenge_controls=[]
+ )
+
+ assert maybe_satisfy_recorded_outcome_grounding_requirement(ctx) is True
+
+ constraint = ctx.recorded_outcome_binding_constraint
+ assert isinstance(constraint, RecordedOutcomeBindingConstraint)
+ assert constraint.diagnostic_reason == "capture_degraded"
+ assert constraint.frontier_uncrossable is True
+
+
+def test_convergence_reject_only_when_owning_block_frontier_unchanged() -> None:
+ outcome = _outcome(authored_structure_signature="different_whole_signature")
+ recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=outcome.structural_key or "",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_records"],
+ recorded_block_signatures={"search_records": recorded_sig},
+ )
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_binding_constraint=constraint,
+ )
+
+ unchanged = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
+ )
+ assert unchanged is not None
+ assert unchanged.reason == "frontier_unchanged"
+ assert unchanged.commit_early_terminal is False
+
+ moved_yaml = _OWNING_BLOCK_WORKFLOW.replace('"123"', '"456"')
+ assert _recorded_outcome_convergence_reject(ctx, workflow_yaml=moved_yaml, code_artifact_metadata=None) is None
+
+
+def test_convergence_reject_identical_authored_structure() -> None:
+ signature = run_execution_module.authored_structure_signature_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
+ outcome = _outcome(authored_structure_signature=signature)
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_binding_constraint=None,
+ )
+
+ decision = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
+ )
+ assert decision is not None
+ assert decision.reason == "identical_authored_structure"
+
+
+def test_convergence_reject_uncrossable_frontier_commits_early_terminal() -> None:
+ outcome = _outcome(authored_structure_signature="different_whole_signature")
+ recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=outcome.structural_key or "",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_records"],
+ diagnostic_reason="challenge_gated",
+ recorded_block_signatures={"search_records": recorded_sig},
+ )
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_binding_constraint=constraint,
+ blocker_signal=None,
+ turn_halt=None,
+ consecutive_non_converging_repair_count=2,
+ )
+
+ decision = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None
+ )
+ assert decision is not None
+ assert decision.reason == "frontier_unchanged"
+ assert decision.commit_early_terminal is True
+
+ _commit_recorded_outcome_early_terminal(ctx)
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind == TurnHaltKind.REPAIR_CEILING_REACHED
+ assert ctx.blocker_signal.internal_reason_code == "repair_ceiling_reached"
+ assert ctx.blocker_signal.renders_final_reply is True
+ assert ctx.blocker_signal.preserves_workflow_draft is True
+ assert ctx.blocker_signal.user_facing_reason.strip()
+ assert "can't get past this page" in ctx.blocker_signal.user_facing_reason
+ assert ctx.turn_halt.blocker_signal is ctx.blocker_signal
+
+
+def test_block_signature_keys_on_code_not_label_or_description() -> None:
+ base = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
+ described = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT, None)
+ renamed = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW_RENAMED, None)
+ code_moved = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW.replace('"123"', '"456"'), None)
+
+ assert described["search_records"] == base["search_records"]
+ assert renamed["renamed_records"] == base["search_records"]
+ assert code_moved["search_records"] != base["search_records"]
+
+
+def test_description_only_edit_reads_as_identical_authored_structure() -> None:
+ signature = run_execution_module.authored_structure_signature_from_workflow(_OWNING_BLOCK_WORKFLOW, None)
+ outcome = _outcome(authored_structure_signature=signature)
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_binding_constraint=None,
+ )
+
+ decision = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_DESCRIPTION_EDIT, code_artifact_metadata=None
+ )
+ assert decision is not None
+ assert decision.reason == "identical_authored_structure"
+
+
+def test_frontier_unchanged_reject_fires_when_whole_signature_moves_but_owning_block_holds() -> None:
+ outcome = _outcome(authored_structure_signature="whole_signature_from_prior_attempt")
+ recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=outcome.structural_key or "",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_records"],
+ recorded_block_signatures={"search_records": recorded_sig},
+ )
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=outcome,
+ recorded_outcome_binding_constraint=constraint,
+ )
+
+ decision = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE, code_artifact_metadata=None
+ )
+ assert decision is not None
+ assert decision.reason == "frontier_unchanged"
+ assert decision.commit_early_terminal is False
+
+
+def test_binding_enforces_across_consecutive_author_time_rejects() -> None:
+ run_outcome = _outcome(authored_structure_signature="run_whole_signature")
+ recorded_sig = authored_block_signatures_from_workflow(_OWNING_BLOCK_WORKFLOW, None)["search_records"]
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=run_outcome.structural_key or "",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_records"],
+ recorded_block_signatures={"search_records": recorded_sig},
+ )
+ ctx = SimpleNamespace(
+ latest_recorded_build_test_outcome=run_outcome,
+ recorded_outcome_binding_constraint=constraint,
+ )
+
+ first = _recorded_outcome_convergence_reject(ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW, code_artifact_metadata=None)
+ assert first is not None
+ assert first.reason == "frontier_unchanged"
+
+ # The first reject re-keys `latest` to an author-time reject outcome whose structural key
+ # differs from the bound run outcome; the binding must still enforce on the next
+ # different-structure-but-frontier-unchanged submit.
+ author_time_latest = _outcome(
+ phase="author_time_reject",
+ reason_code="unchanged_after_recorded_outcome",
+ structural_failure_identity="author_time:rekeyed",
+ authored_structure_signature="author_time_reject_whole_signature",
+ )
+ assert author_time_latest.structural_key != constraint.repeated_structural_key
+ ctx.latest_recorded_build_test_outcome = author_time_latest
+
+ second = _recorded_outcome_convergence_reject(
+ ctx, workflow_yaml=_OWNING_BLOCK_WORKFLOW_SIBLING_MOVE, code_artifact_metadata=None
+ )
+ assert second is not None
+ assert second.reason == "frontier_unchanged"
+
+
+def test_sibling_churn_frontier_unchanged_rejects_reach_honest_churn_stop() -> None:
+ ctx = _ctx()
+
+ for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS):
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="unchanged_after_recorded_outcome",
+ summary="The authored code and output structure are unchanged after the last recorded test outcome.",
+ structural_payload={
+ "reason_code": "unchanged_after_recorded_outcome",
+ "authored_structure_signature": f"moving_whole_signature_{index}",
+ "block_labels": ["search_records"],
+ },
+ authored_structure_signature=f"moving_whole_signature_{index}",
+ block_labels=["search_records"],
+ )
+ _record_code_authoring_guardrail_reject(ctx, frontier_unchanged=True)
+
+ # Sibling churn moves the whole-signature key every turn, so each recorded outcome reads as a
+ # non-repeat; the frontier-unchanged flag keeps the churn counter climbing to the honest stop
+ # instead of resetting and riding to max_turns.
+ assert (
+ ctx.recorded_build_test_outcome_history[-1]["structural_key"]
+ != ctx.recorded_build_test_outcome_history[-2]["structural_key"]
+ )
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == "code_authoring_guardrail_churn"
+
+
+def test_early_terminal_renders_typed_final_reply_and_preserves_draft() -> None:
+ constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key="repeated_key",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ owning_block_labels=["search_records"],
+ diagnostic_reason="challenge_gated",
+ )
+ ctx = SimpleNamespace(
+ recorded_outcome_binding_constraint=constraint,
+ blocker_signal=None,
+ turn_halt=None,
+ consecutive_non_converging_repair_count=3,
+ )
+
+ _commit_recorded_outcome_early_terminal(ctx)
+
+ signal = ctx.blocker_signal
+ assert signal.renders_final_reply is True
+ assert signal.recovery_hint == "report_blocker_to_user"
+ assert signal.blocked_tool == "update_workflow"
+ assert signal.cleared_by_tools == frozenset()
+ assert signal.preserves_workflow_draft is True
+ assert signal.internal_reason_code == "repair_ceiling_reached"
+ assert "can't get past this page" in signal.user_facing_reason
+ assert signal.internal_reason_code not in signal.user_facing_reason
+
+ assert ctx.turn_halt.kind == TurnHaltKind.REPAIR_CEILING_REACHED
+ assert ctx.turn_halt.blocker_signal is signal
+ assert ctx.turn_halt.draft_state == {"preserves_workflow_draft": True}
+ assert ctx.turn_halt.extra["consecutive_identical_repair_count"] == 3
+
+
+def test_update_repair_loop_state_clears_stale_requirement_when_outcome_not_authoritative() -> None:
+ outcome = _outcome()
+ ctx = _ctx(outcome)
+ arm_recorded_outcome_grounding_requirement(ctx)
+ assert ctx.recorded_outcome_grounding_requirement is not None
+ ctx.recorded_outcome_binding_constraint = RecordedOutcomeBindingConstraint(
+ repeated_structural_key=outcome.structural_key or "",
+ phase="persisted_block_run",
+ reason_code="runtime_block_failure",
+ frontier_facet="selector_frontier",
+ )
+
+ ctx.latest_recorded_build_test_outcome = None
+ ctx.recorded_build_test_outcome_history = []
+
+ _update_repair_loop_state(ctx, _contract())
+
+ assert ctx.recorded_outcome_grounding_requirement is None
+ assert ctx.recorded_outcome_binding_constraint is None