Skip to content

Automate browser based workflows with AI

Converge Copilot author-time metadata-gate rejects to a typed terminal (SKY-12594) (#7527)

Andrew Neilson6ce4735cc1d6
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index f6d4ce814..0f684af0d 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -167,18 +167,33 @@ def build_output_source_unobservable_blocker_signal(
"visits, so there is nothing for the workflow to read them from. I've kept the draft; "
"tell me where those values appear and I'll wire them up."
)
+ elif reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:
+ user_facing = (
+ "I rewrote this workflow several times to return the value(s) you asked for"
+ f"{f' ({path_text})' if path_text else ''}, but each version left those outputs "
+ "undeclared or unreturned, so the workflow can't reliably hand them back. I've kept "
+ "the draft; tell me where each of those values appears and I'll declare and return them."
+ )
else:
user_facing = (
"I couldn't shape this workflow so it reliably returns the values you asked for"
f"{f' ({path_text})' if path_text else ''}. I've kept the current draft; let me know "
"where those values show up and I'll try a different structure."
)
- agent_steer = (
- "STOP: the requested output contract has no observable extraction source in the current "
- f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "
- "re-authoring the same draft. Report the missing source to the user and ask where the "
- "values appear. The prior draft is preserved; do not rerun the blocks."
- )
+ if reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:
+ agent_steer = (
+ "STOP: the output contract kept failing its declaration/return coverage across repeated "
+ f"re-authored candidates for required path(s) [{path_text or '(unknown)'}]. This is not "
+ "repairable by re-authoring the same draft. Report the missing output declarations to the "
+ "user and ask where the values appear. The prior draft is preserved; do not rerun the blocks."
+ )
+ else:
+ agent_steer = (
+ "STOP: the requested output contract has no observable extraction source in the current "
+ f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "
+ "re-authoring the same draft. Report the missing source to the user and ask where the "
+ "values appear. The prior draft is preserved; do not rerun the blocks."
+ )
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
agent_steering_text=agent_steer,
@@ -379,8 +394,13 @@ UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE = "tool_error_uncovered_output_rescou
RECORDED_OUTCOME_GROUNDING_REASON_CODE = "recorded_outcome_grounding_required"
DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE = "definition_contract_unsatisfied"
SCHEMA_INCOMPATIBILITY_REASON_CODE = "schema_incompatibility"
+OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE = "output_contract_reject_budget_exhausted"
_OUTPUT_CONTRACT_TERMINAL_REASON_CODES = frozenset(
- {OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE}
+ {
+ OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
+ OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
+ }
)
# A held blocker whose reason code is in this set must win both the rendered reply and the typed
@@ -396,6 +416,7 @@ GENUINELY_TERMINAL_BLOCKER_REASON_CODES: frozenset[str] = frozenset(
SCHEMA_INCOMPATIBILITY_REASON_CODE,
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
"advisory_dispatch_stalled",
DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,
"repair_ceiling_reached",
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 07efbd1f7..fb4b958e0 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -172,6 +172,11 @@ class PreRunPageReference:
workflow_run_id: str
+@dataclass(frozen=True)
+class RejectedCodeArtifactMetadataCapture:
+ payload: Any
+
+
@dataclass(frozen=True)
class RegisteredArtifactEntry:
artifact_id: str
@@ -393,6 +398,8 @@ class AgentContext:
# only normalizes and carries the metadata; sufficiency checks live elsewhere.
code_artifact_metadata: dict[str, CodeArtifactMetadataPayload] = field(default_factory=dict)
raw_code_artifact_metadata: object | None = None
+ submitted_code_artifact_metadata_snapshot: Any = None
+ rejected_code_artifact_metadata_captures: list[RejectedCodeArtifactMetadataCapture] = field(default_factory=list)
# Hydrated at turn start from StructuredContext.observed_acted_pages; lets the
# composition gate credit a page observed on a prior turn when this turn's
# flow_evidence does not cover it (closes the spent-inspection-budget
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 175887b59..4ca37a843 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -28,7 +28,9 @@ from skyvern.forge.sdk.api.llm.schema_validator import validate_schema
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
+ blocker_signal_is_genuinely_terminal,
build_definition_contract_unsatisfied_blocker_signal,
build_output_source_unobservable_blocker_signal,
clear_terminal_evidence_on_workflow_edit,
@@ -176,6 +178,7 @@ from skyvern.forge.sdk.copilot.runtime import (
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
AgentContext,
AuthorTimeGateAblationPayload,
+ RejectedCodeArtifactMetadataCapture,
ScoutedFieldParameterBinding,
ScoutedInteraction,
ScoutedSubmitRungBinding,
@@ -2549,9 +2552,22 @@ _SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE = "separated_spine_shape_required"
_SEPARATED_BROWSER_SPINE_PLUS_EXTRACTION_STRUCTURE = "separated_browser_spine_plus_extraction"
_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"
_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE = "value_bearing_output_required"
-_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = "output_contract_reject_budget_exhausted"
+_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH = "output"
+_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
_MAX_OUTPUT_CONTRACT_REJECTS = 4
_MAX_OUTPUT_CONTRACT_DEFERRALS = 3
+_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY = "missing_code_artifact_metadata"
+_METADATA_NORMALIZATION_REJECT_FAMILY = "metadata_normalization"
+# Families sharing the reject-budget ladder: a candidate rotating its structural fingerprint within the
+# family must not reset its streak, though an imposition landing still does (definition is already terminal).
+_METADATA_FAMILY_REJECT_FAMILIES = frozenset(
+ {
+ _OUTPUT_CONTRACT_REJECT_REASON_CODE,
+ _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ _MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
+ _METADATA_NORMALIZATION_REJECT_FAMILY,
+ }
+)
_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3
_OUTPUT_CONTRACT_ABLATION_GATE_ID = "output_contract_actuation"
_METADATA_PREFLIGHT_ABLATION_GATE_ID = "metadata_run_preflight_reject"
@@ -3751,12 +3767,17 @@ def _record_output_contract_reject(
authored_structural_fingerprint: str = "",
workflow_yaml: str = "",
) -> dict[str, Any]:
+ # A candidate so incomplete that no required child paths are derivable must still consume the
+ # reject budget: an empty set would silently skip counting and degrade the turn to the generic
+ # churn stop (loop_detected) instead of this ladder's typed terminal.
+ counted_paths = evaluation.required_paths or {_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH}
count = _record_output_contract_family_reject(
ctx,
- evaluation.required_paths,
+ counted_paths,
reject_family=str(evaluation.payload.get("reason_code") or _OUTPUT_CONTRACT_REJECT_REASON_CODE),
authored_structural_fingerprint=authored_structural_fingerprint,
)
+ _capture_rejected_code_artifact_metadata(ctx)
payload = dict(evaluation.payload)
payload["output_contract_reject_count"] = count
payload["output_contract_reject_budget"] = _MAX_OUTPUT_CONTRACT_REJECTS
@@ -3778,30 +3799,9 @@ def _record_output_contract_reject(
# The typed adjudication now owns this same deficiency. Keep the historical reject for
# ceiling/liveness evidence, but do not render it again as a fresh rejection next turn.
record_build_test_outcome(ctx, None)
- if count >= _MAX_OUTPUT_CONTRACT_REJECTS:
- if run_backed_repair_evidence_exists(ctx):
- payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- else:
- deferral_count = _record_output_contract_deferral(ctx, evaluation.required_paths)
- if deferral_count >= _MAX_OUTPUT_CONTRACT_DEFERRALS:
- payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
- LOG.info(
- "copilot_output_contract_budget_deferral_cap_reached",
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- output_contract_reject_count=count,
- output_contract_deferral_count=deferral_count,
- )
- else:
- LOG.info(
- "copilot_output_contract_budget_rewrite_deferred_no_run",
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- output_contract_reject_count=count,
- output_contract_deferral_count=deferral_count,
- )
+ if _adjudicate_output_contract_budget(ctx, counted_paths, count=count, block_label=evaluation.block_label):
+ payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
+ payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE
if actuation is None or actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE:
structural_payload = _output_contract_author_time_structural_payload(
ctx,
@@ -3818,7 +3818,8 @@ def _record_output_contract_reject(
if isinstance(payload.get("missing_requested_output_facts"), list)
else None,
)
- _record_code_authoring_guardrail_reject(ctx)
+ if count < 1:
+ _record_code_authoring_guardrail_reject(ctx)
return payload
@@ -3846,7 +3847,9 @@ def _record_output_contract_family_reject(
if authored_structural_fingerprint:
prior_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)
imposed_since = ctx.output_contract_imposed_since_last_reject_by_signature.get(signature, False)
- if imposed_since or (prior_fingerprint is not None and prior_fingerprint != authored_structural_fingerprint):
+ fingerprint_rotated = prior_fingerprint is not None and prior_fingerprint != authored_structural_fingerprint
+ rotation_resets = fingerprint_rotated and reject_family not in _METADATA_FAMILY_REJECT_FAMILIES
+ if imposed_since or rotation_resets:
count_by_signature[signature] = 0
ctx.output_contract_imposed_since_last_reject_by_signature[signature] = False
LOG.info(
@@ -3882,6 +3885,80 @@ def _record_output_contract_deferral(ctx: AgentContext, required_paths: set[str]
return count
+def _adjudicate_output_contract_budget(
+ ctx: AgentContext,
+ required_paths: set[str],
+ *,
+ count: int,
+ block_label: str,
+) -> bool:
+ if count < _MAX_OUTPUT_CONTRACT_REJECTS:
+ return False
+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
+ if not run_backed_repair_evidence_exists(ctx):
+ deferral_count = _record_output_contract_deferral(ctx, required_paths)
+ if deferral_count < _MAX_OUTPUT_CONTRACT_DEFERRALS:
+ LOG.info(
+ "copilot_output_contract_budget_rewrite_deferred_no_run",
+ block_label=block_label,
+ canonical_output_contract_signature=signature,
+ output_contract_reject_count=count,
+ output_contract_deferral_count=deferral_count,
+ )
+ return False
+ LOG.info(
+ "copilot_output_contract_budget_deferral_cap_reached",
+ block_label=block_label,
+ canonical_output_contract_signature=signature,
+ output_contract_reject_count=count,
+ output_contract_deferral_count=deferral_count,
+ )
+ _stash_output_contract_reject_budget_terminal(
+ ctx, required_paths=required_paths, block_label=block_label, signature=signature
+ )
+ return True
+
+
+def _stash_output_contract_reject_budget_terminal(
+ ctx: AgentContext,
+ *,
+ required_paths: set[str],
+ block_label: str,
+ signature: str,
+) -> None:
+ if ctx.turn_halt is not None or blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
+ return
+ signal = build_output_source_unobservable_blocker_signal(
+ reason_code=_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE,
+ required_paths=required_paths,
+ block_label=block_label,
+ )
+ stash_blocker_signal(ctx, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
+ LOG.info(
+ "copilot_output_contract_reject_budget_terminal",
+ block_label=block_label,
+ canonical_output_contract_signature=signature,
+ canonical_required_child_paths=sorted(required_paths),
+ )
+
+
+def _capture_rejected_code_artifact_metadata(ctx: AgentContext) -> None:
+ snapshot = ctx.submitted_code_artifact_metadata_snapshot
+ if snapshot is None:
+ return
+ captures = ctx.rejected_code_artifact_metadata_captures
+ captures.append(RejectedCodeArtifactMetadataCapture(payload=snapshot))
+ del captures[:-5]
+ LOG.info(
+ "copilot_rejected_code_artifact_metadata_captured",
+ capture_count=len(captures),
+ submitted_metadata_labels=sorted(snapshot) if isinstance(snapshot, dict) else None,
+ submitted_metadata_type=type(snapshot).__name__,
+ rejected_code_artifact_metadata_payload=snapshot,
+ )
+
+
def _output_contract_reject_result(
evaluation: _OutputContractEvaluation,
*,
@@ -5405,6 +5482,8 @@ def _metadata_contract_run_preflight_reject(
*,
enforce_untagged_declared_inputs: bool = False,
) -> dict[str, Any] | None:
+ if isinstance(ctx, AgentContext):
+ ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(raw_code_artifact_metadata)
definition_reject = (
_definition_plane_preflight_reject(
ctx,
@@ -11216,6 +11295,7 @@ async def _update_workflow(
ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
+ ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(params.get("code_artifact_metadata"))
# Imposition reconciles synthesized aliases/parameters before the persisted YAML contract is checked.
_enrich_scout_trajectory_input_correspondences(workflow_yaml, ctx)
runtime_parameters = params.get("parameters")
@@ -11551,10 +11631,10 @@ async def _update_workflow(
source=output_path_coverage_source,
reason_code=output_path_coverage_reason_code,
)
- _record_output_contract_family_reject(
+ missing_metadata_reject_count = _record_output_contract_family_reject(
ctx,
required_child_output_paths,
- reject_family="missing_code_artifact_metadata",
+ reject_family=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
)
_record_author_time_reject_outcome(
ctx,
@@ -11575,6 +11655,7 @@ async def _update_workflow(
block_labels=missing_labels,
missing_requested_output_facts=missing_metadata_output_facts,
)
+ _capture_rejected_code_artifact_metadata(ctx)
credential_scout_errors = (
[]
if _request_policy_allows_untested_code_block_draft(ctx)
@@ -11584,7 +11665,16 @@ async def _update_workflow(
block_labels=params.get("block_labels"),
)
)
- _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=bool(credential_scout_errors))
+ budget_terminal = _adjudicate_output_contract_budget(
+ ctx,
+ required_child_output_paths,
+ count=missing_metadata_reject_count,
+ block_label=missing_labels[0] if len(missing_labels) == 1 else "",
+ )
+ if credential_scout_errors:
+ _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
+ elif missing_metadata_reject_count < 1 and not budget_terminal:
+ _record_code_authoring_guardrail_reject(ctx)
return reject(
error=missing_metadata_error,
user_facing_summary=_compiled_authoring_user_summary(),
@@ -11613,10 +11703,10 @@ async def _update_workflow(
code_artifact_metadata_error = normalization.error
if code_artifact_metadata_error is not None:
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
- _record_output_contract_family_reject(
+ normalization_reject_count = _record_output_contract_family_reject(
ctx,
required_child_output_paths,
- reject_family="metadata_normalization",
+ reject_family=_METADATA_NORMALIZATION_REJECT_FAMILY,
)
_record_author_time_reject_outcome(
ctx,
@@ -11635,6 +11725,13 @@ async def _update_workflow(
),
block_labels=normalization.offending_labels,
)
+ _capture_rejected_code_artifact_metadata(ctx)
+ _adjudicate_output_contract_budget(
+ ctx,
+ required_child_output_paths,
+ count=normalization_reject_count,
+ block_label=normalization.offending_labels[0] if len(normalization.offending_labels) == 1 else "",
+ )
if normalization.schema_incompatibilities:
incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)
if incompatibility is not None:
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index a60372046..dd333effe 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
GENUINELY_TERMINAL_BLOCKER_REASON_CODES as GENUINELY_TERMINAL_BLOCKER_REASON_CODES,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
blocker_signal_is_genuinely_terminal,
build_output_source_unobservable_blocker_signal,
@@ -95,6 +96,7 @@ _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset(
{
OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,
OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
ADVISORY_DISPATCH_STALLED_REASON_CODE,
}
)
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 351915d3a..8b5d59c0b 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -17,8 +17,10 @@ from structlog.testing import capture_logs
from skyvern.config import settings
from skyvern.forge.sdk.copilot import enforcement
from skyvern.forge.sdk.copilot.blocker_signal import (
+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
+ blocker_signal_is_genuinely_terminal,
build_output_source_unobservable_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome
@@ -46,6 +48,7 @@ from skyvern.forge.sdk.copilot.request_slots import (
)
from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_page
from skyvern.forge.sdk.copilot.runtime import (
+ RejectedCodeArtifactMetadataCapture,
output_contract_ladder_unresolved,
record_author_time_gate_ablation_event,
)
@@ -553,6 +556,8 @@ def _advisory_ctx() -> SimpleNamespace:
output_contract_run_output_observed_by_signature={},
output_contract_run_bound_required_path_by_signature={},
output_contract_bail_actuated_this_call=False,
+ submitted_code_artifact_metadata_snapshot=None,
+ rejected_code_artifact_metadata_captures=[],
author_time_gate_log_only_ids=frozenset(),
author_time_gate_ablation_events=[],
latest_recorded_build_test_outcome=None,
@@ -2205,3 +2210,280 @@ def test_observation_production_is_not_click_only_spine() -> None:
code = 'await page.click("#submit")\nreturn {"output": {"record_id": "x"}}'
assert wu._output_contract_click_only_spine(code, {"output.blocker"}) is False
+
+
+_METADATA_REQUIRED = {"output.confirmation_number"}
+
+
+def test_metadata_family_reject_holds_streak_across_fingerprint_rotation() -> None:
+ ctx = _counter_ctx()
+ for family in ("missing_code_artifact_metadata", "metadata_normalization", "output_contract_required"):
+ sub = _counter_ctx()
+ counts = [
+ wu._record_output_contract_family_reject(
+ sub, _METADATA_REQUIRED, reject_family=family, authored_structural_fingerprint=f"fp_{i}"
+ )
+ for i in range(4)
+ ]
+ assert counts == [1, 2, 3, 4], family
+ imposed = wu._record_output_contract_family_reject(
+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_0"
+ )
+ signature = next(iter(ctx.output_contract_reject_count_by_signature))
+ ctx.output_contract_imposed_since_last_reject_by_signature[signature] = True
+ after_imposition = wu._record_output_contract_family_reject(
+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_1"
+ )
+ assert (imposed, after_imposition) == (1, 1)
+
+
+def test_out_of_family_reject_resets_on_fingerprint_rotation() -> None:
+ ctx = _counter_ctx()
+ first = wu._record_output_contract_family_reject(
+ ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_a"
+ )
+ rotated = wu._record_output_contract_family_reject(
+ ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_b"
+ )
+ assert (first, rotated) == (1, 1)
+
+
+def test_reject_budget_exhausted_reason_code_is_genuinely_terminal() -> None:
+ signal = build_output_source_unobservable_blocker_signal(
+ reason_code=OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
+ required_paths=_METADATA_REQUIRED,
+ block_label="collect",
+ )
+ assert blocker_signal_is_genuinely_terminal(signal) is True
+
+
+def test_metadata_family_budget_reaches_typed_terminal_at_bound_without_churn() -> None:
+ ctx = make_copilot_ctx(turn_id="turn_budget_bound")
+ terminal_at = None
+ for candidate in range(1, 7):
+ count = wu._record_output_contract_family_reject(
+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata"
+ )
+ assert count == candidate
+ exhausted = wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=count, block_label="collect")
+ if exhausted and terminal_at is None:
+ terminal_at = candidate
+ assert terminal_at == 6
+ assert ctx.turn_halt is not None
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
+ assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True
+ assert ctx.code_authoring_guardrail_reject_count == 0
+
+
+def test_budget_terminal_names_paths_and_omits_generic_churn_string() -> None:
+ ctx = make_copilot_ctx(turn_id="turn_budget_copy")
+ wu._stash_output_contract_reject_budget_terminal(
+ ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_copy"
+ )
+ assert ctx.blocker_signal is not None
+ user_facing = ctx.blocker_signal.user_facing_reason
+ assert_clean_user_facing_text(user_facing)
+ assert "output.confirmation_number" in user_facing
+ assert "safety checks rejected each version" not in user_facing
+ assert ctx.blocker_signal.renders_final_reply is True
+
+
+def test_main_gate_defers_churn_to_budget_ladder_when_reject_counted(monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = make_copilot_ctx(turn_id="turn_budget_defer")
+ calls: list[object] = []
+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))
+ count = wu._record_output_contract_reject(
+ ctx,
+ _make_evaluation("", block_label="collect"),
+ summary="Typed deficiency.",
+ authored_structural_fingerprint="fp_defer",
+ workflow_yaml=_PAGE_READ_YAML,
+ )
+ assert count["output_contract_reject_count"] >= 1
+ assert calls == []
+
+
+def test_undeclared_contract_rejects_count_toward_budget_and_reach_typed_terminal(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # A candidate deriving NO required child paths must still consume the reject budget: skipping
+ # the count degrades repeated preflight rejects to the generic churn stop (loop_detected).
+ ctx = make_copilot_ctx(turn_id="turn_budget_undeclared")
+ calls: list[object] = []
+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))
+ malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}
+ ctx.submitted_code_artifact_metadata_snapshot = malformed
+
+ def undeclared_evaluation() -> wu._OutputContractEvaluation:
+ return wu._OutputContractEvaluation(
+ block_label="collect",
+ artifact_id="",
+ required_paths=set(),
+ observation_paths=set(),
+ declaration_paths=set(),
+ source="requested_output",
+ reason_code="output_contract_required",
+ missing_metadata_paths=[],
+ missing_schema_paths=[],
+ missing_return_paths=[],
+ shape_violations=[],
+ canonical_signature="",
+ payload={},
+ repair_context=None,
+ )
+
+ terminal_at = None
+ for candidate in range(1, 8):
+ payload = wu._record_output_contract_reject(
+ ctx,
+ undeclared_evaluation(),
+ summary="Undeclared contract deficiency.",
+ authored_structural_fingerprint=f"fp_{candidate}",
+ workflow_yaml="",
+ )
+ assert payload["output_contract_reject_count"] == candidate
+ if ctx.turn_halt is not None and terminal_at is None:
+ terminal_at = candidate
+ break
+ assert terminal_at is not None and terminal_at <= 7
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
+ assert calls == []
+ assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)
+
+
+def test_recorded_identical_reject_shape_replays_to_budget_terminal_with_per_reject_capture(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Replays the recorded live churn shape — identical value_bearing_output_required rejects on a
+ # candidate deriving no required paths — through the real seam to budget exhaustion.
+ ctx = make_copilot_ctx(turn_id="turn_recorded_shape_replay")
+ churn_calls: list[object] = []
+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: churn_calls.append(_ctx))
+ malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}
+ ctx.submitted_code_artifact_metadata_snapshot = malformed
+
+ def recorded_shape_evaluation() -> wu._OutputContractEvaluation:
+ return wu._OutputContractEvaluation(
+ block_label="collect",
+ artifact_id="",
+ required_paths=set(),
+ observation_paths=set(),
+ declaration_paths=set(),
+ source="requested_output",
+ reason_code=wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,
+ missing_metadata_paths=[],
+ missing_schema_paths=[],
+ missing_return_paths=[],
+ shape_violations=[wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE],
+ canonical_signature="",
+ payload={"reason_code": wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE},
+ repair_context=None,
+ )
+
+ terminal_at = None
+ for candidate in range(1, 8):
+ payload = wu._record_output_contract_reject(
+ ctx,
+ recorded_shape_evaluation(),
+ summary="Identical value-bearing deficiency.",
+ authored_structural_fingerprint="fp_identical",
+ workflow_yaml="",
+ )
+ assert payload["output_contract_reject_count"] == candidate
+ assert len(ctx.rejected_code_artifact_metadata_captures) == min(candidate, 5)
+ if ctx.turn_halt is not None:
+ terminal_at = candidate
+ break
+ assert terminal_at is not None and terminal_at <= 7
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE
+ assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True
+ assert churn_calls == []
+
+
+def test_no_reject_leaves_no_budget_terminal() -> None:
+ ctx = make_copilot_ctx(turn_id="turn_budget_yes")
+ assert wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=0, block_label="collect") is False
+ assert ctx.turn_halt is None
+ assert ctx.blocker_signal is None
+
+
+def test_rejected_code_artifact_metadata_capture_is_byte_exact_and_bounded() -> None:
+ ctx = make_copilot_ctx(turn_id="turn_capture")
+ malformed = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}
+ ctx.submitted_code_artifact_metadata_snapshot = malformed
+ wu._capture_rejected_code_artifact_metadata(ctx)
+ assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)
+ for index in range(7):
+ ctx.submitted_code_artifact_metadata_snapshot = {"collect": {"variant": index}}
+ wu._capture_rejected_code_artifact_metadata(ctx)
+ captures = ctx.rejected_code_artifact_metadata_captures
+ assert len(captures) == 5
+ assert [c.payload["collect"]["variant"] for c in captures] == [2, 3, 4, 5, 6]
+
+
+def test_rejected_capture_log_carries_payload_kept_off_user_facing_text() -> None:
+ ctx = make_copilot_ctx(turn_id="turn_capture_log")
+ payload = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}
+ ctx.submitted_code_artifact_metadata_snapshot = payload
+ with capture_logs() as logs:
+ wu._capture_rejected_code_artifact_metadata(ctx)
+ entries = [line for line in logs if line.get("event") == "copilot_rejected_code_artifact_metadata_captured"]
+ assert entries
+ assert entries[0]["rejected_code_artifact_metadata_payload"] == payload
+ assert entries[0]["submitted_metadata_labels"] == ["collect"]
+
+ wu._stash_output_contract_reject_budget_terminal(
+ ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_capture"
+ )
+ assert ctx.blocker_signal is not None
+ assert "extraction_schema" not in ctx.blocker_signal.user_facing_reason
+ assert "goal_value_paths" not in ctx.blocker_signal.user_facing_reason
+
+
+def test_contract_satisfying_candidate_admits_run_no_reject() -> None:
+ ctx = _antecedent_ctx(
+ CompletionCriterion(
+ id="c_record", outcome="The returned record includes record id.", output_path="output.record_id"
+ )
+ )
+ workflow_yaml = _declaration_waiver_yaml(
+ 'record_id = await page.inner_text("#record")\nreturn {"output": {"record_id": record_id}}'
+ )
+ metadata = [
+ wu._metadata_contract_template(
+ block_label="extract_record",
+ required_paths={"output.record_id"},
+ source="requested_output_contract",
+ reason_code="requested_output_contract_missing_output_coverage",
+ )
+ ]
+
+ evaluation = wu._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
+
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is False
+ assert evaluation.missing_return_paths == []
+ assert evaluation.missing_metadata_paths == []
+
+
+def test_deficient_candidate_still_rejected_by_gate() -> None:
+ ctx = _antecedent_ctx(
+ CompletionCriterion(
+ id="c_record", outcome="The returned record includes record id.", output_path="output.record_id"
+ )
+ )
+ workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")\nreturn record_id')
+
+ evaluation = wu._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, [], enforce_value_bearing_liveness=True
+ )
+
+ assert evaluation is not None
+ assert evaluation.has_deficiencies is True
+ assert "output.record_id" in evaluation.missing_return_paths
+ assert "output.record_id" in evaluation.missing_metadata_paths
diff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.py
index 4b98633b8..69b2e8827 100644
--- a/tests/unit/test_copilot_turn_halt.py
+++ b/tests/unit/test_copilot_turn_halt.py
@@ -4,7 +4,11 @@ from types import SimpleNamespace
import pytest
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ _OUTPUT_CONTRACT_TERMINAL_REASON_CODES,
+ CopilotToolBlockerSignal,
+ stash_blocker_signal,
+)
from skyvern.forge.sdk.copilot.enforcement import (
_check_enforcement,
_maybe_stash_terminal_challenge_halt,
@@ -87,6 +91,16 @@ def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expect
assert halt.blocker_signal is signal
+def test_every_output_contract_terminal_reason_code_maps_to_a_turn_halt() -> None:
+ # Fail-closed guard: an output-contract terminal reason code with no turn_halt mapping mints no
+ # halt, so the turn degrades to loop_detected instead of terminating (the SKY-12594 e25ec5 regression).
+ assert _OUTPUT_CONTRACT_TERMINAL_REASON_CODES
+ for reason_code in _OUTPUT_CONTRACT_TERMINAL_REASON_CODES:
+ halt = turn_halt_from_blocker_signal(_signal(internal_reason_code=reason_code), source="hook")
+ assert halt is not None, f"{reason_code} maps to no TurnHalt; the turn would degrade to loop_detected"
+ assert halt.kind is TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE
+
+
def _halt_ctx(**overrides: object) -> SimpleNamespace:
fields: dict[str, object] = {
"turn_halt": None,