Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Converge Copilot author-time metadata-gate rejects to a typed terminal (SKY-12594) (#7527)
Andrew Neilson6ce4735cc1d6
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex f6d4ce814..0f684af0d 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -167,18 +167,33 @@ def build_output_source_unobservable_blocker_signal("visits, so there is nothing for the workflow to read them from. I've kept the draft; ""tell me where those values appear and I'll wire them up.")+ elif reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:+ user_facing = (+ "I rewrote this workflow several times to return the value(s) you asked for"+ f"{f' ({path_text})' if path_text else ''}, but each version left those outputs "+ "undeclared or unreturned, so the workflow can't reliably hand them back. I've kept "+ "the draft; tell me where each of those values appears and I'll declare and return them."+ )else:user_facing = ("I couldn't shape this workflow so it reliably returns the values you asked for"f"{f' ({path_text})' if path_text else ''}. I've kept the current draft; let me know ""where those values show up and I'll try a different structure.")- agent_steer = (- "STOP: the requested output contract has no observable extraction source in the current "- f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "- "re-authoring the same draft. Report the missing source to the user and ask where the "- "values appear. The prior draft is preserved; do not rerun the blocks."- )+ if reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE:+ agent_steer = (+ "STOP: the output contract kept failing its declaration/return coverage across repeated "+ f"re-authored candidates for required path(s) [{path_text or '(unknown)'}]. This is not "+ "repairable by re-authoring the same draft. Report the missing output declarations to the "+ "user and ask where the values appear. The prior draft is preserved; do not rerun the blocks."+ )+ else:+ agent_steer = (+ "STOP: the requested output contract has no observable extraction source in the current "+ f"trajectory for required path(s) [{path_text or '(unknown)'}]. This is not repairable by "+ "re-authoring the same draft. Report the missing source to the user and ask where the "+ "values appear. The prior draft is preserved; do not rerun the blocks."+ )return CopilotToolBlockerSignal(blocker_kind="tool_error",agent_steering_text=agent_steer,@@ -379,8 +394,13 @@ UNCOVERED_OUTPUT_RESCOUT_STEER_REASON_CODE = "tool_error_uncovered_output_rescouRECORDED_OUTCOME_GROUNDING_REASON_CODE = "recorded_outcome_grounding_required"DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE = "definition_contract_unsatisfied"SCHEMA_INCOMPATIBILITY_REASON_CODE = "schema_incompatibility"+OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE = "output_contract_reject_budget_exhausted"_OUTPUT_CONTRACT_TERMINAL_REASON_CODES = frozenset(- {OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE, OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE}+ {+ OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,+ OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,+ })# A held blocker whose reason code is in this set must win both the rendered reply and the typed@@ -396,6 +416,7 @@ GENUINELY_TERMINAL_BLOCKER_REASON_CODES: frozenset[str] = frozenset(SCHEMA_INCOMPATIBILITY_REASON_CODE,OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,"advisory_dispatch_stalled",DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,"repair_ceiling_reached",diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 07efbd1f7..fb4b958e0 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -172,6 +172,11 @@ class PreRunPageReference:workflow_run_id: str+@dataclass(frozen=True)+class RejectedCodeArtifactMetadataCapture:+ payload: Any++@dataclass(frozen=True)class RegisteredArtifactEntry:artifact_id: str@@ -393,6 +398,8 @@ class AgentContext:# only normalizes and carries the metadata; sufficiency checks live elsewhere.code_artifact_metadata: dict[str, CodeArtifactMetadataPayload] = field(default_factory=dict)raw_code_artifact_metadata: object | None = None+ submitted_code_artifact_metadata_snapshot: Any = None+ rejected_code_artifact_metadata_captures: list[RejectedCodeArtifactMetadataCapture] = field(default_factory=list)# Hydrated at turn start from StructuredContext.observed_acted_pages; lets the# composition gate credit a page observed on a prior turn when this turn's# flow_evidence does not cover it (closes the spent-inspection-budgetdiff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 175887b59..4ca37a843 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -28,7 +28,9 @@ from skyvern.forge.sdk.api.llm.schema_validator import validate_schemafrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stampfrom skyvern.forge.sdk.copilot.blocker_signal import (CREDENTIAL_SCOUT_VERIFY_REPLY,+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,CopilotToolBlockerSignal,+ blocker_signal_is_genuinely_terminal,build_definition_contract_unsatisfied_blocker_signal,build_output_source_unobservable_blocker_signal,clear_terminal_evidence_on_workflow_edit,@@ -176,6 +178,7 @@ from skyvern.forge.sdk.copilot.runtime import (SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,AgentContext,AuthorTimeGateAblationPayload,+ RejectedCodeArtifactMetadataCapture,ScoutedFieldParameterBinding,ScoutedInteraction,ScoutedSubmitRungBinding,@@ -2549,9 +2552,22 @@ _SEPARATED_SPINE_SHAPE_REQUIRED_REASON_CODE = "separated_spine_shape_required"_SEPARATED_BROWSER_SPINE_PLUS_EXTRACTION_STRUCTURE = "separated_browser_spine_plus_extraction"_OUTPUT_CONTRACT_REJECT_REASON_CODE = "output_contract_required"_OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE = "value_bearing_output_required"-_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = "output_contract_reject_budget_exhausted"+_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH = "output"+_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE = OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE_MAX_OUTPUT_CONTRACT_REJECTS = 4_MAX_OUTPUT_CONTRACT_DEFERRALS = 3+_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY = "missing_code_artifact_metadata"+_METADATA_NORMALIZATION_REJECT_FAMILY = "metadata_normalization"+# Families sharing the reject-budget ladder: a candidate rotating its structural fingerprint within the+# family must not reset its streak, though an imposition landing still does (definition is already terminal).+_METADATA_FAMILY_REJECT_FAMILIES = frozenset(+ {+ _OUTPUT_CONTRACT_REJECT_REASON_CODE,+ _OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,+ _MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,+ _METADATA_NORMALIZATION_REJECT_FAMILY,+ }+)_MAX_OUTPUT_CONTRACT_ACTUATIONS_WITHOUT_RUN = 3_OUTPUT_CONTRACT_ABLATION_GATE_ID = "output_contract_actuation"_METADATA_PREFLIGHT_ABLATION_GATE_ID = "metadata_run_preflight_reject"@@ -3751,12 +3767,17 @@ def _record_output_contract_reject(authored_structural_fingerprint: str = "",workflow_yaml: str = "",) -> dict[str, Any]:+ # A candidate so incomplete that no required child paths are derivable must still consume the+ # reject budget: an empty set would silently skip counting and degrade the turn to the generic+ # churn stop (loop_detected) instead of this ladder's typed terminal.+ counted_paths = evaluation.required_paths or {_OUTPUT_CONTRACT_UNDECLARED_SENTINEL_PATH}count = _record_output_contract_family_reject(ctx,- evaluation.required_paths,+ counted_paths,reject_family=str(evaluation.payload.get("reason_code") or _OUTPUT_CONTRACT_REJECT_REASON_CODE),authored_structural_fingerprint=authored_structural_fingerprint,)+ _capture_rejected_code_artifact_metadata(ctx)payload = dict(evaluation.payload)payload["output_contract_reject_count"] = countpayload["output_contract_reject_budget"] = _MAX_OUTPUT_CONTRACT_REJECTS@@ -3778,30 +3799,9 @@ def _record_output_contract_reject(# The typed adjudication now owns this same deficiency. Keep the historical reject for# ceiling/liveness evidence, but do not render it again as a fresh rejection next turn.record_build_test_outcome(ctx, None)- if count >= _MAX_OUTPUT_CONTRACT_REJECTS:- if run_backed_repair_evidence_exists(ctx):- payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE- payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE- else:- deferral_count = _record_output_contract_deferral(ctx, evaluation.required_paths)- if deferral_count >= _MAX_OUTPUT_CONTRACT_DEFERRALS:- payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE- payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE- LOG.info(- "copilot_output_contract_budget_deferral_cap_reached",- block_label=evaluation.block_label,- canonical_output_contract_signature=evaluation.canonical_signature,- output_contract_reject_count=count,- output_contract_deferral_count=deferral_count,- )- else:- LOG.info(- "copilot_output_contract_budget_rewrite_deferred_no_run",- block_label=evaluation.block_label,- canonical_output_contract_signature=evaluation.canonical_signature,- output_contract_reject_count=count,- output_contract_deferral_count=deferral_count,- )+ if _adjudicate_output_contract_budget(ctx, counted_paths, count=count, block_label=evaluation.block_label):+ payload["reason_code"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE+ payload["reject_reason"] = _OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODEif actuation is None or actuation.kind == OutputContractActuationKind.STRUCTURE_DIRECTIVE:structural_payload = _output_contract_author_time_structural_payload(ctx,@@ -3818,7 +3818,8 @@ def _record_output_contract_reject(if isinstance(payload.get("missing_requested_output_facts"), list)else None,)- _record_code_authoring_guardrail_reject(ctx)+ if count < 1:+ _record_code_authoring_guardrail_reject(ctx)return payload@@ -3846,7 +3847,9 @@ def _record_output_contract_family_reject(if authored_structural_fingerprint:prior_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)imposed_since = ctx.output_contract_imposed_since_last_reject_by_signature.get(signature, False)- if imposed_since or (prior_fingerprint is not None and prior_fingerprint != authored_structural_fingerprint):+ fingerprint_rotated = prior_fingerprint is not None and prior_fingerprint != authored_structural_fingerprint+ rotation_resets = fingerprint_rotated and reject_family not in _METADATA_FAMILY_REJECT_FAMILIES+ if imposed_since or rotation_resets:count_by_signature[signature] = 0ctx.output_contract_imposed_since_last_reject_by_signature[signature] = FalseLOG.info(@@ -3882,6 +3885,80 @@ def _record_output_contract_deferral(ctx: AgentContext, required_paths: set[str]return count+def _adjudicate_output_contract_budget(+ ctx: AgentContext,+ required_paths: set[str],+ *,+ count: int,+ block_label: str,+) -> bool:+ if count < _MAX_OUTPUT_CONTRACT_REJECTS:+ return False+ signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)+ if not run_backed_repair_evidence_exists(ctx):+ deferral_count = _record_output_contract_deferral(ctx, required_paths)+ if deferral_count < _MAX_OUTPUT_CONTRACT_DEFERRALS:+ LOG.info(+ "copilot_output_contract_budget_rewrite_deferred_no_run",+ block_label=block_label,+ canonical_output_contract_signature=signature,+ output_contract_reject_count=count,+ output_contract_deferral_count=deferral_count,+ )+ return False+ LOG.info(+ "copilot_output_contract_budget_deferral_cap_reached",+ block_label=block_label,+ canonical_output_contract_signature=signature,+ output_contract_reject_count=count,+ output_contract_deferral_count=deferral_count,+ )+ _stash_output_contract_reject_budget_terminal(+ ctx, required_paths=required_paths, block_label=block_label, signature=signature+ )+ return True+++def _stash_output_contract_reject_budget_terminal(+ ctx: AgentContext,+ *,+ required_paths: set[str],+ block_label: str,+ signature: str,+) -> None:+ if ctx.turn_halt is not None or blocker_signal_is_genuinely_terminal(ctx.blocker_signal):+ return+ signal = build_output_source_unobservable_blocker_signal(+ reason_code=_OUTPUT_CONTRACT_REJECT_BUDGET_REASON_CODE,+ required_paths=required_paths,+ block_label=block_label,+ )+ stash_blocker_signal(ctx, signal)+ stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")+ LOG.info(+ "copilot_output_contract_reject_budget_terminal",+ block_label=block_label,+ canonical_output_contract_signature=signature,+ canonical_required_child_paths=sorted(required_paths),+ )+++def _capture_rejected_code_artifact_metadata(ctx: AgentContext) -> None:+ snapshot = ctx.submitted_code_artifact_metadata_snapshot+ if snapshot is None:+ return+ captures = ctx.rejected_code_artifact_metadata_captures+ captures.append(RejectedCodeArtifactMetadataCapture(payload=snapshot))+ del captures[:-5]+ LOG.info(+ "copilot_rejected_code_artifact_metadata_captured",+ capture_count=len(captures),+ submitted_metadata_labels=sorted(snapshot) if isinstance(snapshot, dict) else None,+ submitted_metadata_type=type(snapshot).__name__,+ rejected_code_artifact_metadata_payload=snapshot,+ )++def _output_contract_reject_result(evaluation: _OutputContractEvaluation,*,@@ -5405,6 +5482,8 @@ def _metadata_contract_run_preflight_reject(*,enforce_untagged_declared_inputs: bool = False,) -> dict[str, Any] | None:+ if isinstance(ctx, AgentContext):+ ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(raw_code_artifact_metadata)definition_reject = (_definition_plane_preflight_reject(ctx,@@ -11216,6 +11295,7 @@ async def _update_workflow(ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))+ ctx.submitted_code_artifact_metadata_snapshot = copy.deepcopy(params.get("code_artifact_metadata"))# Imposition reconciles synthesized aliases/parameters before the persisted YAML contract is checked._enrich_scout_trajectory_input_correspondences(workflow_yaml, ctx)runtime_parameters = params.get("parameters")@@ -11551,10 +11631,10 @@ async def _update_workflow(source=output_path_coverage_source,reason_code=output_path_coverage_reason_code,)- _record_output_contract_family_reject(+ missing_metadata_reject_count = _record_output_contract_family_reject(ctx,required_child_output_paths,- reject_family="missing_code_artifact_metadata",+ reject_family=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,)_record_author_time_reject_outcome(ctx,@@ -11575,6 +11655,7 @@ async def _update_workflow(block_labels=missing_labels,missing_requested_output_facts=missing_metadata_output_facts,)+ _capture_rejected_code_artifact_metadata(ctx)credential_scout_errors = ([]if _request_policy_allows_untested_code_block_draft(ctx)@@ -11584,7 +11665,16 @@ async def _update_workflow(block_labels=params.get("block_labels"),))- _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=bool(credential_scout_errors))+ budget_terminal = _adjudicate_output_contract_budget(+ ctx,+ required_child_output_paths,+ count=missing_metadata_reject_count,+ block_label=missing_labels[0] if len(missing_labels) == 1 else "",+ )+ if credential_scout_errors:+ _record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)+ elif missing_metadata_reject_count < 1 and not budget_terminal:+ _record_code_authoring_guardrail_reject(ctx)return reject(error=missing_metadata_error,user_facing_summary=_compiled_authoring_user_summary(),@@ -11613,10 +11703,10 @@ async def _update_workflow(code_artifact_metadata_error = normalization.errorif code_artifact_metadata_error is not None:record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)- _record_output_contract_family_reject(+ normalization_reject_count = _record_output_contract_family_reject(ctx,required_child_output_paths,- reject_family="metadata_normalization",+ reject_family=_METADATA_NORMALIZATION_REJECT_FAMILY,)_record_author_time_reject_outcome(ctx,@@ -11635,6 +11725,13 @@ async def _update_workflow(),block_labels=normalization.offending_labels,)+ _capture_rejected_code_artifact_metadata(ctx)+ _adjudicate_output_contract_budget(+ ctx,+ required_child_output_paths,+ count=normalization_reject_count,+ block_label=normalization.offending_labels[0] if len(normalization.offending_labels) == 1 else "",+ )if normalization.schema_incompatibilities:incompatibility = merge_schema_incompatibilities(normalization.schema_incompatibilities)if incompatibility is not None:diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.pyindex a60372046..dd333effe 100644--- a/skyvern/forge/sdk/copilot/turn_halt.py+++ b/skyvern/forge/sdk/copilot/turn_halt.py@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (GENUINELY_TERMINAL_BLOCKER_REASON_CODES as GENUINELY_TERMINAL_BLOCKER_REASON_CODES,)from skyvern.forge.sdk.copilot.blocker_signal import (+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,CopilotToolBlockerSignal,blocker_signal_is_genuinely_terminal,build_output_source_unobservable_blocker_signal,@@ -95,6 +96,7 @@ _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset({OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODE,OUTPUT_CONTRACT_ACTUATION_EXHAUSTED_REASON_CODE,+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,ADVISORY_DISPATCH_STALLED_REASON_CODE,})diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.pyindex 351915d3a..8b5d59c0b 100644--- a/tests/unit/test_copilot_output_contract_actuation.py+++ b/tests/unit/test_copilot_output_contract_actuation.py@@ -17,8 +17,10 @@ from structlog.testing import capture_logsfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot import enforcementfrom skyvern.forge.sdk.copilot.blocker_signal import (+ OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,CopilotToolBlockerSignal,assert_clean_user_facing_text,+ blocker_signal_is_genuinely_terminal,build_output_source_unobservable_blocker_signal,)from skyvern.forge.sdk.copilot.build_test_outcome import RecordedBuildTestOutcome@@ -46,6 +48,7 @@ from skyvern.forge.sdk.copilot.request_slots import ()from skyvern.forge.sdk.copilot.result_evidence import loaded_result_composition_evidence_from_pagefrom skyvern.forge.sdk.copilot.runtime import (+ RejectedCodeArtifactMetadataCapture,output_contract_ladder_unresolved,record_author_time_gate_ablation_event,)@@ -553,6 +556,8 @@ def _advisory_ctx() -> SimpleNamespace:output_contract_run_output_observed_by_signature={},output_contract_run_bound_required_path_by_signature={},output_contract_bail_actuated_this_call=False,+ submitted_code_artifact_metadata_snapshot=None,+ rejected_code_artifact_metadata_captures=[],author_time_gate_log_only_ids=frozenset(),author_time_gate_ablation_events=[],latest_recorded_build_test_outcome=None,@@ -2205,3 +2210,280 @@ def test_observation_production_is_not_click_only_spine() -> None:code = 'await page.click("#submit")\nreturn {"output": {"record_id": "x"}}'assert wu._output_contract_click_only_spine(code, {"output.blocker"}) is False+++_METADATA_REQUIRED = {"output.confirmation_number"}+++def test_metadata_family_reject_holds_streak_across_fingerprint_rotation() -> None:+ ctx = _counter_ctx()+ for family in ("missing_code_artifact_metadata", "metadata_normalization", "output_contract_required"):+ sub = _counter_ctx()+ counts = [+ wu._record_output_contract_family_reject(+ sub, _METADATA_REQUIRED, reject_family=family, authored_structural_fingerprint=f"fp_{i}"+ )+ for i in range(4)+ ]+ assert counts == [1, 2, 3, 4], family+ imposed = wu._record_output_contract_family_reject(+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_0"+ )+ signature = next(iter(ctx.output_contract_reject_count_by_signature))+ ctx.output_contract_imposed_since_last_reject_by_signature[signature] = True+ after_imposition = wu._record_output_contract_family_reject(+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata", authored_structural_fingerprint="fp_1"+ )+ assert (imposed, after_imposition) == (1, 1)+++def test_out_of_family_reject_resets_on_fingerprint_rotation() -> None:+ ctx = _counter_ctx()+ first = wu._record_output_contract_family_reject(+ ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_a"+ )+ rotated = wu._record_output_contract_family_reject(+ ctx, _METADATA_REQUIRED, reject_family="definition_contract_unsatisfied", authored_structural_fingerprint="fp_b"+ )+ assert (first, rotated) == (1, 1)+++def test_reject_budget_exhausted_reason_code_is_genuinely_terminal() -> None:+ signal = build_output_source_unobservable_blocker_signal(+ reason_code=OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,+ required_paths=_METADATA_REQUIRED,+ block_label="collect",+ )+ assert blocker_signal_is_genuinely_terminal(signal) is True+++def test_metadata_family_budget_reaches_typed_terminal_at_bound_without_churn() -> None:+ ctx = make_copilot_ctx(turn_id="turn_budget_bound")+ terminal_at = None+ for candidate in range(1, 7):+ count = wu._record_output_contract_family_reject(+ ctx, _METADATA_REQUIRED, reject_family="missing_code_artifact_metadata"+ )+ assert count == candidate+ exhausted = wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=count, block_label="collect")+ if exhausted and terminal_at is None:+ terminal_at = candidate+ assert terminal_at == 6+ assert ctx.turn_halt is not None+ assert ctx.blocker_signal is not None+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE+ assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True+ assert ctx.code_authoring_guardrail_reject_count == 0+++def test_budget_terminal_names_paths_and_omits_generic_churn_string() -> None:+ ctx = make_copilot_ctx(turn_id="turn_budget_copy")+ wu._stash_output_contract_reject_budget_terminal(+ ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_copy"+ )+ assert ctx.blocker_signal is not None+ user_facing = ctx.blocker_signal.user_facing_reason+ assert_clean_user_facing_text(user_facing)+ assert "output.confirmation_number" in user_facing+ assert "safety checks rejected each version" not in user_facing+ assert ctx.blocker_signal.renders_final_reply is True+++def test_main_gate_defers_churn_to_budget_ladder_when_reject_counted(monkeypatch: pytest.MonkeyPatch) -> None:+ ctx = make_copilot_ctx(turn_id="turn_budget_defer")+ calls: list[object] = []+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))+ count = wu._record_output_contract_reject(+ ctx,+ _make_evaluation("", block_label="collect"),+ summary="Typed deficiency.",+ authored_structural_fingerprint="fp_defer",+ workflow_yaml=_PAGE_READ_YAML,+ )+ assert count["output_contract_reject_count"] >= 1+ assert calls == []+++def test_undeclared_contract_rejects_count_toward_budget_and_reach_typed_terminal(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # A candidate deriving NO required child paths must still consume the reject budget: skipping+ # the count degrades repeated preflight rejects to the generic churn stop (loop_detected).+ ctx = make_copilot_ctx(turn_id="turn_budget_undeclared")+ calls: list[object] = []+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: calls.append(_ctx))+ malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}+ ctx.submitted_code_artifact_metadata_snapshot = malformed++ def undeclared_evaluation() -> wu._OutputContractEvaluation:+ return wu._OutputContractEvaluation(+ block_label="collect",+ artifact_id="",+ required_paths=set(),+ observation_paths=set(),+ declaration_paths=set(),+ source="requested_output",+ reason_code="output_contract_required",+ missing_metadata_paths=[],+ missing_schema_paths=[],+ missing_return_paths=[],+ shape_violations=[],+ canonical_signature="",+ payload={},+ repair_context=None,+ )++ terminal_at = None+ for candidate in range(1, 8):+ payload = wu._record_output_contract_reject(+ ctx,+ undeclared_evaluation(),+ summary="Undeclared contract deficiency.",+ authored_structural_fingerprint=f"fp_{candidate}",+ workflow_yaml="",+ )+ assert payload["output_contract_reject_count"] == candidate+ if ctx.turn_halt is not None and terminal_at is None:+ terminal_at = candidate+ break+ assert terminal_at is not None and terminal_at <= 7+ assert ctx.blocker_signal is not None+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE+ assert calls == []+ assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)+++def test_recorded_identical_reject_shape_replays_to_budget_terminal_with_per_reject_capture(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ # Replays the recorded live churn shape — identical value_bearing_output_required rejects on a+ # candidate deriving no required paths — through the real seam to budget exhaustion.+ ctx = make_copilot_ctx(turn_id="turn_recorded_shape_replay")+ churn_calls: list[object] = []+ monkeypatch.setattr(wu, "_record_code_authoring_guardrail_reject", lambda _ctx: churn_calls.append(_ctx))+ malformed = {"collect": {"goal_value_paths": [], "extraction_schema": None}}+ ctx.submitted_code_artifact_metadata_snapshot = malformed++ def recorded_shape_evaluation() -> wu._OutputContractEvaluation:+ return wu._OutputContractEvaluation(+ block_label="collect",+ artifact_id="",+ required_paths=set(),+ observation_paths=set(),+ declaration_paths=set(),+ source="requested_output",+ reason_code=wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE,+ missing_metadata_paths=[],+ missing_schema_paths=[],+ missing_return_paths=[],+ shape_violations=[wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE],+ canonical_signature="",+ payload={"reason_code": wu._OUTPUT_CONTRACT_VALUE_REQUIRED_REASON_CODE},+ repair_context=None,+ )++ terminal_at = None+ for candidate in range(1, 8):+ payload = wu._record_output_contract_reject(+ ctx,+ recorded_shape_evaluation(),+ summary="Identical value-bearing deficiency.",+ authored_structural_fingerprint="fp_identical",+ workflow_yaml="",+ )+ assert payload["output_contract_reject_count"] == candidate+ assert len(ctx.rejected_code_artifact_metadata_captures) == min(candidate, 5)+ if ctx.turn_halt is not None:+ terminal_at = candidate+ break+ assert terminal_at is not None and terminal_at <= 7+ assert ctx.blocker_signal is not None+ assert ctx.blocker_signal.internal_reason_code == OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE+ assert blocker_signal_is_genuinely_terminal(ctx.blocker_signal) is True+ assert churn_calls == []+++def test_no_reject_leaves_no_budget_terminal() -> None:+ ctx = make_copilot_ctx(turn_id="turn_budget_yes")+ assert wu._adjudicate_output_contract_budget(ctx, _METADATA_REQUIRED, count=0, block_label="collect") is False+ assert ctx.turn_halt is None+ assert ctx.blocker_signal is None+++def test_rejected_code_artifact_metadata_capture_is_byte_exact_and_bounded() -> None:+ ctx = make_copilot_ctx(turn_id="turn_capture")+ malformed = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}+ ctx.submitted_code_artifact_metadata_snapshot = malformed+ wu._capture_rejected_code_artifact_metadata(ctx)+ assert ctx.rejected_code_artifact_metadata_captures[-1] == RejectedCodeArtifactMetadataCapture(payload=malformed)+ for index in range(7):+ ctx.submitted_code_artifact_metadata_snapshot = {"collect": {"variant": index}}+ wu._capture_rejected_code_artifact_metadata(ctx)+ captures = ctx.rejected_code_artifact_metadata_captures+ assert len(captures) == 5+ assert [c.payload["collect"]["variant"] for c in captures] == [2, 3, 4, 5, 6]+++def test_rejected_capture_log_carries_payload_kept_off_user_facing_text() -> None:+ ctx = make_copilot_ctx(turn_id="turn_capture_log")+ payload = {"collect": {"goal_value_paths": ["output.confirmation_number"], "extraction_schema": None}}+ ctx.submitted_code_artifact_metadata_snapshot = payload+ with capture_logs() as logs:+ wu._capture_rejected_code_artifact_metadata(ctx)+ entries = [line for line in logs if line.get("event") == "copilot_rejected_code_artifact_metadata_captured"]+ assert entries+ assert entries[0]["rejected_code_artifact_metadata_payload"] == payload+ assert entries[0]["submitted_metadata_labels"] == ["collect"]++ wu._stash_output_contract_reject_budget_terminal(+ ctx, required_paths=_METADATA_REQUIRED, block_label="collect", signature="sig_capture"+ )+ assert ctx.blocker_signal is not None+ assert "extraction_schema" not in ctx.blocker_signal.user_facing_reason+ assert "goal_value_paths" not in ctx.blocker_signal.user_facing_reason+++def test_contract_satisfying_candidate_admits_run_no_reject() -> None:+ ctx = _antecedent_ctx(+ CompletionCriterion(+ id="c_record", outcome="The returned record includes record id.", output_path="output.record_id"+ )+ )+ workflow_yaml = _declaration_waiver_yaml(+ 'record_id = await page.inner_text("#record")\nreturn {"output": {"record_id": record_id}}'+ )+ metadata = [+ wu._metadata_contract_template(+ block_label="extract_record",+ required_paths={"output.record_id"},+ source="requested_output_contract",+ reason_code="requested_output_contract_missing_output_coverage",+ )+ ]++ evaluation = wu._evaluate_output_contract_for_code_block(+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True+ )++ assert evaluation is not None+ assert evaluation.has_deficiencies is False+ assert evaluation.missing_return_paths == []+ assert evaluation.missing_metadata_paths == []+++def test_deficient_candidate_still_rejected_by_gate() -> None:+ ctx = _antecedent_ctx(+ CompletionCriterion(+ id="c_record", outcome="The returned record includes record id.", output_path="output.record_id"+ )+ )+ workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")\nreturn record_id')++ evaluation = wu._evaluate_output_contract_for_code_block(+ ctx, workflow_yaml, [], enforce_value_bearing_liveness=True+ )++ assert evaluation is not None+ assert evaluation.has_deficiencies is True+ assert "output.record_id" in evaluation.missing_return_paths+ assert "output.record_id" in evaluation.missing_metadata_pathsdiff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.pyindex 4b98633b8..69b2e8827 100644--- a/tests/unit/test_copilot_turn_halt.py+++ b/tests/unit/test_copilot_turn_halt.py@@ -4,7 +4,11 @@ from types import SimpleNamespaceimport pytest-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, stash_blocker_signal+from skyvern.forge.sdk.copilot.blocker_signal import (+ _OUTPUT_CONTRACT_TERMINAL_REASON_CODES,+ CopilotToolBlockerSignal,+ stash_blocker_signal,+)from skyvern.forge.sdk.copilot.enforcement import (_check_enforcement,_maybe_stash_terminal_challenge_halt,@@ -87,6 +91,16 @@ def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expectassert halt.blocker_signal is signal+def test_every_output_contract_terminal_reason_code_maps_to_a_turn_halt() -> None:+ # Fail-closed guard: an output-contract terminal reason code with no turn_halt mapping mints no+ # halt, so the turn degrades to loop_detected instead of terminating (the SKY-12594 e25ec5 regression).+ assert _OUTPUT_CONTRACT_TERMINAL_REASON_CODES+ for reason_code in _OUTPUT_CONTRACT_TERMINAL_REASON_CODES:+ halt = turn_halt_from_blocker_signal(_signal(internal_reason_code=reason_code), source="hook")+ assert halt is not None, f"{reason_code} maps to no TurnHalt; the turn would degrade to loop_detected"+ assert halt.kind is TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE++def _halt_ctx(**overrides: object) -> SimpleNamespace:fields: dict[str, object] = {"turn_halt": None,