Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Resolve the output-contract pass path through the single actuation resolver (SKY-12907) (#7706)
Andrew Neilsonde8c2b30ccd2
diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.pyindex 11d45ab4c..c84aea29e 100644--- a/skyvern/forge/sdk/copilot/output_contracts.py+++ b/skyvern/forge/sdk/copilot/output_contracts.py@@ -23,6 +23,8 @@ class OutputContractBailFamily(StrEnum):class OutputContractActuationKind(StrEnum):IMPOSED = "imposed"+ CREDIT = "credit"+ IMPOSE_SCOUTED_BINDING = "impose_scouted_binding"STRUCTURE_DIRECTIVE = "structure_directive"ADVISORY_RUN = "advisory_run"BLOCKED_TERMINAL = "blocked_terminal"@@ -63,6 +65,11 @@ class OutputContractActuationEvidence:consumed_run_bound_required_path: bool = Falseconsumed_run_carried_page_extraction: bool = Falseloaded_result_source_producible: bool = False+ # Steering produced no structural movement: the canonical structural fingerprint did not+ # change across the consumed directives. Never a reject count — counts inflate.+ steering_exhausted: bool = False+ credit_available: bool = False+ scouted_binding_impose_available: bool = False@dataclass(frozen=True)@@ -90,9 +97,17 @@ def resolve_output_contract_actuation(page-source extraction, and that it still bound no required path. A consumed run whose output wasobserved but bound nothing without a page-source extraction on board is not exhaustion evidence —it re-enters the ladder once so the stronger page-source imposition can bind the on-screen values a- code static-return provably cannot key."""+ code static-return provably cannot key. Once steering is exhausted the server resolves the+ submission itself — crediting a draft the strict prover can only call uncertain, else imposing the+ binding the recording pins — so a satisfiable submission has a reachable pass path instead of only+ a terminal; both inherit the advisory lane's post-run adjudication."""if evidence.imposed_available:return OutputContractActuation(OutputContractActuationKind.IMPOSED, family)+ if evidence.steering_exhausted:+ if evidence.credit_available:+ return OutputContractActuation(OutputContractActuationKind.CREDIT, family)+ if evidence.scouted_binding_impose_available:+ return OutputContractActuation(OutputContractActuationKind.IMPOSE_SCOUTED_BINDING, family)grantable_source = (evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED)diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 2577eb3da..dc5e1f61f 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -5120,23 +5120,51 @@ class _OutputContractCeilingResolution:mode: Literal["credited", "imposed"]+def _loaded_result_source_producible_for_signature(+ ctx: AgentContext,+ *,+ target_code: str,+ signature: str,+) -> bool:+ """Loaded-result evidence is producible for this contract only while it is unclaimed or+ already claimed by this signature; evidence claimed by another contract does not carry+ over. The first contract to find it producible claims it."""+ if not loaded_result_source_producible(ctx.latest_evaluate_result_composition_steer, target_code=target_code):+ return False+ claimed_signature = ctx.latest_evaluate_result_composition_signature+ if claimed_signature not in (None, signature):+ return False+ ctx.latest_evaluate_result_composition_signature = signature+ return True+++def _every_required_output_value_observed(ctx: AgentContext, required_paths: set[str]) -> bool:+ """Imposition writes a binding for every required path, so authorization needs observation+ coverage for every one of them — a single overlapping path would otherwise license composing+ the rest as constants the recording never saw."""+ if not required_paths:+ return False+ return all(_observed_required_output_values(ctx, {path}) for path in required_paths)++def _output_contract_binding_source_on_record(ctx: AgentContext,evaluation: _OutputContractEvaluation,workflow_yaml: str,) -> bool:"""Whether the recorded exploration already pins where the requested value lives: either the- scout observed the required paths, or a producible loaded-result composition anchors the+ scout observed every required path, or a producible loaded-result composition anchors thetarget block's selectors. Imposition is only justified when this is on record."""binding_paths = evaluation.observation_paths or evaluation.required_paths- if binding_paths and _observed_required_output_values(ctx, binding_paths):+ if _every_required_output_value_observed(ctx, binding_paths):return Truetarget_block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(evaluation.block_label)if target_block is None:return False- return loaded_result_source_producible(- ctx.latest_evaluate_result_composition_steer,+ return _loaded_result_source_producible_for_signature(+ ctx,target_code=str(target_block.get("code") or ""),+ signature=evaluation.canonical_signature,)@@ -5192,12 +5220,24 @@ def _impose_output_contract_binding_at_ceiling(return imposed_yaml, scaffolded_metadata-def _output_contract_ceiling_reached(ctx: AgentContext, evaluation: _OutputContractEvaluation) -> bool:- if not evaluation.canonical_signature or not evaluation.block_label:+def _output_contract_steering_exhausted(+ ctx: AgentContext,+ evaluation: _OutputContractEvaluation,+ workflow_yaml: str,+) -> bool:+ """Steering has stopped moving the draft: this submission came back with the same canonical+ structural fingerprint the last reject already steered on, so re-emitting the directive cannot+ converge. Keyed on that fingerprint rather than reject counts, which inflate, and unreachable+ on a first submission because no prior reject fingerprint exists yet."""+ signature = evaluation.canonical_signature+ if not signature or not evaluation.block_label:return False- if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):+ if not _prior_output_contract_actuation(ctx, signature):return False- return _output_contract_reject_count(ctx, evaluation.canonical_signature) + 1 >= _MAX_OUTPUT_CONTRACT_REJECTS+ steered_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)+ if not steered_fingerprint:+ return False+ return steered_fingerprint == _output_contract_structural_fingerprint(workflow_yaml, signature)def _credit_output_contract_submission(@@ -5206,12 +5246,10 @@ def _credit_output_contract_submission(*,workflow_yaml: str,raw_metadata: object,- actuation_entry: str,) -> _OutputContractEvaluation | None:- """Credit move: re-adjudicate the submission with static uncertainty credited, so a draft- that binds the required paths but that the liveness prover can only call "unknown" ships- instead of looping. Returns the crediting evaluation when the strict rejection was- over-strict, None when the deficiency is genuine."""+ """Re-adjudicate the submission with static uncertainty credited, so a draft that binds the+ required paths but that the liveness prover can only call "unknown" ships instead of looping.+ Returns None when the deficiency is genuine."""credited = _evaluate_output_contract_for_code_block(ctx,workflow_yaml,@@ -5221,15 +5259,6 @@ def _credit_output_contract_submission()if credited is None or credited.has_deficiencies:return None- LOG.info(- "copilot_output_contract_submission_credited_at_ceiling",- block_label=evaluation.block_label,- canonical_output_contract_signature=evaluation.canonical_signature,- steering_reject_count=_output_contract_reject_count(ctx, evaluation.canonical_signature),- credited_shape_violations=list(evaluation.shape_violations),- credited_missing_code_return_paths=list(evaluation.missing_return_paths),- actuation_entry=actuation_entry,- )return credited@@ -5240,34 +5269,49 @@ def _credit_output_contract_submission_at_ceiling(workflow_yaml: str,raw_metadata: object,) -> _OutputContractEvaluation | None:- if not _output_contract_ceiling_reached(ctx, evaluation):- return None- return _credit_output_contract_submission(+ resolution = _credit_or_impose_output_contract(ctx,evaluation,workflow_yaml=workflow_yaml,raw_metadata=raw_metadata,- actuation_entry="reject_ceiling",+ credit_only=True,)+ return resolution.evaluation if resolution is not None else None-def _credit_or_impose_output_contract_at_ceiling(+def _resolve_output_contract_pass_path(ctx: AgentContext,evaluation: _OutputContractEvaluation,*,workflow_yaml: str,raw_metadata: object,-) -> _OutputContractCeilingResolution | None:- """Impose-or-credit at the convergence ceiling (SKY-12907): once steering has consumed the- reject budget on one canonical contract signature, the server stops asking the model."""- if not _output_contract_ceiling_reached(ctx, evaluation):- return None- return _credit_or_impose_output_contract(- ctx,- evaluation,- workflow_yaml=workflow_yaml,- raw_metadata=raw_metadata,- actuation_entry="reject_ceiling",+ credit_only: bool,+) -> OutputContractActuation:+ """Convert the pass-path preconditions to typed evidence and let the single resolver decide.+ Crediting and composing are probed here because the resolver is pure; probing mutates nothing+ and abstains exactly where the strict path already rejects."""+ credited = _credit_output_contract_submission(+ ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata+ )+ impose_available = (+ not credit_only+ and credited is None+ and _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml)+ and _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation) is not None+ )+ evidence = OutputContractActuationEvidence(+ imposed_available=False,+ click_only_spine=False,+ observed_required_values=bool(evaluation.observation_paths)+ and _observed_required_output_values(ctx, evaluation.observation_paths),+ prior_actuation=_prior_output_contract_actuation(ctx, evaluation.canonical_signature),+ prior_directive_unconsumed=False,+ steering_exhausted=_output_contract_steering_exhausted(ctx, evaluation, workflow_yaml),+ credit_available=credited is not None,+ scouted_binding_impose_available=impose_available,+ )+ return resolve_output_contract_actuation(+ family=classify_output_contract_bail_family(evaluation.shape_violations), evidence=evidence)@@ -5277,29 +5321,42 @@ def _credit_or_impose_output_contract(*,workflow_yaml: str,raw_metadata: object,- actuation_entry: str,+ credit_only: bool = False,) -> _OutputContractCeilingResolution | None:- """Credit first — is the strict rejection over-strict for this submission? Impose second —- when the recorded exploration pins the value's source, compose the return envelope and- metadata scaffold server-side from the same templates the steer directives quote. Returns- None when neither move is justified, leaving the caller's reject/ablation path untouched.- Armed by the reject ceiling in enforcing mode and by the ablation seam in log-only mode."""- credited = _credit_output_contract_submission(+ """Resolve a submission the steer loop can no longer move: credit it when the strict prover+ was merely uncertain, else impose the binding the recording pins. Returns None when the+ resolver picks neither, leaving the caller's reject/ablation path untouched."""+ actuation = _resolve_output_contract_pass_path(ctx,evaluation,workflow_yaml=workflow_yaml,raw_metadata=raw_metadata,- actuation_entry=actuation_entry,+ credit_only=credit_only,)- if credited is not None:+ if actuation.kind not in {+ OutputContractActuationKind.CREDIT,+ OutputContractActuationKind.IMPOSE_SCOUTED_BINDING,+ }:+ return None+ if actuation.kind == OutputContractActuationKind.CREDIT:+ credited = _credit_output_contract_submission(+ ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata+ )+ if credited is None:+ return None+ LOG.info(+ "copilot_output_contract_submission_credited_at_ceiling",+ block_label=evaluation.block_label,+ canonical_output_contract_signature=evaluation.canonical_signature,+ credited_shape_violations=list(evaluation.shape_violations),+ credited_missing_code_return_paths=list(evaluation.missing_return_paths),+ )return _OutputContractCeilingResolution(workflow_yaml=workflow_yaml,code_artifact_metadata=raw_metadata,evaluation=credited,mode="credited",)- if not _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml):- return Noneimposed = _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation)if imposed is None:return None@@ -5313,13 +5370,10 @@ def _credit_or_impose_output_contract()if reevaluated is None or reevaluated.has_deficiencies:return None- _mark_output_contract_imposed(ctx, evaluation.canonical_signature)LOG.info("copilot_output_contract_binding_imposed_at_ceiling",block_label=evaluation.block_label,canonical_output_contract_signature=evaluation.canonical_signature,- steering_reject_count=_output_contract_reject_count(ctx, evaluation.canonical_signature),- actuation_entry=actuation_entry,)return _OutputContractCeilingResolution(workflow_yaml=imposed_yaml,@@ -6312,12 +6366,9 @@ def _actuate_output_contract_bail(click_only_spine = _output_contract_click_only_spine(target_code, declaration_paths)observed_required_values = _observed_required_output_values(ctx, required_paths)loaded_result_evidence = ctx.latest_evaluate_result_composition_steer- result_source_producible = loaded_result_source_producible(loaded_result_evidence, target_code=target_code)- claimed_signature = ctx.latest_evaluate_result_composition_signature- if result_source_producible and claimed_signature not in (None, signature):- result_source_producible = False- elif result_source_producible:- ctx.latest_evaluate_result_composition_signature = signature+ result_source_producible = _loaded_result_source_producible_for_signature(+ ctx, target_code=target_code, signature=signature+ )if not click_only_spine or observed_required_values or result_source_producible:_clear_declick_attempt(ctx, signature)evidence = OutputContractActuationEvidence(@@ -13459,6 +13510,8 @@ async def _update_workflow() -> dict[str, Any]:ctx.output_contract_bail_actuated_this_call = False+ output_contract_imposed_signature = ""+def reject(*,error: str,@@ -13467,6 +13520,9 @@ async def _update_workflow(repair_context: CodeAuthoringRepairContext | None = None,record_repair_context_outcome: bool = True,) -> dict[str, Any]:+ # An imposition that never reached persistence must not refund the reject streak.+ if output_contract_imposed_signature:+ ctx.output_contract_imposed_since_last_reject_by_signature.pop(output_contract_imposed_signature, None)if repair_context is None:_clear_code_authoring_repair_context(ctx)elif record_repair_context_outcome:@@ -13805,52 +13861,53 @@ async def _update_workflow(output_contract_static_advisory_allowed = (output_contract_evaluation is not None and output_contract_evaluation.can_attempt_run)- output_contract_ceiling_resolution: _OutputContractCeilingResolution | None = None- if (+ output_contract_deficient = (output_contract_evaluation is not Noneand output_contract_evaluation.has_deficienciesand not output_contract_static_advisory_allowed- ):- output_contract_ceiling_resolution = _credit_or_impose_output_contract_at_ceiling(+ )+ output_contract_ceiling_resolution: _OutputContractCeilingResolution | None = None+ if output_contract_deficient:+ assert output_contract_evaluation is not None+ authored_fingerprint = _output_contract_structural_fingerprint(+ workflow_yaml, output_contract_evaluation.canonical_signature+ )+ # Recorded before the pass path resolves so the gate's would-block population still+ # counts the submissions credit-or-impose goes on to handle.+ output_contract_ablated = _record_output_contract_ablation_event(+ ctx,+ output_contract_evaluation,+ gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,+ blocked_tool="update_workflow",+ fingerprint=authored_fingerprint,+ )+ output_contract_ceiling_resolution = _credit_or_impose_output_contract(ctx,output_contract_evaluation,workflow_yaml=workflow_yaml,raw_metadata=params.get("code_artifact_metadata"),)- # Under log-only the reject ceiling never arms (no rejects are counted), so the deficient- # draft would persist as-is; arm the same composer at the ablation seam instead.- if output_contract_ceiling_resolution is None and copilot_author_time_gate_log_only_enabled(- ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID- ):- output_contract_ceiling_resolution = _credit_or_impose_output_contract(- ctx,- output_contract_evaluation,- workflow_yaml=workflow_yaml,- raw_metadata=params.get("code_artifact_metadata"),- actuation_entry="log_only_ablation",- )if output_contract_ceiling_resolution is not None:+ assert output_contract_evaluation is not Noneworkflow_yaml = output_contract_ceiling_resolution.workflow_yamlparams["workflow_yaml"] = workflow_yamlparams["code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadataparams["raw_code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadatactx.raw_code_artifact_metadata = output_contract_ceiling_resolution.code_artifact_metadata- output_contract_evaluation = output_contract_ceiling_resolution.evaluation- elif (- output_contract_evaluation is not None- and output_contract_evaluation.has_deficiencies- and not output_contract_static_advisory_allowed- ):- authored_fingerprint = _output_contract_structural_fingerprint(- workflow_yaml, output_contract_evaluation.canonical_signature- )- if not _record_output_contract_ablation_event(+ # The pass path inherits the advisory lane's adjudication: the draft persists now, and a+ # run that binds nothing reopens the ladder instead of shipping an unverified workflow.+ _arm_pending_run_evidence(ctx,- output_contract_evaluation,- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,- blocked_tool="update_workflow",- fingerprint=authored_fingerprint,- ):+ output_contract_evaluation.canonical_signature,+ set(output_contract_evaluation.observation_paths),+ )+ if output_contract_ceiling_resolution.mode == "imposed":+ _mark_output_contract_imposed(ctx, output_contract_evaluation.canonical_signature)+ output_contract_imposed_signature = output_contract_evaluation.canonical_signature+ output_contract_evaluation = output_contract_ceiling_resolution.evaluation+ elif output_contract_deficient:+ assert output_contract_evaluation is not None+ if not output_contract_ablated:payload = _record_output_contract_reject(ctx,output_contract_evaluation,diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.pyindex 9fec26a2d..b7853c49e 100644--- a/tests/unit/test_copilot_output_contract_actuation.py+++ b/tests/unit/test_copilot_output_contract_actuation.py@@ -7,10 +7,8 @@ OSS-synced: only synthetic labels and RFC-2606 placeholder identifiers.from __future__ import annotations-import hashlibimport itertoolsimport json-from pathlib import Pathfrom types import SimpleNamespaceimport pytest@@ -549,6 +547,7 @@ def _advisory_ctx() -> SimpleNamespace:output_contract_reject_count_by_signature={},output_contract_imposed_since_last_reject_by_signature={},output_contract_armed_directive_fingerprint_by_signature={},+ output_contract_last_reject_fingerprint_by_signature={},output_contract_output_owner_directive_candidates_by_signature={},output_contract_actuation_by_signature={},output_contract_actuation_count_by_signature={},@@ -718,6 +717,7 @@ def test_log_only_metadata_preflight_records_without_rejecting_or_consuming(missing_schema_paths=[],missing_return_paths=[],shape_violations=[],+ observation_paths=set(),)monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))@@ -754,6 +754,7 @@ def test_log_only_metadata_preflight_skips_run_attemptable_without_advisory(missing_schema_paths=[],missing_return_paths=[],shape_violations=[],+ observation_paths=set(),)monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))@@ -2761,14 +2762,11 @@ def test_ceiling_credits_uncertain_but_bound_submission() -> None:assert strict is not None and strict.has_deficiencies# Below the ceiling the server keeps steering: no credit, no imposition.- assert (- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)- is None- )+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)with capture_logs() as logs:- resolution = wu._credit_or_impose_output_contract_at_ceiling(+ resolution = wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)assert resolution is not None@@ -2786,6 +2784,7 @@ def test_credit_preserves_reject_streak_so_downstream_reject_keeps_ceiling() ->ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1credited = wu._credit_output_contract_submission_at_ceiling(@@ -2797,7 +2796,9 @@ def test_credit_preserves_reject_streak_so_downstream_reject_keeps_ceiling() ->ctx,strict.required_paths,reject_family="requested_output_contract_missing_output_coverage",- authored_structural_fingerprint="fp_downstream",+ authored_structural_fingerprint=wu._output_contract_structural_fingerprint(+ workflow_yaml, strict.canonical_signature+ ),)assert downstream == wu._MAX_OUTPUT_CONTRACT_REJECTS@@ -2811,17 +2812,14 @@ def test_ceiling_imposes_binding_from_scouted_observation() -> None:)assert strict is not Noneassert "output.record_id" in strict.missing_return_paths- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)# Enforcing-mode negative: without a recorded source, the reject path stays untouched.- assert (- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)- is None- )+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is Nonectx.scouted_output_covered_paths = {"output.record_id"}with capture_logs() as logs:- resolution = wu._credit_or_impose_output_contract_at_ceiling(+ resolution = wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)assert resolution is not None@@ -2831,7 +2829,6 @@ def test_ceiling_imposes_binding_from_scouted_observation() -> None:)assert 'return {"output": {"record_id": record_id}}' in imposed_codeassert resolution.evaluation.has_deficiencies is False- assert ctx.output_contract_imposed_since_last_reject_by_signature[strict.canonical_signature] is Trueassert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)@@ -2843,29 +2840,30 @@ def test_ceiling_abstains_when_no_binding_composable_even_with_observation() ->ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)ctx.scouted_output_covered_paths = {"output.record_id"}- assert (- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)- is None- )+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is Noneassert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None-def test_ceiling_hook_inert_in_log_only_mode() -> None:- ctx, workflow_yaml, metadata = _declaration_only_root_contract()- strict = wu._evaluate_output_contract_for_code_block(- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True- )- assert strict is not None- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})-- assert (- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)- is None- )+def test_pass_path_resolves_the_same_way_under_log_only_as_when_enforcing() -> None:+ resolutions = []+ for log_only in (False, True):+ ctx, workflow_yaml, metadata = _declaration_only_root_contract()+ if log_only:+ ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})+ strict = wu._evaluate_output_contract_for_code_block(+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True+ )+ assert strict is not None+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)+ resolution = wu._credit_or_impose_output_contract(+ ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata+ )+ assert resolution is not None+ resolutions.append(resolution.mode)+ assert resolutions == ["credited", "credited"]def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejects() -> None:@@ -2875,6 +2873,7 @@ def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejecctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)assert ctx.output_contract_reject_count_by_signature == {}with capture_logs() as logs:@@ -2883,15 +2882,11 @@ def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejecstrict,workflow_yaml=workflow_yaml,raw_metadata=metadata,- actuation_entry="log_only_ablation",)assert resolution is not Noneassert resolution.mode == "credited"assert resolution.evaluation.has_deficiencies is False- credited_logs = [- log for log in logs if log.get("event") == "copilot_output_contract_submission_credited_at_ceiling"- ]- assert credited_logs and credited_logs[0].get("actuation_entry") == "log_only_ablation"+ assert any(log.get("event") == "copilot_output_contract_submission_credited_at_ceiling" for log in logs)def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> None:@@ -2904,6 +2899,7 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> Nctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)assert ctx.output_contract_reject_count_by_signature == {}with capture_logs() as logs:@@ -2912,7 +2908,6 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> Nstrict,workflow_yaml=workflow_yaml,raw_metadata=metadata,- actuation_entry="log_only_ablation",)assert resolution is not Noneassert resolution.mode == "imposed"@@ -2920,8 +2915,7 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> Nwu._workflow_yaml_code_blocks_by_label(resolution.workflow_yaml)["extract_record"].get("code") or "")assert 'return {"output": {"record_id": record_id}}' in imposed_code- imposed_logs = [log for log in logs if log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling"]- assert imposed_logs and imposed_logs[0].get("actuation_entry") == "log_only_ablation"+ assert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_as_today() -> None:@@ -2934,6 +2928,7 @@ def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies+ _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)assert (wu._credit_or_impose_output_contract(@@ -2941,7 +2936,6 @@ def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_strict,workflow_yaml=omitted_yaml,raw_metadata=metadata,- actuation_entry="log_only_ablation",)is None)@@ -2956,12 +2950,9 @@ def test_ceiling_refuses_omitted_value_without_observation_byte_identically() ->ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None and strict.has_deficiencies- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1+ _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)- assert (- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=omitted_yaml, raw_metadata=metadata)- is None- )+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=omitted_yaml, raw_metadata=metadata) is Nonecredited = wu._evaluate_output_contract_for_code_block(ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True, credit_convergence_ceiling=True)@@ -2971,51 +2962,131 @@ def test_ceiling_refuses_omitted_value_without_observation_byte_identically() ->assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None-def test_ceiling_reached_at_final_counted_reject_not_before() -> None:+def test_steering_exhausted_requires_an_armed_directive_the_draft_did_not_move() -> None:ctx, workflow_yaml, metadata = _declaration_only_root_contract()strict = wu._evaluate_output_contract_for_code_block(ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True)assert strict is not None- counts = ctx.output_contract_reject_count_by_signature+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False- counts[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 2- assert wu._output_contract_ceiling_reached(ctx, strict) is False+ ctx.output_contract_actuation_count_by_signature[strict.canonical_signature] = 1+ ctx.output_contract_last_reject_fingerprint_by_signature[strict.canonical_signature] = "fp_from_another_draft"+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False- counts[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1- assert wu._output_contract_ceiling_reached(ctx, strict) is True+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is True-_PROD_METADATA_CORPUS = Path(- "/Users/andrewneilson/sky/.gauntlet-cache/output-contract-binder-20260722/prod_rejected_metadata_captures.json"-)-_PROD_METADATA_CORPUS_SHA256 = "0f99fa0ddb8ce52ad4d6191ea307277ff373ccfed3a647caa46d80d6ee2aa060"+def test_loaded_result_evidence_claimed_by_another_contract_does_not_authorize_imposition() -> None:+ ctx = _record_criterion_ctx()+ ctx.scouted_output_covered_paths = set()+ workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')+ metadata = _record_contract_metadata()+ strict = wu._evaluate_output_contract_for_code_block(+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True+ )+ assert strict is not None and strict.has_deficiencies+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)+ ctx.latest_evaluate_result_composition_steer = loaded_result_composition_evidence_from_page(+ {"result_containers": [{"selector": "#record", "row_count": 1, "sample_rows": ["Example record"]}]},+ source_tool="evaluate",+ source_url="https://example.com/results",+ )+ ctx.latest_evaluate_result_composition_signature = "sig_for_a_different_contract"++ assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is False+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None+ assert ctx.latest_evaluate_result_composition_signature == "sig_for_a_different_contract"+ ctx.latest_evaluate_result_composition_signature = None+ assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is True+ assert ctx.latest_evaluate_result_composition_signature == strict.canonical_signature-def _prod_capture_binding_dimension(payload: list[dict[str, object]]) -> tuple[bool, bool]:- element = payload[0]- observations = element.get("exploration_observations") or []- statuses = {str(observation.get("status")) for observation in observations}- binding_source_on_record = len(observations) > 0- liveness_uncertain = bool(statuses) and statuses <= {"observed_not_verified"}- return binding_source_on_record, liveness_uncertain+def _steering_exhausted_evidence(**overrides: object) -> OutputContractActuationEvidence:+ base = dict(+ imposed_available=False,+ click_only_spine=False,+ observed_required_values=False,+ prior_actuation=True,+ prior_directive_unconsumed=True,+ steering_exhausted=True,+ )+ base.update(overrides)+ return OutputContractActuationEvidence(**base) # type: ignore[arg-type][email protected](not _PROD_METADATA_CORPUS.exists(), reason="prod evidence archive not present in this environment")-def test_prod_rejected_metadata_corpus_adjudication() -> None:- raw = _PROD_METADATA_CORPUS.read_bytes()- assert hashlib.sha256(raw).hexdigest() == _PROD_METADATA_CORPUS_SHA256- captures = [- row["attributes"]["attributes"]["rejected_code_artifact_metadata_payload"] for row in json.loads(raw)["data"]- ]- assert len(captures) == 6-- for payload in captures:- element = payload[0]- # One shared extraction block label across all captures; the exact name stays in- # the local evidence archive (never committed — it identifies the target site).- assert element["block_label"] == captures[0][0]["block_label"]- binding_source_on_record, liveness_uncertain = _prod_capture_binding_dimension(payload)- assert binding_source_on_record is True- assert liveness_uncertain is True+def _arm_steering_exhausted(ctx: object, signature: str, workflow_yaml: str) -> None:+ """Steering is exhausted once a directive is armed and the draft's canonical structural+ fingerprint is unchanged across it — the typed replacement for a reject-count ceiling."""+ ctx.output_contract_actuation_count_by_signature[signature] = 1+ ctx.output_contract_last_reject_fingerprint_by_signature[signature] = wu._output_contract_structural_fingerprint(+ workflow_yaml, signature+ )+++def test_resolver_credits_before_imposing_once_steering_is_exhausted() -> None:+ credit_and_impose = _steering_exhausted_evidence(credit_available=True, scouted_binding_impose_available=True)+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=credit_and_impose).kind+ == OutputContractActuationKind.CREDIT+ )++ impose_only = _steering_exhausted_evidence(scouted_binding_impose_available=True)+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=impose_only).kind+ == OutputContractActuationKind.IMPOSE_SCOUTED_BINDING+ )+++def test_resolver_pass_path_preempts_the_unobservable_source_terminal() -> None:+ would_terminal = _steering_exhausted_evidence(+ click_only_spine=True,+ declick_attempt_failed=True,+ credit_available=True,+ )+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=would_terminal).kind+ == OutputContractActuationKind.CREDIT+ )++ without_pass_path = _steering_exhausted_evidence(click_only_spine=True, declick_attempt_failed=True)+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=without_pass_path).kind+ == OutputContractActuationKind.BLOCKED_TERMINAL+ )+++def test_resolver_ignores_credit_and_impose_until_steering_is_exhausted() -> None:+ for kind_flags in ({"credit_available": True}, {"scouted_binding_impose_available": True}):+ armed = _steering_exhausted_evidence(steering_exhausted=False, **kind_flags)+ unarmed = _steering_exhausted_evidence(steering_exhausted=False)+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=armed).kind+ == resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=unarmed).kind+ )+++def test_resolver_keeps_existing_imposition_ahead_of_the_pass_path() -> None:+ both = _steering_exhausted_evidence(imposed_available=True, credit_available=True)+ assert (+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=both).kind+ == OutputContractActuationKind.IMPOSED+ )+++def test_observing_one_path_does_not_authorize_imposing_a_multi_path_contract() -> None:+ ctx = _record_criterion_ctx()+ ctx.scouted_output_covered_paths = {"output.record_id"}+ evaluation = SimpleNamespace(+ observation_paths={"output.record_id", "output.record_total"},+ required_paths={"output.record_id", "output.record_total"},+ block_label="extract_record",+ canonical_signature="sig_multi",+ )++ assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is False++ ctx.scouted_output_covered_paths = {"output.record_id", "output.record_total"}+ assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is True