Skip to content

Automate browser based workflows with AI

Resolve the output-contract pass path through the single actuation resolver (SKY-12907) (#7706)

Andrew Neilsonde8c2b30ccd2
diff --git a/skyvern/forge/sdk/copilot/output_contracts.py b/skyvern/forge/sdk/copilot/output_contracts.py
index 11d45ab4c..c84aea29e 100644
--- a/skyvern/forge/sdk/copilot/output_contracts.py
+++ b/skyvern/forge/sdk/copilot/output_contracts.py
@@ -23,6 +23,8 @@ class OutputContractBailFamily(StrEnum):
class OutputContractActuationKind(StrEnum):
IMPOSED = "imposed"
+ CREDIT = "credit"
+ IMPOSE_SCOUTED_BINDING = "impose_scouted_binding"
STRUCTURE_DIRECTIVE = "structure_directive"
ADVISORY_RUN = "advisory_run"
BLOCKED_TERMINAL = "blocked_terminal"
@@ -63,6 +65,11 @@ class OutputContractActuationEvidence:
consumed_run_bound_required_path: bool = False
consumed_run_carried_page_extraction: bool = False
loaded_result_source_producible: bool = False
+ # Steering produced no structural movement: the canonical structural fingerprint did not
+ # change across the consumed directives. Never a reject count — counts inflate.
+ steering_exhausted: bool = False
+ credit_available: bool = False
+ scouted_binding_impose_available: bool = False
@dataclass(frozen=True)
@@ -90,9 +97,17 @@ def resolve_output_contract_actuation(
page-source extraction, and that it still bound no required path. A consumed run whose output was
observed but bound nothing without a page-source extraction on board is not exhaustion evidence —
it re-enters the ladder once so the stronger page-source imposition can bind the on-screen values a
- code static-return provably cannot key."""
+ code static-return provably cannot key. Once steering is exhausted the server resolves the
+ submission itself — crediting a draft the strict prover can only call uncertain, else imposing the
+ binding the recording pins — so a satisfiable submission has a reachable pass path instead of only
+ a terminal; both inherit the advisory lane's post-run adjudication."""
if evidence.imposed_available:
return OutputContractActuation(OutputContractActuationKind.IMPOSED, family)
+ if evidence.steering_exhausted:
+ if evidence.credit_available:
+ return OutputContractActuation(OutputContractActuationKind.CREDIT, family)
+ if evidence.scouted_binding_impose_available:
+ return OutputContractActuation(OutputContractActuationKind.IMPOSE_SCOUTED_BINDING, family)
grantable_source = (
evidence.advisory_run_grantable and evidence.advisory_state != OutputContractAdvisoryState.CONSUMED
)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 2577eb3da..dc5e1f61f 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -5120,23 +5120,51 @@ class _OutputContractCeilingResolution:
mode: Literal["credited", "imposed"]
+def _loaded_result_source_producible_for_signature(
+ ctx: AgentContext,
+ *,
+ target_code: str,
+ signature: str,
+) -> bool:
+ """Loaded-result evidence is producible for this contract only while it is unclaimed or
+ already claimed by this signature; evidence claimed by another contract does not carry
+ over. The first contract to find it producible claims it."""
+ if not loaded_result_source_producible(ctx.latest_evaluate_result_composition_steer, target_code=target_code):
+ return False
+ claimed_signature = ctx.latest_evaluate_result_composition_signature
+ if claimed_signature not in (None, signature):
+ return False
+ ctx.latest_evaluate_result_composition_signature = signature
+ return True
+
+
+def _every_required_output_value_observed(ctx: AgentContext, required_paths: set[str]) -> bool:
+ """Imposition writes a binding for every required path, so authorization needs observation
+ coverage for every one of them — a single overlapping path would otherwise license composing
+ the rest as constants the recording never saw."""
+ if not required_paths:
+ return False
+ return all(_observed_required_output_values(ctx, {path}) for path in required_paths)
+
+
def _output_contract_binding_source_on_record(
ctx: AgentContext,
evaluation: _OutputContractEvaluation,
workflow_yaml: str,
) -> bool:
"""Whether the recorded exploration already pins where the requested value lives: either the
- scout observed the required paths, or a producible loaded-result composition anchors the
+ scout observed every required path, or a producible loaded-result composition anchors the
target block's selectors. Imposition is only justified when this is on record."""
binding_paths = evaluation.observation_paths or evaluation.required_paths
- if binding_paths and _observed_required_output_values(ctx, binding_paths):
+ if _every_required_output_value_observed(ctx, binding_paths):
return True
target_block = _workflow_yaml_code_blocks_by_label(workflow_yaml).get(evaluation.block_label)
if target_block is None:
return False
- return loaded_result_source_producible(
- ctx.latest_evaluate_result_composition_steer,
+ return _loaded_result_source_producible_for_signature(
+ ctx,
target_code=str(target_block.get("code") or ""),
+ signature=evaluation.canonical_signature,
)
@@ -5192,12 +5220,24 @@ def _impose_output_contract_binding_at_ceiling(
return imposed_yaml, scaffolded_metadata
-def _output_contract_ceiling_reached(ctx: AgentContext, evaluation: _OutputContractEvaluation) -> bool:
- if not evaluation.canonical_signature or not evaluation.block_label:
+def _output_contract_steering_exhausted(
+ ctx: AgentContext,
+ evaluation: _OutputContractEvaluation,
+ workflow_yaml: str,
+) -> bool:
+ """Steering has stopped moving the draft: this submission came back with the same canonical
+ structural fingerprint the last reject already steered on, so re-emitting the directive cannot
+ converge. Keyed on that fingerprint rather than reject counts, which inflate, and unreachable
+ on a first submission because no prior reject fingerprint exists yet."""
+ signature = evaluation.canonical_signature
+ if not signature or not evaluation.block_label:
return False
- if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
+ if not _prior_output_contract_actuation(ctx, signature):
return False
- return _output_contract_reject_count(ctx, evaluation.canonical_signature) + 1 >= _MAX_OUTPUT_CONTRACT_REJECTS
+ steered_fingerprint = ctx.output_contract_last_reject_fingerprint_by_signature.get(signature)
+ if not steered_fingerprint:
+ return False
+ return steered_fingerprint == _output_contract_structural_fingerprint(workflow_yaml, signature)
def _credit_output_contract_submission(
@@ -5206,12 +5246,10 @@ def _credit_output_contract_submission(
*,
workflow_yaml: str,
raw_metadata: object,
- actuation_entry: str,
) -> _OutputContractEvaluation | None:
- """Credit move: re-adjudicate the submission with static uncertainty credited, so a draft
- that binds the required paths but that the liveness prover can only call "unknown" ships
- instead of looping. Returns the crediting evaluation when the strict rejection was
- over-strict, None when the deficiency is genuine."""
+ """Re-adjudicate the submission with static uncertainty credited, so a draft that binds the
+ required paths but that the liveness prover can only call "unknown" ships instead of looping.
+ Returns None when the deficiency is genuine."""
credited = _evaluate_output_contract_for_code_block(
ctx,
workflow_yaml,
@@ -5221,15 +5259,6 @@ def _credit_output_contract_submission(
)
if credited is None or credited.has_deficiencies:
return None
- LOG.info(
- "copilot_output_contract_submission_credited_at_ceiling",
- block_label=evaluation.block_label,
- canonical_output_contract_signature=evaluation.canonical_signature,
- steering_reject_count=_output_contract_reject_count(ctx, evaluation.canonical_signature),
- credited_shape_violations=list(evaluation.shape_violations),
- credited_missing_code_return_paths=list(evaluation.missing_return_paths),
- actuation_entry=actuation_entry,
- )
return credited
@@ -5240,34 +5269,49 @@ def _credit_output_contract_submission_at_ceiling(
workflow_yaml: str,
raw_metadata: object,
) -> _OutputContractEvaluation | None:
- if not _output_contract_ceiling_reached(ctx, evaluation):
- return None
- return _credit_output_contract_submission(
+ resolution = _credit_or_impose_output_contract(
ctx,
evaluation,
workflow_yaml=workflow_yaml,
raw_metadata=raw_metadata,
- actuation_entry="reject_ceiling",
+ credit_only=True,
)
+ return resolution.evaluation if resolution is not None else None
-def _credit_or_impose_output_contract_at_ceiling(
+def _resolve_output_contract_pass_path(
ctx: AgentContext,
evaluation: _OutputContractEvaluation,
*,
workflow_yaml: str,
raw_metadata: object,
-) -> _OutputContractCeilingResolution | None:
- """Impose-or-credit at the convergence ceiling (SKY-12907): once steering has consumed the
- reject budget on one canonical contract signature, the server stops asking the model."""
- if not _output_contract_ceiling_reached(ctx, evaluation):
- return None
- return _credit_or_impose_output_contract(
- ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=raw_metadata,
- actuation_entry="reject_ceiling",
+ credit_only: bool,
+) -> OutputContractActuation:
+ """Convert the pass-path preconditions to typed evidence and let the single resolver decide.
+ Crediting and composing are probed here because the resolver is pure; probing mutates nothing
+ and abstains exactly where the strict path already rejects."""
+ credited = _credit_output_contract_submission(
+ ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata
+ )
+ impose_available = (
+ not credit_only
+ and credited is None
+ and _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml)
+ and _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation) is not None
+ )
+ evidence = OutputContractActuationEvidence(
+ imposed_available=False,
+ click_only_spine=False,
+ observed_required_values=bool(evaluation.observation_paths)
+ and _observed_required_output_values(ctx, evaluation.observation_paths),
+ prior_actuation=_prior_output_contract_actuation(ctx, evaluation.canonical_signature),
+ prior_directive_unconsumed=False,
+ steering_exhausted=_output_contract_steering_exhausted(ctx, evaluation, workflow_yaml),
+ credit_available=credited is not None,
+ scouted_binding_impose_available=impose_available,
+ )
+ return resolve_output_contract_actuation(
+ family=classify_output_contract_bail_family(evaluation.shape_violations), evidence=evidence
)
@@ -5277,29 +5321,42 @@ def _credit_or_impose_output_contract(
*,
workflow_yaml: str,
raw_metadata: object,
- actuation_entry: str,
+ credit_only: bool = False,
) -> _OutputContractCeilingResolution | None:
- """Credit first — is the strict rejection over-strict for this submission? Impose second —
- when the recorded exploration pins the value's source, compose the return envelope and
- metadata scaffold server-side from the same templates the steer directives quote. Returns
- None when neither move is justified, leaving the caller's reject/ablation path untouched.
- Armed by the reject ceiling in enforcing mode and by the ablation seam in log-only mode."""
- credited = _credit_output_contract_submission(
+ """Resolve a submission the steer loop can no longer move: credit it when the strict prover
+ was merely uncertain, else impose the binding the recording pins. Returns None when the
+ resolver picks neither, leaving the caller's reject/ablation path untouched."""
+ actuation = _resolve_output_contract_pass_path(
ctx,
evaluation,
workflow_yaml=workflow_yaml,
raw_metadata=raw_metadata,
- actuation_entry=actuation_entry,
+ credit_only=credit_only,
)
- if credited is not None:
+ if actuation.kind not in {
+ OutputContractActuationKind.CREDIT,
+ OutputContractActuationKind.IMPOSE_SCOUTED_BINDING,
+ }:
+ return None
+ if actuation.kind == OutputContractActuationKind.CREDIT:
+ credited = _credit_output_contract_submission(
+ ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata
+ )
+ if credited is None:
+ return None
+ LOG.info(
+ "copilot_output_contract_submission_credited_at_ceiling",
+ block_label=evaluation.block_label,
+ canonical_output_contract_signature=evaluation.canonical_signature,
+ credited_shape_violations=list(evaluation.shape_violations),
+ credited_missing_code_return_paths=list(evaluation.missing_return_paths),
+ )
return _OutputContractCeilingResolution(
workflow_yaml=workflow_yaml,
code_artifact_metadata=raw_metadata,
evaluation=credited,
mode="credited",
)
- if not _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml):
- return None
imposed = _impose_output_contract_binding_at_ceiling(ctx, workflow_yaml, raw_metadata, evaluation)
if imposed is None:
return None
@@ -5313,13 +5370,10 @@ def _credit_or_impose_output_contract(
)
if reevaluated is None or reevaluated.has_deficiencies:
return None
- _mark_output_contract_imposed(ctx, evaluation.canonical_signature)
LOG.info(
"copilot_output_contract_binding_imposed_at_ceiling",
block_label=evaluation.block_label,
canonical_output_contract_signature=evaluation.canonical_signature,
- steering_reject_count=_output_contract_reject_count(ctx, evaluation.canonical_signature),
- actuation_entry=actuation_entry,
)
return _OutputContractCeilingResolution(
workflow_yaml=imposed_yaml,
@@ -6312,12 +6366,9 @@ def _actuate_output_contract_bail(
click_only_spine = _output_contract_click_only_spine(target_code, declaration_paths)
observed_required_values = _observed_required_output_values(ctx, required_paths)
loaded_result_evidence = ctx.latest_evaluate_result_composition_steer
- result_source_producible = loaded_result_source_producible(loaded_result_evidence, target_code=target_code)
- claimed_signature = ctx.latest_evaluate_result_composition_signature
- if result_source_producible and claimed_signature not in (None, signature):
- result_source_producible = False
- elif result_source_producible:
- ctx.latest_evaluate_result_composition_signature = signature
+ result_source_producible = _loaded_result_source_producible_for_signature(
+ ctx, target_code=target_code, signature=signature
+ )
if not click_only_spine or observed_required_values or result_source_producible:
_clear_declick_attempt(ctx, signature)
evidence = OutputContractActuationEvidence(
@@ -13459,6 +13510,8 @@ async def _update_workflow(
) -> dict[str, Any]:
ctx.output_contract_bail_actuated_this_call = False
+ output_contract_imposed_signature = ""
+
def reject(
*,
error: str,
@@ -13467,6 +13520,9 @@ async def _update_workflow(
repair_context: CodeAuthoringRepairContext | None = None,
record_repair_context_outcome: bool = True,
) -> dict[str, Any]:
+ # An imposition that never reached persistence must not refund the reject streak.
+ if output_contract_imposed_signature:
+ ctx.output_contract_imposed_since_last_reject_by_signature.pop(output_contract_imposed_signature, None)
if repair_context is None:
_clear_code_authoring_repair_context(ctx)
elif record_repair_context_outcome:
@@ -13805,52 +13861,53 @@ async def _update_workflow(
output_contract_static_advisory_allowed = (
output_contract_evaluation is not None and output_contract_evaluation.can_attempt_run
)
- output_contract_ceiling_resolution: _OutputContractCeilingResolution | None = None
- if (
+ output_contract_deficient = (
output_contract_evaluation is not None
and output_contract_evaluation.has_deficiencies
and not output_contract_static_advisory_allowed
- ):
- output_contract_ceiling_resolution = _credit_or_impose_output_contract_at_ceiling(
+ )
+ output_contract_ceiling_resolution: _OutputContractCeilingResolution | None = None
+ if output_contract_deficient:
+ assert output_contract_evaluation is not None
+ authored_fingerprint = _output_contract_structural_fingerprint(
+ workflow_yaml, output_contract_evaluation.canonical_signature
+ )
+ # Recorded before the pass path resolves so the gate's would-block population still
+ # counts the submissions credit-or-impose goes on to handle.
+ output_contract_ablated = _record_output_contract_ablation_event(
+ ctx,
+ output_contract_evaluation,
+ gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
+ blocked_tool="update_workflow",
+ fingerprint=authored_fingerprint,
+ )
+ output_contract_ceiling_resolution = _credit_or_impose_output_contract(
ctx,
output_contract_evaluation,
workflow_yaml=workflow_yaml,
raw_metadata=params.get("code_artifact_metadata"),
)
- # Under log-only the reject ceiling never arms (no rejects are counted), so the deficient
- # draft would persist as-is; arm the same composer at the ablation seam instead.
- if output_contract_ceiling_resolution is None and copilot_author_time_gate_log_only_enabled(
- ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID
- ):
- output_contract_ceiling_resolution = _credit_or_impose_output_contract(
- ctx,
- output_contract_evaluation,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- actuation_entry="log_only_ablation",
- )
if output_contract_ceiling_resolution is not None:
+ assert output_contract_evaluation is not None
workflow_yaml = output_contract_ceiling_resolution.workflow_yaml
params["workflow_yaml"] = workflow_yaml
params["code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadata
params["raw_code_artifact_metadata"] = output_contract_ceiling_resolution.code_artifact_metadata
ctx.raw_code_artifact_metadata = output_contract_ceiling_resolution.code_artifact_metadata
- output_contract_evaluation = output_contract_ceiling_resolution.evaluation
- elif (
- output_contract_evaluation is not None
- and output_contract_evaluation.has_deficiencies
- and not output_contract_static_advisory_allowed
- ):
- authored_fingerprint = _output_contract_structural_fingerprint(
- workflow_yaml, output_contract_evaluation.canonical_signature
- )
- if not _record_output_contract_ablation_event(
+ # The pass path inherits the advisory lane's adjudication: the draft persists now, and a
+ # run that binds nothing reopens the ladder instead of shipping an unverified workflow.
+ _arm_pending_run_evidence(
ctx,
- output_contract_evaluation,
- gate_id=_OUTPUT_CONTRACT_ABLATION_GATE_ID,
- blocked_tool="update_workflow",
- fingerprint=authored_fingerprint,
- ):
+ output_contract_evaluation.canonical_signature,
+ set(output_contract_evaluation.observation_paths),
+ )
+ if output_contract_ceiling_resolution.mode == "imposed":
+ _mark_output_contract_imposed(ctx, output_contract_evaluation.canonical_signature)
+ output_contract_imposed_signature = output_contract_evaluation.canonical_signature
+ output_contract_evaluation = output_contract_ceiling_resolution.evaluation
+ elif output_contract_deficient:
+ assert output_contract_evaluation is not None
+ if not output_contract_ablated:
payload = _record_output_contract_reject(
ctx,
output_contract_evaluation,
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index 9fec26a2d..b7853c49e 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -7,10 +7,8 @@ OSS-synced: only synthetic labels and RFC-2606 placeholder identifiers.
from __future__ import annotations
-import hashlib
import itertools
import json
-from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -549,6 +547,7 @@ def _advisory_ctx() -> SimpleNamespace:
output_contract_reject_count_by_signature={},
output_contract_imposed_since_last_reject_by_signature={},
output_contract_armed_directive_fingerprint_by_signature={},
+ output_contract_last_reject_fingerprint_by_signature={},
output_contract_output_owner_directive_candidates_by_signature={},
output_contract_actuation_by_signature={},
output_contract_actuation_count_by_signature={},
@@ -718,6 +717,7 @@ def test_log_only_metadata_preflight_records_without_rejecting_or_consuming(
missing_schema_paths=[],
missing_return_paths=[],
shape_violations=[],
+ observation_paths=set(),
)
monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))
monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))
@@ -754,6 +754,7 @@ def test_log_only_metadata_preflight_skips_run_attemptable_without_advisory(
missing_schema_paths=[],
missing_return_paths=[],
shape_violations=[],
+ observation_paths=set(),
)
monkeypatch.setattr(wu, "_impose_output_contract_envelope_after_steering", lambda *args: (args[1], args[2], False))
monkeypatch.setattr(wu, "_scaffold_metadata_contract_for_update", lambda *args: (args[2], False))
@@ -2761,14 +2762,11 @@ def test_ceiling_credits_uncertain_but_bound_submission() -> None:
assert strict is not None and strict.has_deficiencies
# Below the ceiling the server keeps steering: no credit, no imposition.
- assert (
- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)
- is None
- )
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract_at_ceiling(
+ resolution = wu._credit_or_impose_output_contract(
ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
)
assert resolution is not None
@@ -2786,6 +2784,7 @@ def test_credit_preserves_reject_streak_so_downstream_reject_keeps_ceiling() ->
ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
credited = wu._credit_output_contract_submission_at_ceiling(
@@ -2797,7 +2796,9 @@ def test_credit_preserves_reject_streak_so_downstream_reject_keeps_ceiling() ->
ctx,
strict.required_paths,
reject_family="requested_output_contract_missing_output_coverage",
- authored_structural_fingerprint="fp_downstream",
+ authored_structural_fingerprint=wu._output_contract_structural_fingerprint(
+ workflow_yaml, strict.canonical_signature
+ ),
)
assert downstream == wu._MAX_OUTPUT_CONTRACT_REJECTS
@@ -2811,17 +2812,14 @@ def test_ceiling_imposes_binding_from_scouted_observation() -> None:
)
assert strict is not None
assert "output.record_id" in strict.missing_return_paths
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
# Enforcing-mode negative: without a recorded source, the reject path stays untouched.
- assert (
- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)
- is None
- )
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
ctx.scouted_output_covered_paths = {"output.record_id"}
with capture_logs() as logs:
- resolution = wu._credit_or_impose_output_contract_at_ceiling(
+ resolution = wu._credit_or_impose_output_contract(
ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
)
assert resolution is not None
@@ -2831,7 +2829,6 @@ def test_ceiling_imposes_binding_from_scouted_observation() -> None:
)
assert 'return {"output": {"record_id": record_id}}' in imposed_code
assert resolution.evaluation.has_deficiencies is False
- assert ctx.output_contract_imposed_since_last_reject_by_signature[strict.canonical_signature] is True
assert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)
@@ -2843,29 +2840,30 @@ def test_ceiling_abstains_when_no_binding_composable_even_with_observation() ->
ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
ctx.scouted_output_covered_paths = {"output.record_id"}
- assert (
- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)
- is None
- )
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None
-def test_ceiling_hook_inert_in_log_only_mode() -> None:
- ctx, workflow_yaml, metadata = _declaration_only_root_contract()
- strict = wu._evaluate_output_contract_for_code_block(
- ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
- )
- assert strict is not None
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
- ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
-
- assert (
- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata)
- is None
- )
+def test_pass_path_resolves_the_same_way_under_log_only_as_when_enforcing() -> None:
+ resolutions = []
+ for log_only in (False, True):
+ ctx, workflow_yaml, metadata = _declaration_only_root_contract()
+ if log_only:
+ ctx.author_time_gate_log_only_ids = frozenset({wu._OUTPUT_CONTRACT_ABLATION_GATE_ID})
+ strict = wu._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
+ assert strict is not None
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
+ resolution = wu._credit_or_impose_output_contract(
+ ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata
+ )
+ assert resolution is not None
+ resolutions.append(resolution.mode)
+ assert resolutions == ["credited", "credited"]
def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejects() -> None:
@@ -2875,6 +2873,7 @@ def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejec
ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
assert ctx.output_contract_reject_count_by_signature == {}
with capture_logs() as logs:
@@ -2883,15 +2882,11 @@ def test_log_only_ablation_entry_credits_qualifying_submission_without_any_rejec
strict,
workflow_yaml=workflow_yaml,
raw_metadata=metadata,
- actuation_entry="log_only_ablation",
)
assert resolution is not None
assert resolution.mode == "credited"
assert resolution.evaluation.has_deficiencies is False
- credited_logs = [
- log for log in logs if log.get("event") == "copilot_output_contract_submission_credited_at_ceiling"
- ]
- assert credited_logs and credited_logs[0].get("actuation_entry") == "log_only_ablation"
+ assert any(log.get("event") == "copilot_output_contract_submission_credited_at_ceiling" for log in logs)
def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> None:
@@ -2904,6 +2899,7 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> N
ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
assert ctx.output_contract_reject_count_by_signature == {}
with capture_logs() as logs:
@@ -2912,7 +2908,6 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> N
strict,
workflow_yaml=workflow_yaml,
raw_metadata=metadata,
- actuation_entry="log_only_ablation",
)
assert resolution is not None
assert resolution.mode == "imposed"
@@ -2920,8 +2915,7 @@ def test_log_only_ablation_entry_imposes_binding_from_scouted_observation() -> N
wu._workflow_yaml_code_blocks_by_label(resolution.workflow_yaml)["extract_record"].get("code") or ""
)
assert 'return {"output": {"record_id": record_id}}' in imposed_code
- imposed_logs = [log for log in logs if log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling"]
- assert imposed_logs and imposed_logs[0].get("actuation_entry") == "log_only_ablation"
+ assert any(log.get("event") == "copilot_output_contract_binding_imposed_at_ceiling" for log in logs)
def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_as_today() -> None:
@@ -2934,6 +2928,7 @@ def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_
ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
+ _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)
assert (
wu._credit_or_impose_output_contract(
@@ -2941,7 +2936,6 @@ def test_log_only_ablation_entry_abstains_without_observation_so_draft_persists_
strict,
workflow_yaml=omitted_yaml,
raw_metadata=metadata,
- actuation_entry="log_only_ablation",
)
is None
)
@@ -2956,12 +2950,9 @@ def test_ceiling_refuses_omitted_value_without_observation_byte_identically() ->
ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None and strict.has_deficiencies
- ctx.output_contract_reject_count_by_signature[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
+ _arm_steering_exhausted(ctx, strict.canonical_signature, omitted_yaml)
- assert (
- wu._credit_or_impose_output_contract_at_ceiling(ctx, strict, workflow_yaml=omitted_yaml, raw_metadata=metadata)
- is None
- )
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=omitted_yaml, raw_metadata=metadata) is None
credited = wu._evaluate_output_contract_for_code_block(
ctx, omitted_yaml, metadata, enforce_value_bearing_liveness=True, credit_convergence_ceiling=True
)
@@ -2971,51 +2962,131 @@ def test_ceiling_refuses_omitted_value_without_observation_byte_identically() ->
assert ctx.output_contract_imposed_since_last_reject_by_signature.get(strict.canonical_signature) is None
-def test_ceiling_reached_at_final_counted_reject_not_before() -> None:
+def test_steering_exhausted_requires_an_armed_directive_the_draft_did_not_move() -> None:
ctx, workflow_yaml, metadata = _declaration_only_root_contract()
strict = wu._evaluate_output_contract_for_code_block(
ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
)
assert strict is not None
- counts = ctx.output_contract_reject_count_by_signature
+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False
- counts[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 2
- assert wu._output_contract_ceiling_reached(ctx, strict) is False
+ ctx.output_contract_actuation_count_by_signature[strict.canonical_signature] = 1
+ ctx.output_contract_last_reject_fingerprint_by_signature[strict.canonical_signature] = "fp_from_another_draft"
+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is False
- counts[strict.canonical_signature] = wu._MAX_OUTPUT_CONTRACT_REJECTS - 1
- assert wu._output_contract_ceiling_reached(ctx, strict) is True
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
+ assert wu._output_contract_steering_exhausted(ctx, strict, workflow_yaml) is True
-_PROD_METADATA_CORPUS = Path(
- "/Users/andrewneilson/sky/.gauntlet-cache/output-contract-binder-20260722/prod_rejected_metadata_captures.json"
-)
-_PROD_METADATA_CORPUS_SHA256 = "0f99fa0ddb8ce52ad4d6191ea307277ff373ccfed3a647caa46d80d6ee2aa060"
+def test_loaded_result_evidence_claimed_by_another_contract_does_not_authorize_imposition() -> None:
+ ctx = _record_criterion_ctx()
+ ctx.scouted_output_covered_paths = set()
+ workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')
+ metadata = _record_contract_metadata()
+ strict = wu._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
+ assert strict is not None and strict.has_deficiencies
+ _arm_steering_exhausted(ctx, strict.canonical_signature, workflow_yaml)
+ ctx.latest_evaluate_result_composition_steer = loaded_result_composition_evidence_from_page(
+ {"result_containers": [{"selector": "#record", "row_count": 1, "sample_rows": ["Example record"]}]},
+ source_tool="evaluate",
+ source_url="https://example.com/results",
+ )
+ ctx.latest_evaluate_result_composition_signature = "sig_for_a_different_contract"
+
+ assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is False
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
+ assert ctx.latest_evaluate_result_composition_signature == "sig_for_a_different_contract"
+ ctx.latest_evaluate_result_composition_signature = None
+ assert wu._output_contract_binding_source_on_record(ctx, strict, workflow_yaml) is True
+ assert ctx.latest_evaluate_result_composition_signature == strict.canonical_signature
-def _prod_capture_binding_dimension(payload: list[dict[str, object]]) -> tuple[bool, bool]:
- element = payload[0]
- observations = element.get("exploration_observations") or []
- statuses = {str(observation.get("status")) for observation in observations}
- binding_source_on_record = len(observations) > 0
- liveness_uncertain = bool(statuses) and statuses <= {"observed_not_verified"}
- return binding_source_on_record, liveness_uncertain
+def _steering_exhausted_evidence(**overrides: object) -> OutputContractActuationEvidence:
+ base = dict(
+ imposed_available=False,
+ click_only_spine=False,
+ observed_required_values=False,
+ prior_actuation=True,
+ prior_directive_unconsumed=True,
+ steering_exhausted=True,
+ )
+ base.update(overrides)
+ return OutputContractActuationEvidence(**base) # type: ignore[arg-type]
[email protected](not _PROD_METADATA_CORPUS.exists(), reason="prod evidence archive not present in this environment")
-def test_prod_rejected_metadata_corpus_adjudication() -> None:
- raw = _PROD_METADATA_CORPUS.read_bytes()
- assert hashlib.sha256(raw).hexdigest() == _PROD_METADATA_CORPUS_SHA256
- captures = [
- row["attributes"]["attributes"]["rejected_code_artifact_metadata_payload"] for row in json.loads(raw)["data"]
- ]
- assert len(captures) == 6
-
- for payload in captures:
- element = payload[0]
- # One shared extraction block label across all captures; the exact name stays in
- # the local evidence archive (never committed — it identifies the target site).
- assert element["block_label"] == captures[0][0]["block_label"]
- binding_source_on_record, liveness_uncertain = _prod_capture_binding_dimension(payload)
- assert binding_source_on_record is True
- assert liveness_uncertain is True
+def _arm_steering_exhausted(ctx: object, signature: str, workflow_yaml: str) -> None:
+ """Steering is exhausted once a directive is armed and the draft's canonical structural
+ fingerprint is unchanged across it — the typed replacement for a reject-count ceiling."""
+ ctx.output_contract_actuation_count_by_signature[signature] = 1
+ ctx.output_contract_last_reject_fingerprint_by_signature[signature] = wu._output_contract_structural_fingerprint(
+ workflow_yaml, signature
+ )
+
+
+def test_resolver_credits_before_imposing_once_steering_is_exhausted() -> None:
+ credit_and_impose = _steering_exhausted_evidence(credit_available=True, scouted_binding_impose_available=True)
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=credit_and_impose).kind
+ == OutputContractActuationKind.CREDIT
+ )
+
+ impose_only = _steering_exhausted_evidence(scouted_binding_impose_available=True)
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=impose_only).kind
+ == OutputContractActuationKind.IMPOSE_SCOUTED_BINDING
+ )
+
+
+def test_resolver_pass_path_preempts_the_unobservable_source_terminal() -> None:
+ would_terminal = _steering_exhausted_evidence(
+ click_only_spine=True,
+ declick_attempt_failed=True,
+ credit_available=True,
+ )
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=would_terminal).kind
+ == OutputContractActuationKind.CREDIT
+ )
+
+ without_pass_path = _steering_exhausted_evidence(click_only_spine=True, declick_attempt_failed=True)
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=without_pass_path).kind
+ == OutputContractActuationKind.BLOCKED_TERMINAL
+ )
+
+
+def test_resolver_ignores_credit_and_impose_until_steering_is_exhausted() -> None:
+ for kind_flags in ({"credit_available": True}, {"scouted_binding_impose_available": True}):
+ armed = _steering_exhausted_evidence(steering_exhausted=False, **kind_flags)
+ unarmed = _steering_exhausted_evidence(steering_exhausted=False)
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=armed).kind
+ == resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=unarmed).kind
+ )
+
+
+def test_resolver_keeps_existing_imposition_ahead_of_the_pass_path() -> None:
+ both = _steering_exhausted_evidence(imposed_available=True, credit_available=True)
+ assert (
+ resolve_output_contract_actuation(family=OutputContractBailFamily.STRUCTURAL, evidence=both).kind
+ == OutputContractActuationKind.IMPOSED
+ )
+
+
+def test_observing_one_path_does_not_authorize_imposing_a_multi_path_contract() -> None:
+ ctx = _record_criterion_ctx()
+ ctx.scouted_output_covered_paths = {"output.record_id"}
+ evaluation = SimpleNamespace(
+ observation_paths={"output.record_id", "output.record_total"},
+ required_paths={"output.record_id", "output.record_total"},
+ block_label="extract_record",
+ canonical_signature="sig_multi",
+ )
+
+ assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is False
+
+ ctx.scouted_output_covered_paths = {"output.record_id", "output.record_total"}
+ assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is True