Skip to content

Automate browser based workflows with AI

Scan tests/unit for the url-substring anti-pattern before it reaches OSS CodeQL (#7743)

Andrew Neilson29542fd2ac99
diff --git a/.env.example b/.env.example
index 8de4fcc0b..795438b21 100644
--- a/.env.example
+++ b/.env.example
@@ -86,8 +86,11 @@ YUTORI_TOOL_SET=""
# LLM_KEY: The chosen language model to use. This should be one of the models
# provided by the enabled LLM providers (e.g., OPENAI_GPT5_5, OPENAI_GPT5_4,
-# ANTHROPIC_CLAUDE5_FABLE, ANTHROPIC_CLAUDE4.7_OPUS, ANTHROPIC_CLAUDE4.6_SONNET,
-# GEMINI_3_PRO, BEDROCK_ANTHROPIC_CLAUDE5_FABLE_INFERENCE_PROFILE,
+# ANTHROPIC_CLAUDE5_OPUS, ANTHROPIC_CLAUDE5_FABLE, ANTHROPIC_CLAUDE4.7_OPUS,
+# ANTHROPIC_CLAUDE4.6_SONNET, GEMINI_3_PRO,
+# BEDROCK_ANTHROPIC_CLAUDE5_OPUS_INFERENCE_PROFILE,
+# BEDROCK_ANTHROPIC_CLAUDE5_OPUS_WITH_FALLBACK,
+# BEDROCK_ANTHROPIC_CLAUDE5_FABLE_INFERENCE_PROFILE,
# BEDROCK_ANTHROPIC_CLAUDE5_FABLE_WITH_FALLBACK).
# See docs: https://www.skyvern.com/docs/self-hosted/llm-configuration
LLM_KEY=""
diff --git a/skyvern-frontend/src/components/ModelSelector.test.tsx b/skyvern-frontend/src/components/ModelSelector.test.tsx
index 9f531311b..5210ddb15 100644
--- a/skyvern-frontend/src/components/ModelSelector.test.tsx
+++ b/skyvern-frontend/src/components/ModelSelector.test.tsx
@@ -82,6 +82,7 @@ describe("ModelSelector in a read-only comparison scope", () => {
"claude-opus-4-6": "Anthropic Claude 4.6 Opus",
"claude-opus-4-7": "Anthropic Claude 4.7 Opus",
"claude-opus-4-8": "Anthropic Claude 4.8 Opus",
+ "claude-opus-5": "Anthropic Claude Opus 5",
},
},
});
@@ -92,7 +93,8 @@ describe("ModelSelector in a read-only comparison scope", () => {
expect(await screen.findByText("Anthropic Claude 4.6 Opus")).toBeTruthy();
expect(await screen.findByText("Anthropic Claude 4.7 Opus")).toBeTruthy();
expect(await screen.findByText("Anthropic Claude 4.8 Opus")).toBeTruthy();
- expect(screen.getAllByText("Enterprise")).toHaveLength(4);
+ expect(await screen.findByText("Anthropic Claude Opus 5")).toBeTruthy();
+ expect(screen.getAllByText("Enterprise")).toHaveLength(5);
});
test("marks selected deprecated Claude Opus 4.5 as enterprise-only", async () => {
diff --git a/skyvern-frontend/src/components/ModelSelector.tsx b/skyvern-frontend/src/components/ModelSelector.tsx
index 202869036..b9ca69182 100644
--- a/skyvern-frontend/src/components/ModelSelector.tsx
+++ b/skyvern-frontend/src/components/ModelSelector.tsx
@@ -46,6 +46,7 @@ const enterpriseModelNames = new Set<string>([
"claude-opus-4-6",
"claude-opus-4-7",
"claude-opus-4-8",
+ "claude-opus-5",
]);
function ModelSelector({
diff --git a/skyvern/cli/llm_setup.py b/skyvern/cli/llm_setup.py
index d87841d49..edcb9ace3 100644
--- a/skyvern/cli/llm_setup.py
+++ b/skyvern/cli/llm_setup.py
@@ -130,6 +130,7 @@ def setup_llm_providers(env_path: Path | str | None = None) -> None:
enabled_providers.append("anthropic")
model_options.extend(
[
+ "ANTHROPIC_CLAUDE5_OPUS",
"ANTHROPIC_CLAUDE5_FABLE",
"ANTHROPIC_CLAUDE4.7_OPUS",
"ANTHROPIC_CLAUDE4.6_OPUS",
diff --git a/skyvern/config.py b/skyvern/config.py
index a4d48697e..18361589d 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -947,6 +947,18 @@ class Settings(BaseSettings):
"label": "Anthropic Claude Fable 5",
}
+ # Anthropic Claude Opus 5: prefer Bedrock when enabled, fall back to direct API
+ if self.ENABLE_BEDROCK_ANTHROPIC:
+ mapping["claude-opus-5"] = {
+ "llm_key": "BEDROCK_ANTHROPIC_CLAUDE5_OPUS_INFERENCE_PROFILE",
+ "label": "Anthropic Claude Opus 5",
+ }
+ else:
+ mapping["claude-opus-5"] = {
+ "llm_key": "ANTHROPIC_CLAUDE5_OPUS",
+ "label": "Anthropic Claude Opus 5",
+ }
+
try:
from skyvern.forge.sdk.api.llm.custom_llm_registry import ( # noqa: PLC0415
get_custom_llm_model_mappings,
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index abf2a05e8..c4bf1ea20 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -860,8 +860,10 @@ class LLMAPIHandlerFactory:
"anthropic/claude-opus-4-7",
"anthropic/claude-opus-4-8",
"anthropic/claude-fable-5",
+ "anthropic/claude-opus-5",
"anthropic-claude-opus-4-8",
"anthropic-claude-fable-5",
+ "anthropic-claude-opus-5",
}
@staticmethod
diff --git a/skyvern/forge/sdk/api/llm/config_registry.py b/skyvern/forge/sdk/api/llm/config_registry.py
index 5fea70a5e..7b1a56ab7 100644
--- a/skyvern/forge/sdk/api/llm/config_registry.py
+++ b/skyvern/forge/sdk/api/llm/config_registry.py
@@ -533,6 +533,17 @@ if settings.ENABLE_ANTHROPIC:
temperature=1,
),
)
+ LLMConfigRegistry.register_config(
+ "ANTHROPIC_CLAUDE5_OPUS",
+ LLMConfig(
+ "anthropic/claude-opus-5",
+ ["ANTHROPIC_API_KEY"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=128000,
+ temperature=1,
+ ),
+ )
if settings.ENABLE_BEDROCK:
# Supported through AWS IAM authentication
LLMConfigRegistry.register_config(
@@ -652,6 +663,17 @@ if settings.ENABLE_BEDROCK:
temperature=1,
),
)
+ LLMConfigRegistry.register_config(
+ "BEDROCK_ANTHROPIC_CLAUDE5_OPUS_INFERENCE_PROFILE",
+ LLMConfig(
+ "bedrock/us.anthropic.claude-opus-5",
+ ["AWS_REGION"],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ max_completion_tokens=128000,
+ temperature=1,
+ ),
+ )
if settings.ENABLE_AZURE:
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 0e2e93672..98e248d5c 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -170,6 +170,7 @@ from skyvern.forge.sdk.copilot.turn_ownership import (
TurnClaimant,
claim_and_stash_blocker_signal,
claim_turn,
+ claim_would_succeed,
claimant_outranks,
current_turn_owner,
emit_blocker_signal_payload,
@@ -627,6 +628,29 @@ def _scouted_spine_turn_end_nudge(ctx: AgentContext) -> str | None:
return nudge
+def _code_authoring_reject_count_resets(repeated_outcome: bool | None, frontier_unchanged: bool) -> bool:
+ # A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each
+ # turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.
+ return repeated_outcome is False and not frontier_unchanged
+
+
+def code_authoring_churn_stop_would_claim(ctx: AgentContext, *, frontier_unchanged: bool = False) -> bool:
+ """Read-only predicate: would recording one more plain (non-credential) guardrail reject reach
+ the churn ceiling AND win the turn? Mirrors _record_code_authoring_guardrail_reject's threshold,
+ terminal-defer, and claim precedence without mutating ctx."""
+ repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
+ base = (
+ 0
+ if _code_authoring_reject_count_resets(repeated_outcome, frontier_unchanged)
+ else ctx.code_authoring_guardrail_reject_count
+ )
+ if base + 1 < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS:
+ return False
+ if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
+ return False
+ return claim_would_succeed(ctx, TurnClaimant.CODE_AUTHORING_CHURN)
+
+
def _record_code_authoring_guardrail_reject(
ctx: AgentContext,
*,
@@ -636,9 +660,7 @@ def _record_code_authoring_guardrail_reject(
) -> None:
# Callers record the current build-test outcome first so repeat detection compares that key to history.
repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
- # A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each
- # turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.
- if repeated_outcome is False and not frontier_unchanged:
+ if _code_authoring_reject_count_resets(repeated_outcome, frontier_unchanged):
ctx.code_authoring_guardrail_reject_count = 0
ctx.code_authoring_guardrail_reject_count += 1
ctx.last_code_authoring_reject_was_credential_priority = defer_churn_stop
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 4eaa1d399..72e7866f7 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -161,6 +161,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_record_code_authoring_guardrail_reject,
_scouted_spine_open_obligation,
arm_credential_scout_reopen,
+ code_authoring_churn_stop_would_claim,
download_satisfied_requested_output_paths,
requested_output_extraction_plan,
requested_output_extraction_plan_changed,
@@ -5290,10 +5291,14 @@ def _resolve_output_contract_pass_path(
workflow_yaml: str,
raw_metadata: object,
credit_only: bool,
+ steering_exhausted: bool = False,
) -> OutputContractActuation:
"""Convert the pass-path preconditions to typed evidence and let the single resolver decide.
Crediting and composing are probed here because the resolver is pure; probing mutates nothing
- and abstains exactly where the strict path already rejects."""
+ and abstains exactly where the strict path already rejects.
+
+ ``steering_exhausted`` lets a caller contribute its own lane's exhaustion evidence — the
+ code-authoring churn ceiling is the same fact reached by a different lane."""
credited = _credit_output_contract_submission(
ctx, evaluation, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata
)
@@ -5310,7 +5315,7 @@ def _resolve_output_contract_pass_path(
and _observed_required_output_values(ctx, evaluation.observation_paths),
prior_actuation=_prior_output_contract_actuation(ctx, evaluation.canonical_signature),
prior_directive_unconsumed=False,
- steering_exhausted=_output_contract_steering_exhausted(ctx, evaluation, workflow_yaml),
+ steering_exhausted=steering_exhausted or _output_contract_steering_exhausted(ctx, evaluation, workflow_yaml),
credit_available=credited is not None,
scouted_binding_impose_available=impose_available,
)
@@ -5326,6 +5331,7 @@ def _credit_or_impose_output_contract(
workflow_yaml: str,
raw_metadata: object,
credit_only: bool = False,
+ steering_exhausted: bool = False,
) -> _OutputContractCeilingResolution | None:
"""Resolve a submission the steer loop can no longer move: credit it when the strict prover
was merely uncertain, else impose the binding the recording pins. Returns None when the
@@ -5336,6 +5342,7 @@ def _credit_or_impose_output_contract(
workflow_yaml=workflow_yaml,
raw_metadata=raw_metadata,
credit_only=credit_only,
+ steering_exhausted=steering_exhausted,
)
if actuation.kind not in {
OutputContractActuationKind.CREDIT,
@@ -5387,6 +5394,53 @@ def _credit_or_impose_output_contract(
)
+def _consult_guardrail_ceiling_actuation(
+ ctx: AgentContext,
+ *,
+ workflow_yaml: str,
+ raw_metadata: object,
+) -> _OutputContractCeilingResolution | None:
+ """At the code-authoring guardrail give-up, re-evaluate the live local draft and, when the
+ requested output's binding is already on record, consult the existing ceiling actuation (credit
+ or impose) before any stopped envelope. Side-effect-free until the actuation itself resolves."""
+ evaluation = _evaluate_output_contract_for_code_block(
+ ctx,
+ workflow_yaml,
+ raw_metadata,
+ enforce_value_bearing_liveness=True,
+ )
+ if evaluation is None or not evaluation.canonical_signature or not evaluation.block_label:
+ return None
+ if not _output_contract_binding_source_on_record(ctx, evaluation, workflow_yaml):
+ return None
+ LOG.info(
+ "copilot_guardrail_ceiling_actuation_consulted",
+ reject_count=ctx.code_authoring_guardrail_reject_count,
+ block_label=evaluation.block_label,
+ canonical_output_contract_signature=evaluation.canonical_signature,
+ binding_source_on_record=True,
+ )
+ return _credit_or_impose_output_contract(
+ ctx,
+ evaluation,
+ workflow_yaml=workflow_yaml,
+ raw_metadata=raw_metadata,
+ steering_exhausted=True,
+ )
+
+
+def _guardrail_ceiling_actuation_resolution(
+ ctx: AgentContext,
+ *,
+ workflow_yaml: str,
+ raw_metadata: object,
+ frontier_unchanged: bool = False,
+) -> _OutputContractCeilingResolution | None:
+ if not code_authoring_churn_stop_would_claim(ctx, frontier_unchanged=frontier_unchanged):
+ return None
+ return _consult_guardrail_ceiling_actuation(ctx, workflow_yaml=workflow_yaml, raw_metadata=raw_metadata)
+
+
def _capture_rejected_code_artifact_metadata(ctx: AgentContext) -> None:
snapshot = ctx.submitted_code_artifact_metadata_snapshot
if snapshot is None:
@@ -6275,6 +6329,21 @@ def _arm_pending_run_evidence(ctx: AgentContext, signature: str, required_paths:
ctx.output_contract_run_bound_required_path_by_signature.pop(signature, None)
+def _apply_output_contract_ceiling_bookkeeping(ctx: AgentContext, resolution: _OutputContractCeilingResolution) -> str:
+ """Arm pending-run evidence for a credited/imposed ceiling draft so a run that binds nothing
+ reopens the ladder, and for an imposed draft mark the imposition so a later reject in the same
+ call refunds the streak. Returns the imposed signature (empty when credited)."""
+ _arm_pending_run_evidence(
+ ctx,
+ resolution.evaluation.canonical_signature,
+ set(resolution.evaluation.observation_paths),
+ )
+ if resolution.mode == "imposed":
+ _mark_output_contract_imposed(ctx, resolution.evaluation.canonical_signature)
+ return resolution.evaluation.canonical_signature
+ return ""
+
+
def _registered_output_paths(result: object) -> set[str]:
paths: set[str] = set()
data = result.get("data") if isinstance(result, dict) else None
@@ -13813,6 +13882,12 @@ async def _update_workflow(
workflow_yaml=workflow_yaml,
raw_metadata=params.get("code_artifact_metadata"),
)
+ if output_contract_ceiling_resolution is None:
+ output_contract_ceiling_resolution = _guardrail_ceiling_actuation_resolution(
+ ctx,
+ workflow_yaml=workflow_yaml,
+ raw_metadata=params.get("code_artifact_metadata"),
+ )
if output_contract_ceiling_resolution is not None:
assert output_contract_evaluation is not None
workflow_yaml = output_contract_ceiling_resolution.workflow_yaml
@@ -14321,38 +14396,57 @@ async def _update_workflow(
code_artifact_metadata=getattr(ctx, "code_artifact_metadata", None),
)
if convergence_reject is not None:
- block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
- _record_author_time_reject_outcome(
+ frontier_unchanged = convergence_reject.reason == "frontier_unchanged"
+ convergence_ceiling_resolution = _guardrail_ceiling_actuation_resolution(
ctx,
- reason_code="unchanged_after_recorded_outcome",
- summary="The authored code and output structure are unchanged after the last recorded test outcome.",
- structural_payload={
- "reason_code": "unchanged_after_recorded_outcome",
- "authored_structure_signature": convergence_reject.authored_structure_signature,
- "block_labels": block_labels,
- },
- authored_structure_signature=convergence_reject.authored_structure_signature,
- block_labels=block_labels,
- )
- _record_code_authoring_guardrail_reject(
- ctx, frontier_unchanged=convergence_reject.reason == "frontier_unchanged"
- )
- LOG.info(
- "copilot recorded outcome convergence behavior",
- convergence_reason=convergence_reject.reason,
- commit_early_terminal=convergence_reject.commit_early_terminal,
- block_labels=block_labels,
- )
- if convergence_reject.commit_early_terminal:
- _commit_recorded_outcome_early_terminal(ctx)
- return reject(
- error=(
- "Submitted workflow left the frontier the last recorded test outcome named unchanged. "
- "Revise the code block or output metadata that owns that frontier before testing again."
- ),
- user_facing_summary=_compiled_authoring_user_summary(),
- data=_code_repair_progress_data(),
- )
+ workflow_yaml=workflow_yaml,
+ raw_metadata=params.get("code_artifact_metadata"),
+ frontier_unchanged=frontier_unchanged,
+ )
+ if convergence_ceiling_resolution is not None:
+ imposed_code_artifact_metadata = convergence_ceiling_resolution.code_artifact_metadata
+ workflow_yaml = convergence_ceiling_resolution.workflow_yaml
+ params["workflow_yaml"] = workflow_yaml
+ params["code_artifact_metadata"] = imposed_code_artifact_metadata
+ params["raw_code_artifact_metadata"] = imposed_code_artifact_metadata
+ ctx.raw_code_artifact_metadata = imposed_code_artifact_metadata
+ if isinstance(imposed_code_artifact_metadata, dict):
+ ctx.code_artifact_metadata = imposed_code_artifact_metadata
+ output_contract_imposed_signature = (
+ _apply_output_contract_ceiling_bookkeeping(ctx, convergence_ceiling_resolution)
+ or output_contract_imposed_signature
+ )
+ else:
+ block_labels = sorted(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="unchanged_after_recorded_outcome",
+ summary="The authored code and output structure are unchanged after the last recorded test outcome.",
+ structural_payload={
+ "reason_code": "unchanged_after_recorded_outcome",
+ "authored_structure_signature": convergence_reject.authored_structure_signature,
+ "block_labels": block_labels,
+ },
+ authored_structure_signature=convergence_reject.authored_structure_signature,
+ block_labels=block_labels,
+ )
+ _record_code_authoring_guardrail_reject(ctx, frontier_unchanged=frontier_unchanged)
+ LOG.info(
+ "copilot recorded outcome convergence behavior",
+ convergence_reason=convergence_reject.reason,
+ commit_early_terminal=convergence_reject.commit_early_terminal,
+ block_labels=block_labels,
+ )
+ if convergence_reject.commit_early_terminal:
+ _commit_recorded_outcome_early_terminal(ctx)
+ return reject(
+ error=(
+ "Submitted workflow left the frontier the last recorded test outcome named unchanged. "
+ "Revise the code block or output metadata that owns that frontier before testing again."
+ ),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(),
+ )
recorded_output_paths_required = output_path_coverage_source == "recorded_outcome"
recorded_missing_output_paths = (
diff --git a/skyvern/forge/sdk/copilot/turn_ownership.py b/skyvern/forge/sdk/copilot/turn_ownership.py
index 1caa1eebd..2ee5f26c2 100644
--- a/skyvern/forge/sdk/copilot/turn_ownership.py
+++ b/skyvern/forge/sdk/copilot/turn_ownership.py
@@ -269,6 +269,13 @@ def current_turn_owner(ctx: AgentContext) -> PrecedenceClaim | None:
return min(live, key=lambda claim: _PRECEDENCE_RANK[claim.claimant])
+def claim_would_succeed(ctx: AgentContext, claimant: TurnClaimant) -> bool:
+ """Read-only mirror of claim_turn's yield check: True iff a claim by ``claimant`` would be
+ owned rather than yielded against the current live owner. Mutates no ctx state."""
+ owner = current_turn_owner(ctx)
+ return owner is None or owner.claimant is claimant or claimant_outranks(claimant, owner.claimant)
+
+
def claim_turn(
ctx: AgentContext,
claimant: TurnClaimant,
diff --git a/skyvern/forge/sdk/enterprise_features.py b/skyvern/forge/sdk/enterprise_features.py
index 7c6e2a2fe..da6b1db56 100644
--- a/skyvern/forge/sdk/enterprise_features.py
+++ b/skyvern/forge/sdk/enterprise_features.py
@@ -13,6 +13,7 @@ ENTERPRISE_GATED_MODEL_NAMES: dict[str, str] = {
"claude-opus-4-7": "Anthropic Claude 4.7 Opus",
"claude-opus-4-8": "Anthropic Claude 4.8 Opus",
"claude-fable-5": "Anthropic Claude Fable 5",
+ "claude-opus-5": "Anthropic Claude Opus 5",
}
diff --git a/tests/unit/test_api_handler_factory.py b/tests/unit/test_api_handler_factory.py
index 6b4e86e38..b5b6419ab 100644
--- a/tests/unit/test_api_handler_factory.py
+++ b/tests/unit/test_api_handler_factory.py
@@ -323,8 +323,10 @@ def test_normalize_llm_model_strips_provider_prefix() -> None:
"anthropic/claude-opus-4-7",
"anthropic/claude-opus-4-8",
"anthropic/claude-fable-5",
+ "anthropic/claude-opus-5",
"anthropic-claude-opus-4-8",
"anthropic-claude-fable-5",
+ "anthropic-claude-opus-5",
],
)
def test_requires_adaptive_thinking_for_direct_anthropic_models(model_name: str) -> None:
@@ -336,6 +338,7 @@ def test_requires_adaptive_thinking_for_direct_anthropic_models(model_name: str)
[
"bedrock/us.anthropic.claude-opus-4-8",
"bedrock/us.anthropic.claude-fable-5",
+ "bedrock/us.anthropic.claude-opus-5",
"anthropic/claude-sonnet-4-6",
None,
],
@@ -349,6 +352,7 @@ def test_requires_adaptive_thinking_does_not_rewrite_other_providers(model_name:
[
"anthropic/claude-opus-4-8",
"anthropic/claude-fable-5",
+ "anthropic/claude-opus-5",
],
)
def test_apply_anthropic_thinking_optimization_uses_adaptive_shape(model_name: str) -> None:
diff --git a/tests/unit/test_copilot_enforcement_hardening.py b/tests/unit/test_copilot_enforcement_hardening.py
index 0c47fb14b..300e00db1 100644
--- a/tests/unit/test_copilot_enforcement_hardening.py
+++ b/tests/unit/test_copilot_enforcement_hardening.py
@@ -56,6 +56,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
SCREENSHOT_PLACEHOLDER,
CopilotNonRetriableNavError,
_check_enforcement,
+ _code_authoring_reject_count_resets,
_is_context_window_error,
_maybe_raise_non_retriable_nav,
_needs_inspect_before_repair_nudge,
@@ -64,6 +65,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_recover_from_context_overflow,
_scouted_spine_missing_text,
_strip_input_images,
+ code_authoring_churn_stop_would_claim,
register_no_progress_interaction_click,
reset_no_progress_interaction_count,
synthesized_trajectory_reaches_goal,
@@ -970,3 +972,79 @@ def test_same_omission_spine_violation_still_refused_after_change() -> None:
assert result.repair_context is not None
assert result.repair_context.reason_code == "scouted_spine_under_build"
assert "#search-submit" in result.violations[0]
+
+
+def test_reject_count_resets_only_on_non_repeat_without_frontier_unchanged() -> None:
+ assert _code_authoring_reject_count_resets(False, False) is True
+ assert _code_authoring_reject_count_resets(False, True) is False
+ assert _code_authoring_reject_count_resets(None, False) is False
+ assert _code_authoring_reject_count_resets(True, False) is False
+
+
+def test_churn_stop_would_claim_true_at_ceiling_and_mutates_nothing() -> None:
+ ctx = _fresh_context()
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+
+ assert code_authoring_churn_stop_would_claim(ctx) is True
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ assert ctx.blocker_signal is None
+
+
+def test_churn_stop_would_claim_false_below_ceiling() -> None:
+ ctx = _fresh_context()
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2
+
+ assert code_authoring_churn_stop_would_claim(ctx) is False
+
+
+def test_churn_stop_would_claim_false_when_terminal_blocker_held() -> None:
+ ctx = _fresh_context()
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ ctx.blocker_signal = CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text="A site verification challenge blocked the run.",
+ user_facing_reason="The site's verification challenge blocked the run.",
+ recovery_hint="report_blocker_to_user",
+ internal_reason_code=TERMINAL_CHALLENGE_BLOCKER_REASON_CODE,
+ blocked_tool="update_and_run_blocks",
+ )
+
+ assert code_authoring_churn_stop_would_claim(ctx) is False
+
+
+def test_churn_stop_would_claim_yields_to_stronger_live_owner_and_mutates_nothing() -> None:
+ ctx = _fresh_context()
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ ctx.output_contract_actuation_by_signature = {"sig": OutputContractAdvisoryState.GRANTED}
+
+ assert code_authoring_churn_stop_would_claim(ctx) is False
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ assert ctx.blocker_signal is None
+
+
+def test_churn_stop_would_claim_matches_recorder_claim_at_ceiling() -> None:
+ predicate_ctx = _fresh_context()
+ predicate_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ recorder_ctx = _fresh_context()
+ recorder_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+
+ would_claim = code_authoring_churn_stop_would_claim(predicate_ctx)
+ _record_code_authoring_guardrail_reject(recorder_ctx)
+
+ assert would_claim is True
+ claimed = recorder_ctx.blocker_signal
+ assert isinstance(claimed, CopilotToolBlockerSignal)
+ assert claimed.internal_reason_code == "code_authoring_guardrail_churn"
+
+
+def test_churn_stop_would_claim_matches_recorder_no_claim_below_ceiling() -> None:
+ predicate_ctx = _fresh_context()
+ predicate_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2
+ recorder_ctx = _fresh_context()
+ recorder_ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2
+
+ would_claim = code_authoring_churn_stop_would_claim(predicate_ctx)
+ _record_code_authoring_guardrail_reject(recorder_ctx)
+
+ assert would_claim is False
+ assert recorder_ctx.blocker_signal is None
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index b7853c49e..3c65196c0 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -7,6 +7,7 @@ OSS-synced: only synthetic labels and RFC-2606 placeholder identifiers.
from __future__ import annotations
+import dataclasses
import itertools
import json
from types import SimpleNamespace
@@ -3090,3 +3091,130 @@ def test_observing_one_path_does_not_authorize_imposing_a_multi_path_contract()
ctx.scouted_output_covered_paths = {"output.record_id", "output.record_total"}
assert wu._output_contract_binding_source_on_record(ctx, evaluation, _declaration_waiver_yaml("pass")) is True
+
+
+# ---------------------------------------------------------------------------
+# The code-authoring guardrail give-up consults the ceiling actuation.
+# ---------------------------------------------------------------------------
+
+
+def _guardrail_impose_ctx() -> tuple[CopilotContext, str, list[dict[str, object]]]:
+ ctx = _record_criterion_ctx()
+ workflow_yaml = _declaration_waiver_yaml('record_id = await page.inner_text("#record")')
+ metadata = _record_contract_metadata()
+ ctx.scouted_output_covered_paths = {"output.record_id"}
+ ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ return ctx, workflow_yaml, metadata
+
+
+def test_guardrail_ceiling_consult_imposes_before_stop_under_log_only(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(settings, "WORKFLOW_COPILOT_AUTHOR_TIME_GATE_LOG_ONLY", True)
+ ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+
+ with capture_logs() as logs:
+ resolution = wu._guardrail_ceiling_actuation_resolution(
+ ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=False
+ )
+
+ assert resolution is not None
+ assert resolution.mode == "imposed"
+ events = [log.get("event") for log in logs]
+ assert "copilot_guardrail_ceiling_actuation_consulted" in events
+ assert events.index("copilot_guardrail_ceiling_actuation_consulted") < events.index(
+ "copilot_output_contract_binding_imposed_at_ceiling"
+ )
+ assert ctx.blocker_signal is None
+ assert ctx.code_authoring_guardrail_reject_count == enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ assert ctx.output_contract_reject_count_by_signature.get(resolution.evaluation.canonical_signature) in (None, 0)
+
+
+def test_ceiling_bookkeeping_imposed_arms_evidence_and_marks_imposed() -> None:
+ seed_ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+ resolution = wu._guardrail_ceiling_actuation_resolution(
+ seed_ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=True
+ )
+ assert resolution is not None and resolution.mode == "imposed"
+ signature = resolution.evaluation.canonical_signature
+ observation_paths = sorted(resolution.evaluation.observation_paths)
+ assert observation_paths
+
+ ctx = _record_criterion_ctx()
+ returned = wu._apply_output_contract_ceiling_bookkeeping(ctx, resolution)
+
+ assert returned == signature
+ assert ctx.output_contract_pending_run_evidence.get(signature) == observation_paths
+ assert ctx.output_contract_imposed_since_last_reject_by_signature.get(signature) is True
+
+
+def test_ceiling_bookkeeping_credited_arms_evidence_without_marking_imposed() -> None:
+ seed_ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+ imposed = wu._guardrail_ceiling_actuation_resolution(
+ seed_ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata, frontier_unchanged=True
+ )
+ assert imposed is not None
+ credited = dataclasses.replace(imposed, mode="credited")
+ signature = credited.evaluation.canonical_signature
+ observation_paths = sorted(credited.evaluation.observation_paths)
+ assert observation_paths
+
+ ctx = _record_criterion_ctx()
+ returned = wu._apply_output_contract_ceiling_bookkeeping(ctx, credited)
+
+ assert returned == ""
+ assert ctx.output_contract_pending_run_evidence.get(signature) == observation_paths
+ assert ctx.output_contract_imposed_since_last_reject_by_signature == {}
+
+
+def test_guardrail_ceiling_consult_abstains_without_recorded_observation() -> None:
+ ctx = _record_criterion_ctx()
+ workflow_yaml = _declaration_waiver_yaml('await page.click("#next")\nreturn "done"')
+ metadata = _record_contract_metadata()
+ ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+
+ with capture_logs() as logs:
+ resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
+
+ assert resolution is None
+ assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
+ assert ctx.blocker_signal is None
+ assert ctx.code_authoring_guardrail_reject_count == enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+ assert ctx.output_contract_imposed_since_last_reject_by_signature == {}
+
+
+def test_guardrail_ceiling_consult_skipped_below_churn_ceiling() -> None:
+ ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+ ctx.code_authoring_guardrail_reject_count = enforcement.MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 2
+
+ with capture_logs() as logs:
+ resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
+
+ assert resolution is None
+ assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
+
+
+def test_guardrail_ceiling_consult_yields_to_stronger_live_owner() -> None:
+ ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+ ctx.output_contract_actuation_by_signature = {"sig": OutputContractAdvisoryState.GRANTED}
+
+ with capture_logs() as logs:
+ resolution = wu._guardrail_ceiling_actuation_resolution(ctx, workflow_yaml=workflow_yaml, raw_metadata=metadata)
+
+ assert resolution is None
+ assert not any(log.get("event") == "copilot_guardrail_ceiling_actuation_consulted" for log in logs)
+ assert ctx.blocker_signal is None
+
+
+def test_guardrail_exhaustion_evidence_reaches_the_pass_path() -> None:
+ ctx, workflow_yaml, metadata = _guardrail_impose_ctx()
+ strict = wu._evaluate_output_contract_for_code_block(
+ ctx, workflow_yaml, metadata, enforce_value_bearing_liveness=True
+ )
+ assert strict is not None
+
+ assert wu._credit_or_impose_output_contract(ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata) is None
+
+ resolution = wu._credit_or_impose_output_contract(
+ ctx, strict, workflow_yaml=workflow_yaml, raw_metadata=metadata, steering_exhausted=True
+ )
+ assert resolution is not None
+ assert resolution.mode == "imposed"
diff --git a/tests/unit/workflow/test_enterprise_feature_gating.py b/tests/unit/workflow/test_enterprise_feature_gating.py
index c14cedf52..1cc07039a 100644
--- a/tests/unit/workflow/test_enterprise_feature_gating.py
+++ b/tests/unit/workflow/test_enterprise_feature_gating.py
@@ -180,6 +180,7 @@ def test_collects_gated_model_on_split_pdf_block() -> None:
("claude-opus-4-7", "Anthropic Claude 4.7 Opus"),
("claude-opus-4-8", "Anthropic Claude 4.8 Opus"),
("claude-fable-5", "Anthropic Claude Fable 5"),
+ ("claude-opus-5", "Anthropic Claude Opus 5"),
],
)
def test_collects_enterprise_model_alias_features(model_name: str, feature_name: str) -> None: