Skip to content

Automate browser based workflows with AI

fix(copilot): converge repeated metadata rejects (SKY-12763) (#7649)

Andrew Neilsone4adab047a54
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 0f684af0d..f1a4f0b07 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import json
import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
@@ -38,6 +39,7 @@ RecoveryHint = Literal[
]
LOG = structlog.get_logger()
+METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE = "metadata_reject_same_key_terminal"
# Matched case-insensitively. Imperative variants are narrow ("do not run" etc.) so plain "do not worry" prose doesn't false-positive.
@@ -149,6 +151,52 @@ class CopilotToolBlockerSignal(BaseModel):
return dict(value)
+def build_metadata_reject_same_key_terminal_signal(
+ *,
+ structural_key: str,
+ reject_family: Literal[
+ "missing_code_artifact_metadata",
+ "metadata_normalization",
+ "recorded_outcome_output_candidate",
+ "recorded_outcome_output_coverage",
+ ],
+ missing_fields_by_label: Mapping[str, list[str]],
+) -> CopilotToolBlockerSignal:
+ exact_fields = {label: list(fields) for label, fields in missing_fields_by_label.items()}
+ rendered_fields = json.dumps(exact_fields, sort_keys=True, separators=(",", ":"))
+ user_facing_reason = (
+ "I couldn't satisfy the code_artifact_metadata gate after repeated identical submissions. "
+ f"missing_fields_by_label={rendered_fields}. I've kept the draft so those exact fields can be added."
+ )
+ try:
+ assert_clean_user_facing_text(user_facing_reason, blocked_tool="update_and_run_blocks")
+ except ValueError:
+ user_facing_reason = (
+ "I couldn't satisfy the required output metadata after repeated identical submissions. "
+ "I've kept the draft so the missing fields can be added."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="loop_detected",
+ agent_steering_text=(
+ "The same structural metadata rejection reached its terminal rung. Preserve the draft and report "
+ f"the code_artifact_metadata gate with missing_fields_by_label={rendered_fields}."
+ ),
+ user_facing_reason=user_facing_reason,
+ recovery_hint="report_blocker_to_user",
+ cleared_by_tools=frozenset(),
+ preserves_workflow_draft=True,
+ renders_final_reply=True,
+ internal_reason_code=METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
+ blocked_tool="update_and_run_blocks",
+ extra={
+ "reject_family": reject_family,
+ "structural_key": structural_key,
+ "gate_id": "code_artifact_metadata",
+ "missing_fields_by_label": exact_fields,
+ },
+ )
+
+
def build_output_source_unobservable_blocker_signal(
*,
reason_code: str,
@@ -420,6 +468,7 @@ GENUINELY_TERMINAL_BLOCKER_REASON_CODES: frozenset[str] = frozenset(
"advisory_dispatch_stalled",
DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE,
"repair_ceiling_reached",
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
}
)
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index 10350ae6d..bf53c19f5 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -67,6 +67,13 @@ BuildTestOutcomeReasonCode = Literal[
"output_source_unobservable",
"actuation_exhausted",
]
+MetadataRejectFamily = Literal[
+ "missing_code_artifact_metadata",
+ "metadata_normalization",
+ "recorded_outcome_output_candidate",
+ "recorded_outcome_output_coverage",
+]
+MetadataRejectLadderAction = Literal["rung_1", "rung_2", "terminal"]
_STRUCTURAL_KEY_VERSION = "recorded_build_test_outcome:v1"
_AUTHORED_STRUCTURE_VERSION = "recorded_build_test_outcome_authored_structure:v1"
@@ -144,6 +151,76 @@ class RecordedBuildTestOutcome(BaseModel):
return self.structural_key is not None
+class MetadataRejectLadderInput(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ reject_family: MetadataRejectFamily
+ structural_key: str = Field(min_length=1)
+ gate_id: Literal["code_artifact_metadata"] = "code_artifact_metadata"
+ missing_fields_by_label: dict[str, list[str]]
+
+
+class MetadataRejectLadderState(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ reject_family: MetadataRejectFamily
+ structural_key: str = Field(min_length=1)
+ gate_id: Literal["code_artifact_metadata"] = "code_artifact_metadata"
+ missing_fields_by_label: dict[str, list[str]]
+ streak_count: int = Field(ge=1)
+
+
+class MetadataRejectLadderDecision(BaseModel):
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+ action: MetadataRejectLadderAction
+ rung: Literal[1, 2] | None
+ gate_id: Literal["code_artifact_metadata"] = "code_artifact_metadata"
+ missing_fields_by_label: dict[str, list[str]]
+ state: MetadataRejectLadderState
+
+
+def adjudicate_metadata_reject_ladder(
+ prior_state: MetadataRejectLadderState | None,
+ reject: MetadataRejectLadderInput,
+) -> MetadataRejectLadderDecision:
+ same_reject = (
+ prior_state is not None
+ and prior_state.reject_family == reject.reject_family
+ and prior_state.structural_key == reject.structural_key
+ and prior_state.gate_id == reject.gate_id
+ and prior_state.missing_fields_by_label == reject.missing_fields_by_label
+ )
+ if same_reject:
+ assert prior_state is not None
+ streak_count = prior_state.streak_count + 1
+ else:
+ streak_count = 1
+ state = MetadataRejectLadderState(
+ reject_family=reject.reject_family,
+ structural_key=reject.structural_key,
+ gate_id=reject.gate_id,
+ missing_fields_by_label=reject.missing_fields_by_label,
+ streak_count=streak_count,
+ )
+ if streak_count >= 3:
+ action: MetadataRejectLadderAction = "terminal"
+ rung = None
+ elif streak_count == 2:
+ action = "rung_2"
+ rung = 2
+ else:
+ action = "rung_1"
+ rung = 1
+ return MetadataRejectLadderDecision(
+ action=action,
+ rung=rung,
+ gate_id=reject.gate_id,
+ missing_fields_by_label=reject.missing_fields_by_label,
+ state=state,
+ )
+
+
class RecordedOutcomeGroundingPayload(BaseModel):
model_config = ConfigDict(extra="forbid")
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 0327026aa..8b0d06baa 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -118,6 +118,7 @@ class TurnNarrativePayload(TypedDict):
if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ MetadataRejectLadderState,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
@@ -855,6 +856,7 @@ class CopilotContext(AgentContext):
pending_code_authoring_runtime_repair_context: CodeAuthoringRepairContext | None = None
last_code_authoring_repair_context: CodeAuthoringRepairContext | None = None
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
+ metadata_reject_ladder_state: MetadataRejectLadderState | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
recorded_persisted_block_run_workflow_run_id: str | None = None
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index ccee4b060..1419e4b53 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -48,6 +48,7 @@ if TYPE_CHECKING:
from skyvern.forge.sdk.copilot.authoring_parameter_binding import AuthoringParameterBindingSnapshot
from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ MetadataRejectLadderState,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
@@ -393,6 +394,7 @@ class AgentContext:
last_run_outcome: RecordedRunOutcome | None = None
last_run_outcome_block_labels: list[str] = field(default_factory=list)
latest_recorded_build_test_outcome: RecordedBuildTestOutcome | None = None
+ metadata_reject_ladder_state: MetadataRejectLadderState | None = None
recorded_build_test_outcome_history: list[dict[str, object]] = field(default_factory=list)
recorded_persisted_block_run_workflow_run_id: str | None = None
recorded_outcome_grounding_requirement: RecordedOutcomeGroundingRequirement | None = None
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 790188961..5a85a35fb 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -260,6 +260,7 @@ from .workflow_update import _update_workflow as _update_workflow
from .workflow_update import (
consume_output_contract_advisory_grant_for_run_result as consume_output_contract_advisory_grant_for_run_result,
)
+from .workflow_update import metadata_same_key_terminal_preflight as metadata_same_key_terminal_preflight
LOG = structlog.get_logger()
@@ -668,6 +669,21 @@ async def update_and_run_blocks_tool(
serialized_code_artifact_metadata = scaffolded_code_artifact_metadata
arguments["code_artifact_metadata"] = serialized_code_artifact_metadata
+ metadata_same_key_terminal = metadata_same_key_terminal_preflight(
+ copilot_ctx,
+ workflow_yaml,
+ serialized_code_artifact_metadata,
+ )
+ if metadata_same_key_terminal is not None:
+ record_tool_step_result_for_ctx(
+ copilot_ctx,
+ "update_and_run_blocks",
+ arguments,
+ metadata_same_key_terminal,
+ )
+ sanitized = sanitize_tool_result_for_llm("update_and_run_blocks", metadata_same_key_terminal)
+ return json.dumps(sanitized)
+
if recorded_outcome_grounding_requires_current_page(copilot_ctx):
loop_error = _tool_loop_error(copilot_ctx, "update_and_run_blocks", arguments)
if loop_error:
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 9cc06fb42..8be3dbaa3 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ MetadataRejectLadderState,
RecordedBuildTestOutcome,
RecordedOutcomeGroundingRequirement,
arm_recorded_outcome_grounding_requirement,
@@ -3449,6 +3450,24 @@ def _should_arm_recorded_outcome_grounding(copilot_ctx: Any) -> bool:
return bool(latest.workflow_run_id or getattr(copilot_ctx, "last_run_blocks_workflow_run_id", None))
+def _metadata_reject_ladder_defers_repair_ceiling(copilot_ctx: CopilotContext, *, count: int) -> bool:
+ state = getattr(copilot_ctx, "metadata_reject_ladder_state", None)
+ if not isinstance(state, MetadataRejectLadderState):
+ return False
+ latest = copilot_ctx.latest_recorded_build_test_outcome
+ if (
+ not isinstance(latest, RecordedBuildTestOutcome)
+ or latest.phase != "author_time_reject"
+ or latest.reason_code != "metadata_reject"
+ or latest.structural_key != state.structural_key
+ ):
+ return False
+ threshold = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL
+ # Reserve the first ceiling crossing for same-key confirmation. Beyond it, only the active
+ # rung-2 metadata retry defers; an unrelated latest repair restores the generic fallback.
+ return count == threshold or state.streak_count >= 2
+
+
def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRepairContract) -> None:
"""Count consecutive REPAIR verdicts that made no newly-verified forward progress.
@@ -3512,10 +3531,21 @@ def _update_repair_loop_state(copilot_ctx: CopilotContext, contract: DiagnosisRe
clear_recorded_outcome_grounding_requirement(copilot_ctx)
copilot_ctx.consecutive_non_converging_repair_count = count
copilot_ctx.last_repair_non_convergence_signature = signature
+ repair_ceiling_reached = count >= settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL
+ if repair_ceiling_reached and _metadata_reject_ladder_defers_repair_ceiling(copilot_ctx, count=count):
+ state = copilot_ctx.metadata_reject_ladder_state
+ assert isinstance(state, MetadataRejectLadderState)
+ repair_ceiling_reached = False
+ LOG.info(
+ "copilot_metadata_reject_ladder_deferred_repair_ceiling",
+ structural_key=state.structural_key,
+ streak_count=state.streak_count,
+ repair_count=count,
+ )
contract.repair_loop_state = RepairLoopState(
streak_token=signature,
consecutive_identical_repair_count=count,
- ceiling_reached=count >= settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL,
+ ceiling_reached=repair_ceiling_reached,
)
if _should_arm_recorded_outcome_grounding(copilot_ctx):
arm_recorded_outcome_grounding_requirement(copilot_ctx)
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 8d043d6f7..9efdad026 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -41,20 +41,26 @@ from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
)
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
blocker_signal_is_genuinely_terminal,
build_definition_contract_unsatisfied_blocker_signal,
+ build_metadata_reject_same_key_terminal_signal,
build_output_source_unobservable_blocker_signal,
clear_terminal_evidence_on_workflow_edit,
stash_blocker_signal,
)
from skyvern.forge.sdk.copilot.build_test_outcome import (
BuildTestOutcomeReasonCode,
+ MetadataRejectFamily,
+ MetadataRejectLadderDecision,
+ MetadataRejectLadderInput,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
RecordedOutcomeGroundingRequirement,
_stable_hash,
+ adjudicate_metadata_reject_ladder,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
@@ -197,6 +203,7 @@ from skyvern.forge.sdk.copilot.runtime import (
SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_GATE_ID,
AgentContext,
AuthorTimeGateAblationPayload,
+ CodeArtifactMetadataValue,
NeverCapturedObligation,
NeverCapturedReplayPayload,
RejectedCodeArtifactMetadataCapture,
@@ -1319,24 +1326,45 @@ def _record_author_time_reject_outcome(
block_labels: list[str] | None = None,
missing_requested_output_facts: list[dict[str, object]] | None = None,
) -> None:
- prior_outcome = ctx.latest_recorded_build_test_outcome
- observed_page_value_excerpt = (
- prior_outcome.observed_page_value_excerpt if isinstance(prior_outcome, RecordedBuildTestOutcome) else ""
- )
record_build_test_outcome(
ctx,
- recorded_outcome_from_author_time_reject(
+ _build_author_time_reject_outcome(
+ ctx,
reason_code=reason_code,
- block_labels=block_labels or [],
+ summary=summary,
structural_payload=structural_payload,
authored_structure_signature=authored_structure_signature,
- observed_evidence_summary=summary,
- observed_page_value_excerpt=observed_page_value_excerpt,
- missing_requested_output_facts=missing_requested_output_facts or [],
+ block_labels=block_labels,
+ missing_requested_output_facts=missing_requested_output_facts,
),
)
+def _build_author_time_reject_outcome(
+ ctx: AgentContext,
+ *,
+ reason_code: BuildTestOutcomeReasonCode,
+ summary: str,
+ structural_payload: Mapping[str, object] | None = None,
+ authored_structure_signature: str | None = None,
+ block_labels: list[str] | None = None,
+ missing_requested_output_facts: list[dict[str, object]] | None = None,
+) -> RecordedBuildTestOutcome:
+ prior_outcome = ctx.latest_recorded_build_test_outcome
+ observed_page_value_excerpt = (
+ prior_outcome.observed_page_value_excerpt if isinstance(prior_outcome, RecordedBuildTestOutcome) else ""
+ )
+ return recorded_outcome_from_author_time_reject(
+ reason_code=reason_code,
+ block_labels=block_labels or [],
+ structural_payload=structural_payload,
+ authored_structure_signature=authored_structure_signature,
+ observed_evidence_summary=summary,
+ observed_page_value_excerpt=observed_page_value_excerpt,
+ missing_requested_output_facts=missing_requested_output_facts or [],
+ )
+
+
def _code_safety_reject_payload(errors: list[str | CodeBlockSecurityError]) -> Mapping[str, object] | None:
entries: list[dict[str, object]] = []
for error in errors:
@@ -4542,9 +4570,7 @@ def _adjudicate_metadata_reject_ladder(
missing_labels: list[str],
required_paths: set[str],
) -> dict[str, Any] | None:
- """Route the metadata-less reject through the shared actuation ladder in steer-only mode so a
- repeated identical structural key escalates the convergence directive instead of re-issuing the
- same steer, never minting an advisory run or a terminal at this seam."""
+ """Adjudicate an enforcing metadata reject without coupling it to unrelated outcome state."""
if copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID):
return None
signature = _output_contract_signature(ctx=ctx, required_paths=required_paths)
@@ -4554,39 +4580,156 @@ def _adjudicate_metadata_reject_ladder(
missing_fields_by_label = _metadata_missing_required_fields_by_label(
raw_metadata, labels=missing_labels, missing_labels=missing_labels
)
- seam_fingerprint = _metadata_reject_seam_fingerprint(workflow_yaml, signature, missing_fields_by_label)
- escalate = _prior_output_contract_directive_unconsumed(ctx, signature, seam_fingerprint)
- evaluation = _OutputContractEvaluation(
- block_label=block_label,
- artifact_id="",
- required_paths=required_paths,
- observation_paths=required_paths,
- declaration_paths=set(),
- source="missing_code_artifact_metadata",
- reason_code="metadata_reject",
- missing_metadata_paths=[],
- missing_schema_paths=[],
- missing_return_paths=[],
- shape_violations=[],
- canonical_signature=signature,
- payload={"reason_code": "metadata_reject"},
- repair_context=None,
+ reject_payload = _code_artifact_metadata_reject_payload(
+ workflow_yaml=workflow_yaml,
+ raw_metadata=raw_metadata,
+ offending_labels=[],
+ missing_labels=missing_labels,
+ violation_categories=["missing_code_artifact_metadata"],
+ )
+ latest_outcome = ctx.latest_recorded_build_test_outcome
+ candidate_outcome = (
+ latest_outcome
+ if isinstance(latest_outcome, RecordedBuildTestOutcome)
+ and latest_outcome.phase == "author_time_reject"
+ and latest_outcome.reason_code == "metadata_reject"
+ else _build_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary="Submitted workflow is missing required code artifact metadata.",
+ structural_payload=reject_payload,
+ block_labels=missing_labels,
+ missing_requested_output_facts=_missing_requested_output_facts(
+ required_paths,
+ reason_code=_required_child_output_paths_for_authoring(ctx)[2],
+ ),
+ )
)
- actuation = _adjudicate_output_contract_ladder_after_reject(
+ seam_fingerprint = _metadata_reject_seam_fingerprint(workflow_yaml, signature, missing_fields_by_label)
+ if ctx.metadata_reject_ladder_state is None and _prior_output_contract_directive_unconsumed(
ctx,
- evaluation,
- workflow_yaml=workflow_yaml,
- current_fingerprint=seam_fingerprint,
- steer_only=True,
+ signature,
+ seam_fingerprint,
+ ):
+ structural_key = candidate_outcome.structural_key
+ if structural_key is not None:
+ ctx.metadata_reject_ladder_state = adjudicate_metadata_reject_ladder(
+ None,
+ MetadataRejectLadderInput(
+ reject_family="missing_code_artifact_metadata",
+ structural_key=structural_key,
+ missing_fields_by_label=missing_fields_by_label,
+ ),
+ ).state
+ decision = _metadata_reject_ladder_decision(
+ ctx,
+ candidate_outcome=candidate_outcome,
+ reject_family="missing_code_artifact_metadata",
+ missing_fields_by_label=missing_fields_by_label,
)
- if actuation is None or actuation.kind != OutputContractActuationKind.STRUCTURE_DIRECTIVE:
+ if decision is None:
return None
+ if decision.action == "terminal":
+ return _metadata_reject_terminal_data(decision)
+ assert decision.rung is not None
+ _record_armed_directive_fingerprint(ctx, signature, seam_fingerprint)
return _emit_metadata_convergence_directive(
signature=signature,
block_label=block_label,
missing_fields_by_label=missing_fields_by_label,
required_paths=required_paths,
- escalate=escalate,
+ escalate=decision.rung == 2,
+ )
+
+
+def _metadata_reject_ladder_decision(
+ ctx: AgentContext,
+ *,
+ candidate_outcome: RecordedBuildTestOutcome,
+ reject_family: MetadataRejectFamily,
+ missing_fields_by_label: dict[str, list[str]],
+) -> MetadataRejectLadderDecision | None:
+ structural_key = candidate_outcome.structural_key
+ if structural_key is None or not missing_fields_by_label:
+ return None
+ prior_state = ctx.metadata_reject_ladder_state
+ latest_outcome = ctx.latest_recorded_build_test_outcome
+ if (
+ prior_state is not None
+ and isinstance(latest_outcome, RecordedBuildTestOutcome)
+ and latest_outcome.phase == "author_time_reject"
+ and (
+ latest_outcome.reason_code != "metadata_reject"
+ or latest_outcome.structural_key != prior_state.structural_key
+ )
+ ):
+ _reset_metadata_reject_ladder_state(
+ ctx,
+ superseding_reject_family=latest_outcome.reason_code,
+ )
+ prior_state = None
+ decision = adjudicate_metadata_reject_ladder(
+ prior_state,
+ MetadataRejectLadderInput(
+ reject_family=reject_family,
+ structural_key=structural_key,
+ missing_fields_by_label=missing_fields_by_label,
+ ),
+ )
+ ctx.metadata_reject_ladder_state = decision.state
+ if decision.state.streak_count >= 2:
+ LOG.info(
+ "copilot_metadata_reject_same_key_detected",
+ reject_family=reject_family,
+ structural_key=structural_key,
+ gate_id=decision.gate_id,
+ streak_count=decision.state.streak_count,
+ missing_fields_by_label=decision.missing_fields_by_label,
+ )
+ if decision.action == "terminal":
+ signal = build_metadata_reject_same_key_terminal_signal(
+ structural_key=structural_key,
+ reject_family=reject_family,
+ missing_fields_by_label=decision.missing_fields_by_label,
+ )
+ claim_and_stash_blocker_signal(ctx, TurnClaimant.GENUINELY_TERMINAL, signal)
+ stash_turn_halt_from_blocker_signal(ctx, signal, source="workflow_update")
+ LOG.info(
+ "copilot_metadata_reject_same_key_terminal_selected",
+ reject_family=reject_family,
+ structural_key=structural_key,
+ gate_id=decision.gate_id,
+ streak_count=decision.state.streak_count,
+ missing_fields_by_label=decision.missing_fields_by_label,
+ )
+ return decision
+
+
+def _reset_metadata_reject_ladder_state(ctx: AgentContext, *, superseding_reject_family: str) -> None:
+ prior_state = ctx.metadata_reject_ladder_state
+ if prior_state is None:
+ return
+ ctx.metadata_reject_ladder_state = None
+ LOG.info(
+ "copilot_metadata_reject_streak_reset",
+ prior_reject_family=prior_state.reject_family,
+ prior_structural_key=prior_state.structural_key,
+ prior_streak_count=prior_state.streak_count,
+ superseding_reject_family=superseding_reject_family,
+ )
+
+
+def _metadata_reject_terminal_data(decision: MetadataRejectLadderDecision) -> dict[str, CodeArtifactMetadataValue]:
+ return cast(
+ dict[str, CodeArtifactMetadataValue],
+ {
+ "reason_code": METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
+ "reject_family": decision.state.reject_family,
+ "structural_key": decision.state.structural_key,
+ "gate_id": decision.gate_id,
+ "missing_fields_by_label": decision.missing_fields_by_label,
+ "preserves_workflow_draft": True,
+ },
)
@@ -4599,6 +4742,10 @@ def _synthesized_metadata_reject_directive(
required_paths: set[str],
) -> dict[str, Any] | None:
if len(label_candidates) != 1:
+ _reset_metadata_reject_ladder_state(
+ ctx,
+ superseding_reject_family="recorded_outcome_metadata_reject",
+ )
return None
label = label_candidates[0]
full_required_fields = {"declared_goal", *_CODE_ARTIFACT_REQUIRED_LIST_FIELDS, "evidence_refs_or_observation_refs"}
@@ -4606,6 +4753,10 @@ def _synthesized_metadata_reject_directive(
raw_metadata, labels=[label], missing_labels=[]
)
if set(missing_fields_by_label.get(label) or []) != full_required_fields:
+ _reset_metadata_reject_ladder_state(
+ ctx,
+ superseding_reject_family="recorded_outcome_metadata_reject",
+ )
return None
return _adjudicate_metadata_reject_ladder(
ctx,
@@ -6645,6 +6796,14 @@ def _metadata_contract_run_preflight_reject(
ctx.last_code_authoring_repair_context = evaluation.repair_context
if _metadata_preflight_reject_yields_to_ladder(ctx):
return None
+ if payload.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE:
+ assert ctx.blocker_signal is not None
+ return {
+ "ok": False,
+ "error": ctx.blocker_signal.user_facing_reason,
+ "user_facing_summary": ctx.blocker_signal.user_facing_reason,
+ "data": payload,
+ }
if "value_bearing_convergence_directive" in payload:
return _output_contract_reject_result(
evaluation,
@@ -6667,6 +6826,74 @@ def _metadata_contract_run_preflight_reject(
}
+def metadata_same_key_terminal_preflight(
+ ctx: AgentContext,
+ workflow_yaml: str,
+ raw_code_artifact_metadata: object,
+) -> dict[str, CodeArtifactMetadataValue] | None:
+ prior_state = ctx.metadata_reject_ladder_state
+ if (
+ prior_state is None
+ or prior_state.streak_count < 2
+ or copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID)
+ ):
+ return None
+ missing_error = _missing_code_artifact_metadata_error(workflow_yaml, ctx, raw_code_artifact_metadata)
+ if missing_error is None:
+ return None
+ missing_labels = _missing_code_artifact_metadata_labels(workflow_yaml, ctx, raw_code_artifact_metadata)
+ if len(missing_labels) != 1:
+ return None
+ required_paths, _, coverage_reason_code = _required_child_output_paths_for_authoring(ctx)
+ reject_payload = _code_artifact_metadata_reject_payload(
+ workflow_yaml=workflow_yaml,
+ raw_metadata=raw_code_artifact_metadata,
+ offending_labels=[],
+ missing_labels=missing_labels,
+ violation_categories=["missing_code_artifact_metadata"],
+ )
+ candidate_outcome = _build_author_time_reject_outcome(
+ ctx,
+ reason_code="metadata_reject",
+ summary=missing_error,
+ structural_payload=reject_payload,
+ block_labels=missing_labels,
+ missing_requested_output_facts=_missing_requested_output_facts(
+ required_paths,
+ reason_code=coverage_reason_code,
+ ),
+ )
+ structural_key = candidate_outcome.structural_key
+ missing_fields_by_label = _metadata_missing_required_fields_by_label(
+ raw_code_artifact_metadata,
+ labels=missing_labels,
+ missing_labels=missing_labels,
+ )
+ if (
+ structural_key is None
+ or prior_state.reject_family != "missing_code_artifact_metadata"
+ or prior_state.structural_key != structural_key
+ or prior_state.gate_id != "code_artifact_metadata"
+ or prior_state.missing_fields_by_label != missing_fields_by_label
+ ):
+ return None
+ decision = _metadata_reject_ladder_decision(
+ ctx,
+ candidate_outcome=candidate_outcome,
+ reject_family="missing_code_artifact_metadata",
+ missing_fields_by_label=missing_fields_by_label,
+ )
+ if decision is None or decision.action != "terminal":
+ return None
+ assert ctx.blocker_signal is not None
+ return {
+ "ok": False,
+ "error": ctx.blocker_signal.user_facing_reason,
+ "user_facing_summary": ctx.blocker_signal.user_facing_reason,
+ "data": _metadata_reject_terminal_data(decision),
+ }
+
+
def output_contract_value_bearing_run_reject(
ctx: AgentContext,
code_by_label: Mapping[str, str],
@@ -13156,30 +13383,57 @@ async def _update_workflow(
source=output_path_coverage_source,
reason_code=output_path_coverage_reason_code,
)
- missing_metadata_reject_count = _record_output_contract_family_reject(
- ctx,
- required_child_output_paths,
- reject_family=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
+ metadata_reject_payload = _code_artifact_metadata_reject_payload(
+ workflow_yaml=workflow_yaml,
+ raw_metadata=params.get("code_artifact_metadata"),
+ offending_labels=[],
+ missing_labels=missing_labels,
+ violation_categories=["missing_code_artifact_metadata"],
)
- _record_author_time_reject_outcome(
+ candidate_outcome = _build_author_time_reject_outcome(
ctx,
reason_code="metadata_reject",
summary=missing_metadata_error,
- structural_payload=_output_contract_author_time_structural_payload(
- ctx,
- required_child_output_paths,
- block_label=missing_labels[0] if len(missing_labels) == 1 else "",
- )
- or _code_artifact_metadata_reject_payload(
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- offending_labels=[],
- missing_labels=missing_labels,
- violation_categories=["missing_code_artifact_metadata"],
- ),
+ structural_payload=metadata_reject_payload,
block_labels=missing_labels,
missing_requested_output_facts=missing_metadata_output_facts,
)
+ missing_fields_by_label = _metadata_missing_required_fields_by_label(
+ params.get("code_artifact_metadata"),
+ labels=missing_labels,
+ missing_labels=missing_labels,
+ )
+ if len(missing_labels) != 1:
+ _reset_metadata_reject_ladder_state(
+ ctx,
+ superseding_reject_family="multi_label_missing_code_artifact_metadata",
+ )
+ metadata_ladder_decision = (
+ None
+ if len(missing_labels) != 1
+ or copilot_author_time_gate_log_only_enabled(ctx, _OUTPUT_CONTRACT_ABLATION_GATE_ID)
+ else _metadata_reject_ladder_decision(
+ ctx,
+ candidate_outcome=candidate_outcome,
+ reject_family="missing_code_artifact_metadata",
+ missing_fields_by_label=missing_fields_by_label,
+ )
+ )
+ if metadata_ladder_decision is not None and metadata_ladder_decision.action == "terminal":
+ assert ctx.blocker_signal is not None
+ return reject(
+ error=ctx.blocker_signal.user_facing_reason,
+ user_facing_summary=ctx.blocker_signal.user_facing_reason,
+ data=_metadata_reject_terminal_data(metadata_ladder_decision),
+ repair_context=metadata_repair_context,
+ record_repair_context_outcome=False,
+ )
+ missing_metadata_reject_count = _record_output_contract_family_reject(
+ ctx,
+ required_child_output_paths,
+ reject_family=_MISSING_CODE_ARTIFACT_METADATA_REJECT_FAMILY,
+ )
+ record_build_test_outcome(ctx, candidate_outcome)
_capture_rejected_code_artifact_metadata(ctx)
credential_scout_errors = (
[]
@@ -13200,12 +13454,29 @@ async def _update_workflow(
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
elif missing_metadata_reject_count < 1 and not budget_terminal:
_record_code_authoring_guardrail_reject(ctx)
- metadata_convergence_directive = _adjudicate_metadata_reject_ladder(
- ctx,
- workflow_yaml=workflow_yaml,
- raw_metadata=params.get("code_artifact_metadata"),
- missing_labels=missing_labels,
- required_paths=required_child_output_paths,
+ metadata_signature = _output_contract_signature(ctx=ctx, required_paths=required_child_output_paths)
+ if metadata_ladder_decision is not None and metadata_ladder_decision.rung is not None:
+ _record_armed_directive_fingerprint(
+ ctx,
+ metadata_signature,
+ _metadata_reject_seam_fingerprint(
+ workflow_yaml,
+ metadata_signature,
+ missing_fields_by_label,
+ ),
+ )
+ metadata_convergence_directive = (
+ _emit_metadata_convergence_directive(
+ signature=metadata_signature,
+ block_label=missing_labels[0],
+ missing_fields_by_label=missing_fields_by_label,
+ required_paths=required_child_output_paths,
+ escalate=metadata_ladder_decision.rung == 2,
+ )
+ if metadata_ladder_decision is not None
+ and metadata_ladder_decision.rung is not None
+ and len(missing_labels) == 1
+ else None
)
metadata_reject_data = _code_repair_progress_data(
metadata_repair_context,
@@ -13257,6 +13528,10 @@ async def _update_workflow(
):
code_artifact_metadata_error = None
if code_artifact_metadata_error is not None:
+ _reset_metadata_reject_ladder_state(
+ ctx,
+ superseding_reject_family=_METADATA_NORMALIZATION_REJECT_FAMILY,
+ )
record_code_artifact_violations(ctx, normalization.violations, normalization.offending_labels)
normalization_reject_count = _record_output_contract_family_reject(
ctx,
@@ -13570,6 +13845,17 @@ async def _update_workflow(
label_candidates=output_empty_labels,
required_paths=unresolved_recorded_output_paths,
)
+ if (
+ metadata_convergence_directive is not None
+ and metadata_convergence_directive.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ ):
+ assert ctx.blocker_signal is not None
+ return reject(
+ error=ctx.blocker_signal.user_facing_reason,
+ user_facing_summary=ctx.blocker_signal.user_facing_reason,
+ data=metadata_convergence_directive,
+ record_repair_context_outcome=False,
+ )
if metadata_convergence_directive is not None:
reject_data["metadata_convergence_directive"] = metadata_convergence_directive
return reject(
@@ -13636,6 +13922,17 @@ async def _update_workflow(
label_candidates=block_labels,
required_paths=set(missing_output_paths),
)
+ if (
+ metadata_convergence_directive is not None
+ and metadata_convergence_directive.get("reason_code") == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ ):
+ assert ctx.blocker_signal is not None
+ return reject(
+ error=ctx.blocker_signal.user_facing_reason,
+ user_facing_summary=ctx.blocker_signal.user_facing_reason,
+ data=metadata_convergence_directive,
+ record_repair_context_outcome=False,
+ )
if metadata_convergence_directive is not None:
reject_data["metadata_convergence_directive"] = metadata_convergence_directive
return reject(
@@ -13750,6 +14047,7 @@ async def _update_workflow(
ctx.persisted_draft_browser_calls = _workflow_yaml_browser_call_pairs(workflow_yaml)
ctx.code_authoring_guardrail_reject_count = 0
ctx.last_code_authoring_reject_was_credential_priority = False
+ ctx.metadata_reject_ladder_state = None
_clear_code_authoring_repair_context(ctx)
_clear_held_churn_signals(ctx)
accepted_metadata = getattr(ctx, "code_artifact_metadata", None)
diff --git a/skyvern/forge/sdk/copilot/turn_halt.py b/skyvern/forge/sdk/copilot/turn_halt.py
index dd333effe..c7c28faed 100644
--- a/skyvern/forge/sdk/copilot/turn_halt.py
+++ b/skyvern/forge/sdk/copilot/turn_halt.py
@@ -15,6 +15,7 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
GENUINELY_TERMINAL_BLOCKER_REASON_CODES as GENUINELY_TERMINAL_BLOCKER_REASON_CODES,
)
from skyvern.forge.sdk.copilot.blocker_signal import (
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
OUTPUT_CONTRACT_REJECT_BUDGET_EXHAUSTED_REASON_CODE,
CopilotToolBlockerSignal,
blocker_signal_is_genuinely_terminal,
@@ -62,6 +63,7 @@ class TurnHaltKind(StrEnum):
OUTPUT_SOURCE_UNOBSERVABLE = "output_source_unobservable"
DELIVERED_UNVERIFIED = "delivered_unverified"
DEFINITION_CONTRACT_UNSATISFIED = "definition_contract_unsatisfied"
+ METADATA_REJECT_SAME_KEY = "metadata_reject_same_key"
class TurnHaltVerdict(StrEnum):
@@ -101,6 +103,7 @@ _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES = frozenset(
}
)
_DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES = frozenset({DEFINITION_CONTRACT_UNSATISFIED_REASON_CODE})
+_METADATA_REJECT_SAME_KEY_REASON_CODES = frozenset({METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE})
# Halts the agent did not choose: a verified outcome may suppress these.
# ACTIVE_TERMINAL_CHALLENGE is voluntary and is deliberately excluded so a
@@ -113,6 +116,7 @@ _INVOLUNTARY_TURN_HALT_KINDS = frozenset(
TurnHaltKind.SCHEMA_INCOMPATIBILITY,
TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE,
TurnHaltKind.DEFINITION_CONTRACT_UNSATISFIED,
+ TurnHaltKind.METADATA_REJECT_SAME_KEY,
}
)
_INVOLUNTARY_BLOCKER_REASON_CODES = (
@@ -121,6 +125,7 @@ _INVOLUNTARY_BLOCKER_REASON_CODES = (
| _SCHEMA_INCOMPATIBILITY_REASON_CODES
| _OUTPUT_SOURCE_UNOBSERVABLE_REASON_CODES
| _DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES
+ | _METADATA_REJECT_SAME_KEY_REASON_CODES
| frozenset({REPAIR_CEILING_REASON_CODE})
)
_VERIFIED_SUPPRESSIBLE_ACTIVE_TERMINAL_REASON_CODES = frozenset({ACTIVE_RUN_TERMINAL_EVIDENCE_REASON_CODE})
@@ -157,6 +162,8 @@ def _kind_for_blocker_signal(signal: CopilotToolBlockerSignal) -> TurnHaltKind |
return TurnHaltKind.OUTPUT_SOURCE_UNOBSERVABLE
if reason in _DEFINITION_CONTRACT_UNSATISFIED_REASON_CODES:
return TurnHaltKind.DEFINITION_CONTRACT_UNSATISFIED
+ if reason in _METADATA_REJECT_SAME_KEY_REASON_CODES:
+ return TurnHaltKind.METADATA_REJECT_SAME_KEY
return None
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
index 4b65fb39b..20473e9af 100644
--- a/tests/unit/test_copilot_blocker_signal.py
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -6,12 +6,14 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
_LEAK_DENY_TOKENS,
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
SYNTHESIZED_BLOCK_PERSISTENCE_REASON_CODE,
BlockerKind,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
build_llm_tool_error_payload,
build_loop_blocker_signal,
+ build_metadata_reject_same_key_terminal_signal,
clear_blocker_signal_for_reason_codes,
maybe_clear_blocker_signal_on_tool_success,
refresh_held_loop_blocker_evidence,
@@ -85,6 +87,38 @@ def test_to_trace_data_namespaces_extra_so_it_cannot_shadow_explicit_fields() ->
assert trace["extra"] == {"blocker_kind": "evil", "custom_metric": 7}
+def test_metadata_same_key_terminal_names_gate_and_exact_missing_fields() -> None:
+ missing_fields = {"extract_record": ["declared_goal", "claimed_outcomes", "evidence_refs_or_observation_refs"]}
+
+ signal = build_metadata_reject_same_key_terminal_signal(
+ structural_key="same-key",
+ reject_family="missing_code_artifact_metadata",
+ missing_fields_by_label=missing_fields,
+ )
+
+ assert signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert signal.preserves_workflow_draft is True
+ assert signal.renders_final_reply is True
+ assert "code_artifact_metadata" in signal.user_facing_reason
+ assert '"extract_record"' in signal.user_facing_reason
+ assert signal.extra["gate_id"] == "code_artifact_metadata"
+ assert signal.extra["missing_fields_by_label"] == missing_fields
+
+
+def test_metadata_same_key_terminal_degrades_leaky_label_in_user_facing_reason() -> None:
+ missing_fields = {"safe_reason_code": ["declared_goal"]}
+
+ signal = build_metadata_reject_same_key_terminal_signal(
+ structural_key="same-key",
+ reject_family="missing_code_artifact_metadata",
+ missing_fields_by_label=missing_fields,
+ )
+
+ assert "safe_reason_code" not in signal.user_facing_reason
+ assert signal.extra["missing_fields_by_label"] == missing_fields
+ assert_clean_user_facing_text(signal.user_facing_reason, blocked_tool=signal.blocked_tool)
+
+
@pytest.mark.parametrize("token", _LEAK_DENY_TOKENS)
def test_assert_clean_raises_on_each_deny_token(token: str) -> None:
with pytest.raises(ValueError):
diff --git a/tests/unit/test_copilot_build_test_outcome.py b/tests/unit/test_copilot_build_test_outcome.py
index ed9562c05..2d07d71b8 100644
--- a/tests/unit/test_copilot_build_test_outcome.py
+++ b/tests/unit/test_copilot_build_test_outcome.py
@@ -3,9 +3,11 @@ from __future__ import annotations
from types import SimpleNamespace
from skyvern.forge.sdk.copilot.build_test_outcome import (
+ MetadataRejectLadderInput,
RecordedBuildTestOutcome,
RecordedOutcomeBindingConstraint,
_binding_frontier_facet,
+ adjudicate_metadata_reject_ladder,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
@@ -25,6 +27,49 @@ from skyvern.forge.sdk.copilot.result_evidence import LoadedResultCompositionEvi
from skyvern.forge.sdk.copilot.run_outcome import RecordedRunOutcome
+def _metadata_reject_ladder_input(*, structural_key: str = "reject-key") -> MetadataRejectLadderInput:
+ return MetadataRejectLadderInput(
+ reject_family="missing_code_artifact_metadata",
+ structural_key=structural_key,
+ gate_id="code_artifact_metadata",
+ missing_fields_by_label={
+ "extract_record": ["declared_goal", "claimed_outcomes", "evidence_refs_or_observation_refs"]
+ },
+ )
+
+
+def test_metadata_reject_ladder_reaches_exact_field_terminal_on_third_same_key() -> None:
+ reject = _metadata_reject_ladder_input()
+
+ first = adjudicate_metadata_reject_ladder(None, reject)
+ second = adjudicate_metadata_reject_ladder(first.state, reject)
+ third = adjudicate_metadata_reject_ladder(second.state, reject)
+
+ assert first.action == "rung_1"
+ assert first.rung == 1
+ assert second.action == "rung_2"
+ assert second.rung == 2
+ assert second.missing_fields_by_label == reject.missing_fields_by_label
+ assert third.action == "terminal"
+ assert third.rung is None
+ assert third.gate_id == "code_artifact_metadata"
+ assert third.missing_fields_by_label == reject.missing_fields_by_label
+ assert third.state.streak_count == 3
+
+
+def test_metadata_reject_ladder_changed_structural_key_starts_new_streak() -> None:
+ first = adjudicate_metadata_reject_ladder(None, _metadata_reject_ladder_input())
+ second = adjudicate_metadata_reject_ladder(
+ first.state,
+ _metadata_reject_ladder_input(structural_key="changed-key"),
+ )
+
+ assert second.action == "rung_1"
+ assert second.rung == 1
+ assert second.state.structural_key == "changed-key"
+ assert second.state.streak_count == 1
+
+
def test_structural_key_changes_when_page_or_result_structure_changes() -> None:
first = RecordedBuildTestOutcome(
phase="persisted_block_run",
@@ -1353,3 +1398,4 @@ def test_persisted_run_prose_only_failure_is_not_authoritative() -> None:
assert outcome is None or outcome.is_authoritative is False
assert outcome is None or outcome.structural_key is None
+ (adjudicate_metadata_reject_ladder,)
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 14c19125b..c477f560e 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -21,6 +21,7 @@ import pytest
import yaml
from structlog.testing import capture_logs
+from skyvern.config import settings
from skyvern.forge.sdk.copilot import agent as agent_module
from skyvern.forge.sdk.copilot import code_block_synthesis as code_block_synthesis_module
from skyvern.forge.sdk.copilot import enforcement as enforcement_module
@@ -32,6 +33,7 @@ from skyvern.forge.sdk.copilot.authoring_parameter_binding import (
)
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
build_definition_contract_unsatisfied_blocker_signal,
@@ -13589,6 +13591,206 @@ class TestMetadataLessRejectArming:
rung_two_emits = [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT]
assert len(rung_two_emits) == 1 and rung_two_emits[0]["rung"] == 2
+ @pytest.mark.asyncio
+ async def test_third_same_key_is_typed_terminal_without_recording_third_reject(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ yaml_text = _metadata_less_output_yaml()
+ first = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="scout_evaluate",
+ attempted_tool="evaluate",
+ verdict="progress_observed",
+ reason_code="loaded_result_targets_observed",
+ structural_failure_identity="scout:interleaved",
+ ),
+ )
+ second = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
+ metadata_rejects_before_terminal = sum(
+ row.get("reason_code") == "metadata_reject" for row in ctx.recorded_build_test_outcome_history
+ )
+
+ with capture_logs() as logs:
+ third = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
+
+ assert first["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert second["data"]["metadata_convergence_directive"]["rung"] == 2
+ assert third["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert third["data"]["gate_id"] == "code_artifact_metadata"
+ assert (
+ third["data"]["missing_fields_by_label"]
+ == second["data"]["metadata_convergence_directive"]["missing_fields_by_label"]
+ )
+ assert ctx.blocker_signal is not None
+ assert ctx.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
+ assert (
+ sum(row.get("reason_code") == "metadata_reject" for row in ctx.recorded_build_test_outcome_history)
+ == metadata_rejects_before_terminal
+ )
+ terminal_logs = [log for log in logs if log["event"] == "copilot_metadata_reject_same_key_terminal_selected"]
+ assert len(terminal_logs) == 1
+ assert terminal_logs[0]["missing_fields_by_label"] == third["data"]["missing_fields_by_label"]
+
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_native_preflight_terminals_on_third_same_key(self) -> None:
+ ctx = _code_only_ctx()
+ tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
+ arguments = json.dumps(
+ {
+ "workflow_yaml": _metadata_less_output_yaml(),
+ "block_labels": ["extract_provider"],
+ "code_artifact_metadata": [],
+ }
+ )
+
+ first = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+ second = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+ third = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+
+ assert first["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert second["data"]["metadata_convergence_directive"]["rung"] == 2
+ assert "code_artifact_metadata" in third["error"]
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
+ assert ctx.turn_halt.blocker_signal is not None
+ assert ctx.turn_halt.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert (
+ ctx.turn_halt.blocker_signal.extra["missing_fields_by_label"]
+ == second["data"]["metadata_convergence_directive"]["missing_fields_by_label"]
+ )
+
+ @pytest.mark.asyncio
+ async def test_update_and_run_blocks_metadata_ladder_defers_run_backed_repair_ceiling(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
+ tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
+ arguments = json.dumps(
+ {
+ "workflow_yaml": _metadata_less_output_yaml(),
+ "block_labels": ["extract_provider"],
+ "code_artifact_metadata": [],
+ }
+ )
+
+ first = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+ second = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+ third = json.loads(await tools_module.update_and_run_blocks_tool.on_invoke_tool(tool_ctx, arguments))
+
+ assert first["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert second["data"]["metadata_convergence_directive"]["rung"] == 2
+ assert third["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
+
+ @pytest.mark.asyncio
+ async def test_metadata_ladder_does_not_defer_ceiling_for_unrelated_latest_repair(self) -> None:
+ ctx = _code_only_ctx()
+ yaml_text = _metadata_less_output_yaml()
+ for rung in (1, 2):
+ rejected = await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
+ assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=8) is True
+
+ record_build_test_outcome(
+ ctx,
+ RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="run_blocks",
+ verdict="repairable_failure",
+ reason_code="runtime_block_failure",
+ structural_failure_identity="runtime:unrelated",
+ ),
+ )
+
+ assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=9) is False
+
+ def test_metadata_ladder_does_not_defer_ceiling_for_legacy_context(self) -> None:
+ legacy_ctx: Any = SimpleNamespace()
+
+ assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(legacy_ctx, count=8) is False
+
+ @pytest.mark.asyncio
+ async def test_changed_metadata_key_reenables_run_backed_repair_ceiling(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
+ ctx.consecutive_non_converging_repair_count = settings.COPILOT_REPAIR_CEILING_CONSECUTIVE_IDENTICAL - 1
+ tool_ctx = SimpleNamespace(context=ctx, tool_name="update_and_run_blocks")
+
+ first = json.loads(
+ await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ tool_ctx,
+ json.dumps(
+ {
+ "workflow_yaml": _metadata_less_output_yaml(),
+ "block_labels": ["extract_provider"],
+ "code_artifact_metadata": [],
+ }
+ ),
+ )
+ )
+ changed = json.loads(
+ await tools_module.update_and_run_blocks_tool.on_invoke_tool(
+ tool_ctx,
+ json.dumps(
+ {
+ "workflow_yaml": _metadata_less_page_read_yaml(),
+ "block_labels": ["extract_provider"],
+ "code_artifact_metadata": [],
+ }
+ ),
+ )
+ )
+
+ assert first["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert changed["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.REPAIR_CEILING_REACHED
+
+ @pytest.mark.asyncio
+ async def test_accepted_persist_resets_metadata_ladder_and_generic_churn(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ rejected = await _update_workflow(
+ {"workflow_yaml": _metadata_less_output_yaml()},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert rejected["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert ctx.metadata_reject_ladder_state is not None
+ assert ctx.code_authoring_guardrail_reject_count == 1
+
+ accepted = await _update_workflow(
+ {
+ "workflow_yaml": _SAFE_EXTRACTION_CODE_YAML,
+ "code_artifact_metadata": [_terminal_metadata("search_registry", "search the registry")],
+ },
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert accepted["ok"] is True
+ assert ctx.metadata_reject_ladder_state is None
+ assert ctx.code_authoring_guardrail_reject_count == 0
+ restarted = await _update_workflow(
+ {"workflow_yaml": _metadata_less_output_yaml()},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
+
@pytest.mark.asyncio
async def test_structural_progress_resets_to_rung_one(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
@@ -13604,19 +13806,120 @@ class TestMetadataLessRejectArming:
assert progressed["data"]["metadata_convergence_directive"]["rung"] == 1
@pytest.mark.asyncio
- async def test_log_only_mode_leaves_seam_inert(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_intervening_normalization_reject_breaks_missing_metadata_streak(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
_stub_successful_update(monkeypatch)
- monkeypatch.setattr(
- workflow_update_module, "copilot_author_time_gate_log_only_enabled", lambda *_args, **_kwargs: True
+ ctx = _code_only_ctx()
+ yaml_text = _metadata_less_output_yaml()
+ for rung in (1, 2):
+ rejected = await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
+
+ malformed_metadata = {
+ "block_label": "extract_provider",
+ "artifact_id": "code_artifact:extract_provider",
+ "declared_goal": "extract the provider",
+ "claimed_outcomes": [
+ {
+ "id": "claim:provider",
+ "scope": "outcome",
+ "text": "provider extracted",
+ "status": "satisfied",
+ "depends_on": ["dependency:page"],
+ "covered_criteria": ["criterion:provider"],
+ "goal_value_paths": ["provider"],
+ }
+ ],
+ "page_dependencies": [{"id": "dependency:page", "scope": "page", "status": "satisfied"}],
+ "completion_criteria": [{"id": "criterion:provider", "text": "provider extracted", "level": "terminal"}],
+ "terminal_verifier_expectations": [
+ {
+ "id": "expectation:provider",
+ "text": "provider extracted",
+ "criteria_ids": ["criterion:provider"],
+ "goal_value_paths": ["provider"],
+ }
+ ],
+ "observation_refs": [
+ {
+ "observation_ref": "obs:page",
+ "dependency_id": "dependency:page",
+ "status": "satisfied",
+ "source_tool": "scout_interaction",
+ }
+ ],
+ }
+ intervening = await _update_workflow(
+ {"workflow_yaml": yaml_text, "code_artifact_metadata": [malformed_metadata]},
+ ctx,
+ allow_missing_credentials=True,
)
+ assert intervening["ok"] is False
+ assert "metadata_convergence_directive" not in intervening["data"]
+
+ restarted = await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert ctx.turn_halt is None
+
+ @pytest.mark.asyncio
+ async def test_intervening_non_metadata_author_reject_breaks_streak(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
+ yaml_text = _metadata_less_output_yaml()
+ for rung in (1, 2):
+ rejected = await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ assert rejected["data"]["metadata_convergence_directive"]["rung"] == rung
+
+ record_build_test_outcome(
+ ctx,
+ recorded_outcome_from_author_time_reject(
+ reason_code="code_safety_reject",
+ structural_payload={"reason_code": "code_safety_reject", "surface": "page.request"},
+ ),
+ )
+ restarted = await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+
+ assert restarted["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert ctx.turn_halt is None
+
+ @pytest.mark.asyncio
+ async def test_log_only_mode_persists_draft_and_leaves_ladder_inert(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.author_time_gate_log_only_ids = {workflow_update_module._OUTPUT_CONTRACT_ABLATION_GATE_ID}
with capture_logs() as logs:
result = await _update_workflow(
- {"workflow_yaml": _metadata_less_output_yaml()}, ctx, allow_missing_credentials=True
+ {"workflow_yaml": _metadata_less_output_yaml()},
+ ctx,
+ allow_missing_credentials=True,
+ allow_static_output_uncertainty=True,
)
- assert result["ok"] is False
- assert "metadata_convergence_directive" not in result["data"]
+ assert result["ok"] is True
+ assert ctx.latest_recorded_build_test_outcome is None
+ assert ctx.metadata_reject_ladder_state is None
assert not ctx.output_contract_armed_directive_fingerprint_by_signature
+ event = ctx.author_time_gate_ablation_events[-1]
+ assert event.gate_id == workflow_update_module._OUTPUT_CONTRACT_ABLATION_GATE_ID
+ assert event.reason_code == "missing_code_artifact_metadata"
+ assert event.log_only is True
assert [log for log in logs if log["event"] == _METADATA_LESS_DIRECTIVE_EVENT] == []
@pytest.mark.asyncio
@@ -13629,24 +13932,24 @@ class TestMetadataLessRejectArming:
assert result["ok"] is False
assert ctx.latest_recorded_build_test_outcome.reason_code == "metadata_reject"
assert "metadata_convergence_directive" not in result["data"]
+ assert ctx.metadata_reject_ladder_state is None
@pytest.mark.asyncio
- async def test_churn_stash_fires_at_cap_with_arming_active(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ async def test_same_key_terminal_preempts_generic_churn_cap(self, monkeypatch: pytest.MonkeyPatch) -> None:
_stub_successful_update(monkeypatch)
ctx = _code_only_ctx()
yaml_text = _metadata_less_output_yaml()
- for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
+ for rung in (1, 2):
result = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
assert result["ok"] is False
- assert result["data"]["metadata_convergence_directive"]["rung"] in (1, 2)
- assert ctx.code_authoring_guardrail_reject_count == index + 1
+ assert result["data"]["metadata_convergence_directive"]["rung"] == rung
assert ctx.blocker_signal is None
- final = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
- assert final["ok"] is False
- assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
- churn = ctx.blocker_signal
- assert isinstance(churn, CopilotToolBlockerSignal)
- assert churn.internal_reason_code == "code_authoring_guardrail_churn"
+ terminal = await _update_workflow({"workflow_yaml": yaml_text}, ctx, allow_missing_credentials=True)
+ assert terminal["ok"] is False
+ assert terminal["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert ctx.code_authoring_guardrail_reject_count < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ assert isinstance(ctx.blocker_signal, CopilotToolBlockerSignal)
+ assert ctx.blocker_signal.internal_reason_code == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
@pytest.mark.asyncio
async def test_seam_replay_emits_once_then_escalates(self, monkeypatch: pytest.MonkeyPatch) -> None:
@@ -13688,6 +13991,55 @@ class TestMetadataLessRejectArming:
assert escalated["rung"] == 2
assert _RECORDED_METADATA_REJECT_BLOCK_LABEL in escalated["metadata_fill_in_skeleton"]
+ @pytest.mark.asyncio
+ async def test_recorded_reject_identity_defers_ceiling_and_terminals_on_third(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_missing_code_artifact_metadata_error",
+ lambda *_args, **_kwargs: None,
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "_evaluate_output_contract_for_code_block",
+ lambda *_args, **_kwargs: None,
+ )
+ ctx = _code_only_ctx()
+ ctx.recorded_persisted_block_run_workflow_run_id = "wr_prior"
+ yaml_text = _recorded_metadata_reject_yaml()
+ recorded_run_outcome = RecordedBuildTestOutcome(
+ phase="persisted_block_run",
+ attempted_tool="update_and_run_blocks",
+ verdict="repairable_failure",
+ reason_code="outcome_not_demonstrated",
+ structural_failure_identity="completion:missing-provider",
+ missing_requested_output_facts=[
+ {"output_path": "provider", "output_root": "provider", "value_status": "no_typed_value"}
+ ],
+ )
+ results = []
+ for _ in range(3):
+ record_build_test_outcome(ctx, recorded_run_outcome)
+ results.append(
+ await _update_workflow(
+ {"workflow_yaml": yaml_text},
+ ctx,
+ allow_missing_credentials=True,
+ )
+ )
+
+ assert results[0]["data"]["metadata_convergence_directive"]["rung"] == 1
+ assert results[1]["data"]["metadata_convergence_directive"]["rung"] == 2
+ assert ctx.metadata_reject_ladder_state is not None
+ assert ctx.latest_recorded_build_test_outcome is not None
+ assert ctx.latest_recorded_build_test_outcome.structural_key == ctx.metadata_reject_ladder_state.structural_key
+ assert run_execution_module._metadata_reject_ladder_defers_repair_ceiling(ctx, count=9) is True
+ assert results[2]["data"]["reason_code"] == METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE
+ assert ctx.turn_halt is not None
+ assert ctx.turn_halt.kind is TurnHaltKind.METADATA_REJECT_SAME_KEY
+
class TestWholeTrajectoryImposition:
def test_imposes_over_unscouted_browser_fill_in_selected_block(self) -> None:
diff --git a/tests/unit/test_copilot_output_contract_actuation.py b/tests/unit/test_copilot_output_contract_actuation.py
index ed419fd9b..503040349 100644
--- a/tests/unit/test_copilot_output_contract_actuation.py
+++ b/tests/unit/test_copilot_output_contract_actuation.py
@@ -1268,6 +1268,7 @@ def _ladder_ctx() -> SimpleNamespace:
ctx = _advisory_ctx()
ctx.turn_halt = None
ctx.output_contract_bail_actuated_this_call = False
+ ctx.metadata_reject_ladder_state = None
return ctx
diff --git a/tests/unit/test_copilot_turn_halt.py b/tests/unit/test_copilot_turn_halt.py
index 69b2e8827..3bbb26424 100644
--- a/tests/unit/test_copilot_turn_halt.py
+++ b/tests/unit/test_copilot_turn_halt.py
@@ -6,6 +6,7 @@ import pytest
from skyvern.forge.sdk.copilot.blocker_signal import (
_OUTPUT_CONTRACT_TERMINAL_REASON_CODES,
+ METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE,
CopilotToolBlockerSignal,
stash_blocker_signal,
)
@@ -81,6 +82,10 @@ def _signal(
TurnHaltKind.ACTIVE_TERMINAL_CHALLENGE,
),
(_signal(internal_reason_code="probable_site_block_stop"), TurnHaltKind.PROBABLE_SITE_BLOCK),
+ (
+ _signal(internal_reason_code=METADATA_REJECT_SAME_KEY_TERMINAL_REASON_CODE),
+ TurnHaltKind.METADATA_REJECT_SAME_KEY,
+ ),
],
)
def test_terminal_blockers_map_to_halts(signal: CopilotToolBlockerSignal, expected_kind: TurnHaltKind) -> None: