Skip to content

Automate browser based workflows with AI

fix(SKY-12182): keep every workflow step the copilot performed, and log why a step is rejected (#7284)

Andrew Neilson2e5aa4920281
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index f5036ebe4..0fa53fef9 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -92,7 +92,9 @@ from skyvern.forge.sdk.copilot.context import (
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
BUILT_UNVERIFIED_REPAIR_INERT_TERMINAL_REASON,
+ SCOUTED_SPINE_TURN_HALT_USER_REASON,
artifact_health_blocked,
+ log_scouted_spine_unresolved_at_turn_halt,
outcome_fully_verified,
recycle_admits_present_completion_contract_ask,
synthesized_persistence_reopened,
@@ -2051,11 +2053,15 @@ async def _build_built_unverified_exit_result(ctx: CopilotContext, global_llm_co
)
+_SCOUTED_SPINE_HALT_REPLY_KINDS = frozenset({TurnHaltKind.LOOP_DETECTED, TurnHaltKind.REPAIR_CEILING_REACHED})
+
+
def _build_turn_halt_exit_result(
ctx: CopilotContext,
global_llm_context: str | None,
halt: TurnHalt,
) -> AgentResult:
+ under_build_open = log_scouted_spine_unresolved_at_turn_halt(ctx)
if halt.kind == TurnHaltKind.DELIVERED_UNVERIFIED:
reply = _delivered_unverified_reply(ctx) or _BUILT_UNVERIFIED_COMPLETED_REPLY
return _build_wip_exit_result(
@@ -2070,11 +2076,16 @@ def _build_turn_halt_exit_result(
if isinstance(signal, CopilotToolBlockerSignal) and signal.blocker_kind == "loop_detected":
refresh_held_loop_blocker_evidence(ctx)
signal = ctx.blocker_signal if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal) else signal
- user_response = (
- signal.user_facing_reason
- if isinstance(signal, CopilotToolBlockerSignal)
- else "I could not continue this turn safely. Tell me what to change and I'll try again."
- )
+ if under_build_open and halt.kind in _SCOUTED_SPINE_HALT_REPLY_KINDS:
+ user_response = SCOUTED_SPINE_TURN_HALT_USER_REASON
+ # The blocker-override finalizer re-renders from the held signal, so the
+ # reframed reason must live there, not just in the local reply.
+ if isinstance(ctx.blocker_signal, CopilotToolBlockerSignal):
+ ctx.blocker_signal = ctx.blocker_signal.model_copy(update={"user_facing_reason": user_response})
+ elif isinstance(signal, CopilotToolBlockerSignal):
+ user_response = signal.user_facing_reason
+ else:
+ user_response = "I could not continue this turn safely. Tell me what to change and I'll try again."
return _build_exit_result(
ctx,
user_response,
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 301d2bb94..b239198d0 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -8,10 +8,12 @@ that runs on the raw ``page`` object the copilot code block executes against.
from __future__ import annotations
+import ast
import io
import json
import keyword
import re
+import textwrap
import tokenize
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
@@ -157,6 +159,10 @@ class SynthesisDiagnostics:
emitted_interaction_count: int = 0
truncated: bool = False
dropped_interactions: list[dict[str, Any]] = field(default_factory=list)
+ # Emission ground truth recorded at each emission branch; with dropped/forgiven it partitions the
+ # retained trajectory indices. Diagnostics-only, never serialized.
+ emitted_interactions: list[dict[str, Any]] = field(default_factory=list)
+ forgiven_interactions: list[dict[str, Any]] = field(default_factory=list)
locator_provenance: list[dict[str, Any]] = field(default_factory=list)
# (trajectory enumerate index -> minted type_text parameter key); diagnostics-only, never serialized.
# Recovers the key for a typed field whose value was withheld from default_value (typed_value == "").
@@ -662,6 +668,21 @@ def synthesize_code_block(
}
)
+ def record_emission(
+ trajectory_index: int, tool_name: str, method: str, locator_expr: str, *, line_start: int, lane: str = ""
+ ) -> None:
+ record: dict[str, Any] = {
+ "trajectory_index": trajectory_index,
+ "tool_name": tool_name,
+ "method": method,
+ "selector": str(trajectory[trajectory_index].get("selector") or "").strip(),
+ "locator": locator_expr,
+ "call_source": textwrap.dedent("\n".join(lines[line_start - 1 :])),
+ }
+ if lane:
+ record["lane"] = lane
+ diagnostics.emitted_interactions.append(record)
+
entry_url = ""
entry_index = -1
entry_replay_condition_active = False
@@ -717,7 +738,7 @@ def synthesize_code_block(
notes.append("entry fallback can resume after authentication when login controls stay hidden")
elif fallback_entry_index > entry_index:
notes.append("entry replay starts at a later durable interaction")
- entry_recovery_clicks: list[str] = []
+ entry_recovery_clicks: list[tuple[int, str]] = []
if fallback_entry_index > entry_index:
for recovery_index in range(entry_index, fallback_entry_index):
recovery_interaction = trajectory[recovery_index]
@@ -732,7 +753,7 @@ def synthesize_code_block(
strict_selectors=strict_selectors,
)
if recovery_locator:
- entry_recovery_clicks.append(recovery_locator)
+ entry_recovery_clicks.append((recovery_index, recovery_locator))
if entry_recovery_clicks:
notes.append("entry fallback replays a generic opener only when the durable target stays hidden")
line_start = len(lines) + 1
@@ -781,13 +802,22 @@ def synthesize_code_block(
recovery_indent = post_goto_indent + 2
else:
recovery_indent = post_goto_indent + 1
- for recovery_locator in entry_recovery_clicks:
+ for recovery_index, recovery_locator in entry_recovery_clicks:
+ recovery_line_start = len(lines) + 1
lines.append(f"{_INDENT * recovery_indent}{_ENTRY_OPENER_VAR} = {recovery_locator}")
lines.append(f"{_INDENT * recovery_indent}if await {_ENTRY_OPENER_VAR}.count() == 1:")
lines.append(f"{_INDENT * (recovery_indent + 1)}await {_ENTRY_OPENER_VAR}.click()")
lines.append(
f"{_INDENT * (recovery_indent + 1)}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})"
)
+ record_emission(
+ recovery_index,
+ "click",
+ "click",
+ recovery_locator,
+ line_start=recovery_line_start,
+ lane="entry_recovery",
+ )
lines.append(f'{_INDENT * recovery_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible")')
else:
lines.append(f'{_INDENT * post_goto_indent}await {_ENTRY_TARGET_VAR}.wait_for(state="visible")')
@@ -824,6 +854,18 @@ def synthesize_code_block(
notes.append(f"trajectory truncated at {_MAX_STEPS} steps")
break
if entry_replay_start_index and trajectory_index < entry_replay_start_index:
+ already_recorded = any(
+ record.get("trajectory_index") == trajectory_index
+ for record in (*diagnostics.emitted_interactions, *diagnostics.dropped_interactions)
+ )
+ if not already_recorded:
+ diagnostics.forgiven_interactions.append(
+ {
+ "trajectory_index": trajectory_index,
+ "tool_name": str(interaction.get("tool_name") or ""),
+ "lane": "entry_replay_prefix",
+ }
+ )
continue
action_indent = action_indent_for(trajectory_index)
tool_name = str(interaction.get("tool_name") or "")
@@ -850,6 +892,7 @@ def synthesize_code_block(
line_start = len(lines) + 1
if locator:
lines.append(f"{action_indent}await {locator}.press({_py_str(key)})")
+ record_emission(trajectory_index, tool_name, "press", locator, line_start=line_start)
else:
if strict_selectors:
diagnostics.dropped_interactions.append(
@@ -861,6 +904,7 @@ def synthesize_code_block(
)
continue
lines.append(f"{action_indent}await page.keyboard.press({_py_str(key)})")
+ record_emission(trajectory_index, tool_name, "press", "page.keyboard", line_start=line_start)
lines.append(f"{action_indent}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})")
append_step(f"Press {key}", "keypress", line_start)
emitted += 1
@@ -890,9 +934,18 @@ def synthesize_code_block(
)
lines.append(f"{action_indent}{_INDENT}except Exception:")
lines.append(f"{action_indent}{_INDENT * 2}pass")
+ record_emission(
+ trajectory_index,
+ tool_name,
+ "click",
+ optional_locator,
+ line_start=line_start,
+ lane="optional_dismissal",
+ )
else:
lines.append(f"{action_indent}await {locator}.click()")
lines.append(f"{action_indent}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})")
+ record_emission(trajectory_index, tool_name, "click", locator, line_start=line_start)
append_step(f"Click {_step_target(interaction)}", "click", line_start)
elif tool_name == "type_text":
typed_identity = _typed_value_identity(interaction)
@@ -933,11 +986,15 @@ def synthesize_code_block(
f"{_py_str(f'{verify_target}: read-only value ')} + repr({_READONLY_DEFERRED_VAR})"
f" + {_py_str(' does not match expected ')} + repr(str({param_key})))"
)
+ record_emission(
+ trajectory_index, tool_name, "input_value", locator, line_start=line_start, lane="readonly_skip"
+ )
append_step(f"Verify {verify_target}", "input_text", line_start)
elif readonly_or_disabled:
deferred_readonly_assertions.append((trajectory_index, locator, param_key, _step_target(interaction)))
else:
lines.append(f"{action_indent}await {locator}.fill(str({param_key}))")
+ record_emission(trajectory_index, tool_name, "fill", locator, line_start=line_start)
append_step(f"Type into {_step_target(interaction)}", "input_text", line_start)
elif tool_name == CREDENTIAL_FILL_TOOL_NAME:
credential_id = str(interaction.get("credential_id") or "").strip()
@@ -961,6 +1018,7 @@ def synthesize_code_block(
lines.append(f"{action_indent}await {locator}.fill(await {credential_param_key}.otp())")
else:
lines.append(f"{action_indent}await {locator}.fill({credential_param_key}.{credential_field})")
+ record_emission(trajectory_index, tool_name, "fill", locator, line_start=line_start)
elif tool_name == "select_option":
value = str(interaction.get("value") or "").strip()
if not value:
@@ -971,6 +1029,7 @@ def synthesize_code_block(
continue
lines.append(f"{action_indent}await {locator}.select_option({_py_str(value)})")
lines.append(f"{action_indent}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})")
+ record_emission(trajectory_index, tool_name, "select_option", locator, line_start=line_start)
append_step(f"Select {value} in {_step_target(interaction)}", "select_option", line_start)
else:
notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")
@@ -1017,7 +1076,16 @@ def synthesize_code_block(
for deferred_index, deferred_locator, deferred_param_key, deferred_target in deferred_readonly_assertions:
if entry_post_auth_resume_index and deferred_index < entry_post_auth_resume_index:
continue
+ deferred_line_start = len(lines) + 1
emit_deferred_readonly_assertion(deferred_base, deferred_locator, deferred_param_key, deferred_target)
+ record_emission(
+ deferred_index,
+ "type_text",
+ "input_value",
+ deferred_locator,
+ line_start=deferred_line_start,
+ lane="readonly_skip",
+ )
pre_resume_deferred = [
entry
@@ -1026,10 +1094,19 @@ def synthesize_code_block(
]
if pre_resume_deferred:
lines.append(f"{deferred_base}if not {_ENTRY_RESUME_AFTER_AUTH_VAR}:")
- for _, deferred_locator, deferred_param_key, deferred_target in pre_resume_deferred:
+ for deferred_index, deferred_locator, deferred_param_key, deferred_target in pre_resume_deferred:
+ deferred_line_start = len(lines) + 1
emit_deferred_readonly_assertion(
deferred_base + _INDENT, deferred_locator, deferred_param_key, deferred_target
)
+ record_emission(
+ deferred_index,
+ "type_text",
+ "input_value",
+ deferred_locator,
+ line_start=deferred_line_start,
+ lane="readonly_skip",
+ )
if compile_download_target and reached_download_target is not None:
# The download affordance is observed in nav_targets, not necessarily a trajectory click, so the
@@ -1069,6 +1146,95 @@ def synthesize_code_block(
return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes, diagnostics=diagnostics, steps=steps)
+SCOUTED_SPINE_UNDER_BUILD_REASON_CODE = "scouted_spine_under_build"
+
+
+def normalized_locator_expr(text: str) -> str:
+ try:
+ return ast.unparse(ast.parse(text, mode="eval"))
+ except SyntaxError:
+ return text
+
+
+def locator_selector_literals(locator: str) -> set[str]:
+ try:
+ tree = ast.parse(locator, mode="eval")
+ except SyntaxError:
+ return set()
+ return {node.value for node in ast.walk(tree) if isinstance(node, ast.Constant) and isinstance(node.value, str)}
+
+
+def _bare_locator_call_selector(receiver: str) -> str | None:
+ try:
+ node = ast.parse(receiver, mode="eval").body
+ except SyntaxError:
+ return None
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "locator"
+ and len(node.args) == 1
+ and not node.keywords
+ and isinstance(node.args[0], ast.Constant)
+ and isinstance(node.args[0].value, str)
+ ):
+ return node.args[0].value
+ return None
+
+
+def emitted_record_covered_by_call(record: Mapping[str, Any], method: str, receiver: str) -> bool:
+ if method != str(record.get("method") or ""):
+ return False
+ locator = str(record.get("locator") or "")
+ if locator and normalized_locator_expr(receiver) == normalized_locator_expr(locator):
+ return True
+ # Literal-membership matching would falsely cover a receiver that merely quotes the captured
+ # selector (a shared name= across different elements), so only a bare .locator(<selector>) counts.
+ selector = str(record.get("selector") or "")
+ return bool(selector) and selector == _bare_locator_call_selector(receiver)
+
+
+def uncovered_required_emitted_interactions(
+ emitted_interactions: Sequence[Mapping[str, Any]],
+ draft_calls: Sequence[tuple[str, str]],
+) -> list[Mapping[str, Any]]:
+ """Required (non-lane) emitted records the draft's ordered (method, receiver) calls do not
+ cover as an ordered subsequence, matched at method + selector/locator level."""
+ required = [record for record in emitted_interactions if not str(record.get("lane") or "")]
+ if not required:
+ return []
+ # Greedy ordered-subsequence scan: a miss consumes the remaining calls, so later rungs over-report (safe superset).
+ uncovered: list[Mapping[str, Any]] = []
+ next_call_index = 0
+ for record in required:
+ match_index = None
+ for call_index in range(next_call_index, len(draft_calls)):
+ method, receiver = draft_calls[call_index]
+ if emitted_record_covered_by_call(record, method, receiver):
+ match_index = call_index
+ break
+ if match_index is None:
+ uncovered.append(record)
+ next_call_index = len(draft_calls)
+ else:
+ next_call_index = match_index + 1
+ return uncovered
+
+
+def missing_rung_text(uncovered: Sequence[Mapping[str, Any]]) -> str:
+ return ", ".join(
+ f"`{str(record.get('method') or '')}` on {str(record.get('selector') or record.get('locator') or '')!r}"
+ for record in uncovered
+ )
+
+
+def render_missing_rung_call_sources(uncovered: Sequence[Mapping[str, Any]]) -> str:
+ sources = [source for record in uncovered if (source := str(record.get("call_source") or "").strip())]
+ if not sources:
+ return ""
+ return "Missing rung source to reuse verbatim:\n```python\n" + "\n".join(sources) + "\n```"
+
+
# Model-owned slots the synthesizer cannot prove; the model fills these.
_FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"
_FILL_CLAIM_ID = "claim:<fill>"
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index aab5f4338..927eba23a 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -7,6 +7,7 @@ import copy
import json
import re
import time
+from collections.abc import Mapping
from dataclasses import replace
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
@@ -29,16 +30,23 @@ from skyvern.forge.sdk.copilot.build_phase import DISCOVERY_PERMITTED_PHASES
from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedBuildTestOutcome,
author_time_reject_missing_output_paths,
+ latest_recorded_build_test_outcome_repeated,
+ record_build_test_outcome,
+ recorded_outcome_from_authoring_repair_context,
run_backed_repair_evidence_exists,
)
from skyvern.forge.sdk.copilot.challenge_evidence import composition_challenge_carrier
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
is_durable_fallback_entry_target,
is_generic_entry_opener_click,
is_optional_dismissal_only_trajectory,
+ missing_rung_text,
+ render_missing_rung_call_sources,
render_synthesized_offer_text,
synthesize_code_block,
trajectory_has_browser_fill_interaction,
+ uncovered_required_emitted_interactions,
)
from skyvern.forge.sdk.copilot.completion_criteria_store import requested_output_paths
from skyvern.forge.sdk.copilot.completion_verification import only_structural_requested_output_abstentions
@@ -70,6 +78,7 @@ from skyvern.forge.sdk.copilot.config import (
CopilotConfig,
normalize_block_authoring_policy,
)
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import (
DiagnosisRepairContract,
RepairLoopState,
@@ -107,7 +116,12 @@ from skyvern.forge.sdk.copilot.run_outcome import (
RecordedRunOutcome,
run_outcome_display_reason,
)
-from skyvern.forge.sdk.copilot.runtime import AuthorTimeGateAblationPayload, record_author_time_gate_ablation_event
+from skyvern.forge.sdk.copilot.runtime import (
+ AgentContext,
+ AuthorTimeGateAblationPayload,
+ output_contract_ladder_unresolved,
+ record_author_time_gate_ablation_event,
+)
from skyvern.forge.sdk.copilot.screenshot_utils import ScreenshotEntry
from skyvern.forge.sdk.copilot.terminal_predicates import (
artifact_health_blocked,
@@ -430,6 +444,144 @@ def credential_priority_authoring_churn_stop_signal(ctx: Any) -> CopilotToolBloc
)
+_CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_priority_authoring_churn"})
+_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL = "persisted_draft"
+
+
+def _scouted_spine_open_obligation(ctx: AgentContext) -> list[Mapping[str, Any]]:
+ """Required scouted rungs the latest persisted draft leaves uncovered; empty when no in-turn
+ persist exists or coverage holds."""
+ persisted_calls = ctx.persisted_draft_browser_calls
+ if persisted_calls is None:
+ return []
+ if not ctx.impose_synthesized_code_block:
+ return []
+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return []
+ trajectory = ctx.scout_trajectory
+ if not trajectory:
+ return []
+ if not str(trajectory[0].get("source_url") or "").strip():
+ return []
+ synthesized = synthesize_code_block(
+ trajectory,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
+ )
+ if synthesized is None:
+ return []
+ return uncovered_required_emitted_interactions(synthesized.diagnostics.emitted_interactions, persisted_calls)
+
+
+def _log_scouted_spine_unresolved(uncovered: list[Mapping[str, Any]], *, site: str) -> None:
+ LOG.info(
+ "copilot_scouted_spine_under_build_unresolved",
+ site=site,
+ missing_rung_count=len(uncovered),
+ missing_rungs=missing_rung_text(uncovered),
+ )
+
+
+SCOUTED_SPINE_TURN_HALT_USER_REASON = (
+ "I couldn't get past the same problem after several attempts. The saved draft is still missing "
+ "steps I demonstrated while scouting, and my rewrites kept leaving them out. "
+ "Tell me what to change and I'll try a different approach."
+)
+
+
+def log_scouted_spine_unresolved_at_turn_halt(ctx: AgentContext) -> bool:
+ """Log-only and never raises: a failed obligation read must not block rendering the halt reply."""
+ try:
+ uncovered = _scouted_spine_open_obligation(ctx)
+ except Exception:
+ LOG.warning("copilot_scouted_spine_turn_halt_check_failed", exc_info=True)
+ return False
+ if not uncovered:
+ return False
+ _log_scouted_spine_unresolved(uncovered, site="turn_halt")
+ return True
+
+
+def _scouted_spine_turn_end_nudge(ctx: AgentContext) -> str | None:
+ try:
+ uncovered = _scouted_spine_open_obligation(ctx)
+ except Exception:
+ LOG.warning("copilot_scouted_spine_turn_end_check_failed", exc_info=True)
+ return None
+ if not uncovered:
+ return None
+ if ctx.scouted_spine_checkpoint_fired:
+ _log_scouted_spine_unresolved(uncovered, site="turn_end")
+ return None
+ ctx.scouted_spine_checkpoint_fired = True
+ repair_context = CodeAuthoringRepairContext(
+ block_label=_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL,
+ reason_code=SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
+ selector=str(uncovered[0].get("selector") or "") or None,
+ )
+ ctx.last_code_authoring_repair_context = repair_context
+ record_build_test_outcome(ctx, recorded_outcome_from_authoring_repair_context(repair_context))
+ _record_code_authoring_guardrail_reject(ctx)
+ missing_text = missing_rung_text(uncovered)
+ LOG.info(
+ "copilot_scouted_spine_under_build",
+ block_label=_SCOUTED_SPINE_CHECKPOINT_BLOCK_LABEL,
+ site="turn_end",
+ missing_rung_count=len(uncovered),
+ missing_rungs=missing_text,
+ )
+ nudge = (
+ f"The persisted draft under-builds the scouted spine ({SCOUTED_SPINE_UNDER_BUILD_REASON_CODE}): "
+ f"missing rung(s): {missing_text}. Resubmit the code block through update_workflow so every scouted "
+ "rung is replayed — reuse the synthesized code block verbatim."
+ )
+ artifact = render_missing_rung_call_sources(uncovered)
+ if artifact:
+ nudge += "\n" + artifact
+ return nudge
+
+
+def _record_code_authoring_guardrail_reject(
+ ctx: AgentContext, *, defer_churn_stop: bool = False, frontier_unchanged: bool = False
+) -> None:
+ # Callers record the current build-test outcome first so repeat detection compares that key to history.
+ repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
+ # A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each
+ # turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.
+ if repeated_outcome is False and not frontier_unchanged:
+ ctx.code_authoring_guardrail_reject_count = 0
+ ctx.code_authoring_guardrail_reject_count += 1
+ ctx.last_code_authoring_reject_was_credential_priority = defer_churn_stop
+ LOG.info(
+ "copilot code-authoring guardrail reject recorded",
+ reject_count=ctx.code_authoring_guardrail_reject_count,
+ credential_priority=defer_churn_stop,
+ )
+ if defer_churn_stop:
+ if ctx.code_authoring_guardrail_reject_count < MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS:
+ return
+ signal: CopilotToolBlockerSignal = credential_priority_authoring_churn_stop_signal(ctx)
+ elif ctx.code_authoring_guardrail_reject_count < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS:
+ return
+ else:
+ signal = code_authoring_churn_stop_signal(ctx)
+ # A genuinely-terminal held blocker keeps both the rendered reply and the
+ # halt kind, so the churn stop defers to it rather than overriding.
+ if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
+ return
+ if output_contract_ladder_unresolved(ctx):
+ return
+ try:
+ unresolved_obligation = _scouted_spine_open_obligation(ctx)
+ except Exception:
+ LOG.warning("copilot_scouted_spine_churn_stop_check_failed", exc_info=True)
+ unresolved_obligation = []
+ if unresolved_obligation:
+ _log_scouted_spine_unresolved(unresolved_obligation, site="churn_stop")
+ stash_blocker_signal(ctx, signal)
+ ctx.blocker_signal = signal
+
+
def no_forward_progress_interaction_stop_signal(ctx: Any) -> CopilotToolBlockerSignal:
count = _get_int(ctx, "consecutive_no_progress_interaction_count")
evidence = loop_blocker_evidence_from_ctx(ctx)
@@ -2788,23 +2940,38 @@ async def run_with_enforcement(
# nudge path and the finalize path.
synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)
+ spine_checkpoint_nudge = False
if nudge is None and synthesized_msg is None:
- _consume_pending_screenshots(ctx)
- _maybe_raise_non_retriable_nav(ctx)
- return result
+ spine_nudge = _scouted_spine_turn_end_nudge(ctx)
+ if spine_nudge is None:
+ _consume_pending_screenshots(ctx)
+ _maybe_raise_non_retriable_nav(ctx)
+ return result
+ nudge = spine_nudge
+ spine_checkpoint_nudge = True
- if nudge is not None and nudge == _nudge(copilot_config, "post_update"):
+ if nudge is not None and not spine_checkpoint_nudge and nudge == _nudge(copilot_config, "post_update"):
if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:
LOG.warning(
"Enforcement exhausted post-update nudges, allowing response",
nudge_count=ctx.post_update_nudge_count,
)
- _consume_pending_screenshots(ctx)
- _maybe_raise_non_retriable_nav(ctx)
- return result
- ctx.post_update_nudge_count += 1
+ spine_nudge = _scouted_spine_turn_end_nudge(ctx)
+ if spine_nudge is None:
+ _consume_pending_screenshots(ctx)
+ _maybe_raise_non_retriable_nav(ctx)
+ return result
+ nudge = spine_nudge
+ spine_checkpoint_nudge = True
+ else:
+ ctx.post_update_nudge_count += 1
- nudge_type = _nudge_type_for_log(nudge, copilot_config) if nudge is not None else "synthesized_block_offer"
+ if spine_checkpoint_nudge:
+ nudge_type = "scouted_spine_under_build_checkpoint"
+ elif nudge is not None:
+ nudge_type = _nudge_type_for_log(nudge, copilot_config)
+ else:
+ nudge_type = "synthesized_block_offer"
LOG.info("Enforcement nudge", nudge_type=nudge_type, iteration=iteration)
# OpenAI rejects images in tool messages, so a queued post-run
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 584787c72..25eaae837 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -372,6 +372,10 @@ class AgentContext:
# Latest typed reached-download target from the scout steer; the synthesizer compiles the terminal
# expect_download step from it. Selector is the observed download link, not necessarily a trajectory click.
reached_download_target: ReachedDownloadTarget | None = None
+ # Ordered (method, receiver) browser mutations of the last successfully persisted draft's code
+ # blocks; None until a persist succeeds this turn. Gates the scouted-spine under-build reject and turn-end nudge.
+ persisted_draft_browser_calls: list[tuple[str, str]] | None = None
+ scouted_spine_checkpoint_fired: bool = False
# Author-time output-contract cross-turn state, keyed by the contract signature; set lazily by workflow_update.
output_contract_pinned_block_label_by_signature: dict[str, str] = field(default_factory=dict)
output_contract_reject_count_by_signature: dict[str, int] = field(default_factory=dict)
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 5e70639fe..d8c55bbb8 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -5,7 +5,7 @@ import hashlib
import json
import re
import time
-from typing import Any, cast
+from typing import Any, Literal, cast
from urllib.parse import urlparse
import structlog
@@ -306,6 +306,29 @@ async def _resolve_scout_role_name(
return "", ""
+def _capped_with_eviction_accounting(
+ items: list[ScoutedInteraction],
+ *,
+ collection: Literal["scout_trajectory", "scouted_interactions"],
+) -> list[ScoutedInteraction]:
+ if len(items) <= _MAX_SCOUTED_INTERACTIONS:
+ return items
+ try:
+ for item in items[: len(items) - _MAX_SCOUTED_INTERACTIONS]:
+ event: dict[str, Any] = {
+ "collection": collection,
+ "tool_name": item.get("tool_name"),
+ "selector": item.get("selector"),
+ "source_url": item.get("source_url"),
+ }
+ if collection == "scout_trajectory":
+ event["trajectory_index"] = item.get("trajectory_index")
+ LOG.info("copilot_scout_interaction_evicted", **event)
+ except Exception:
+ pass
+ return items[-_MAX_SCOUTED_INTERACTIONS:]
+
+
def _record_scouted_interaction(
ctx: AgentContext,
*,
@@ -329,6 +352,12 @@ def _record_scouted_interaction(
selector = _selector_text(selector)
# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
if tool_name != "press_key" and not selector:
+ LOG.info(
+ "copilot_scout_capture_loss",
+ tool_name=tool_name,
+ reason="unresolvable_selector",
+ url=(source_url or "").strip() or None,
+ )
return
_reset_evaluate_tracker(ctx)
artifact: ScoutedInteraction = {"tool_name": tool_name}
@@ -374,13 +403,13 @@ def _record_scouted_interaction(
)
]
interactions.append(artifact)
- ctx.scouted_interactions = interactions[-_MAX_SCOUTED_INTERACTIONS:]
+ ctx.scouted_interactions = _capped_with_eviction_accounting(interactions, collection="scouted_interactions")
trajectory = list(ctx.scout_trajectory)
trajectory_artifact = cast(ScoutedInteraction, artifact.copy())
trajectory_artifact["trajectory_index"] = len(trajectory)
trajectory.append(trajectory_artifact)
- ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
+ ctx.scout_trajectory = _capped_with_eviction_accounting(trajectory, collection="scout_trajectory")
LOG.info(
"copilot_scout_interaction_captured",
@@ -517,7 +546,7 @@ async def _maybe_rebind_prior_fill_carry(
trajectory = list(ctx.scout_trajectory)
for carry in rebound:
trajectory.append(_fill_carry_to_interaction(carry, len(trajectory)))
- ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
+ ctx.scout_trajectory = _capped_with_eviction_accounting(trajectory, collection="scout_trajectory")
LOG.info(
"copilot_fill_carry_rebound",
url=url,
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 26ad8c3ab..87a458ee2 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -36,7 +36,6 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
RecordedOutcomeBindingConstraint,
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
- latest_recorded_build_test_outcome_repeated,
record_build_test_outcome,
recorded_outcome_from_author_time_reject,
recorded_outcome_from_authoring_repair_context,
@@ -54,6 +53,8 @@ from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_
from skyvern.forge.sdk.copilot.code_block_synthesis import (
_BARE_TAG_RE,
_SYNTHESIZED_BLOCK_LABEL,
+ CREDENTIAL_FILL_TOOL_NAME,
+ SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
SynthesisDiagnostics,
SynthesizedCodeBlock,
_get_by_role_expr_strict,
@@ -61,7 +62,12 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
_parse_role_name,
artifact_dependency_id,
artifact_observation_ref_id,
+ locator_selector_literals,
+ missing_rung_text,
+ normalized_locator_expr,
+ render_missing_rung_call_sources,
synthesize_code_block,
+ uncovered_required_emitted_interactions,
)
from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
@@ -77,10 +83,9 @@ from skyvern.forge.sdk.copilot.context import (
)
from skyvern.forge.sdk.copilot.data_write_defaults import default_data_write_continue_on_failure
from skyvern.forge.sdk.copilot.enforcement import (
- MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
- MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
- code_authoring_churn_stop_signal,
- credential_priority_authoring_churn_stop_signal,
+ _CHURN_REASON_CODES,
+ _record_code_authoring_guardrail_reject,
+ _scouted_spine_open_obligation,
synthesized_persistence_reopened_after_failed_run,
)
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
@@ -121,7 +126,6 @@ from skyvern.forge.sdk.copilot.runtime import (
AuthorTimeGateAblationPayload,
ScoutedInteraction,
copilot_author_time_gate_log_only_enabled,
- output_contract_ladder_unresolved,
record_author_time_gate_ablation_event,
)
from skyvern.forge.sdk.copilot.schema_incompatibility import (
@@ -134,7 +138,6 @@ from skyvern.forge.sdk.copilot.schema_incompatibility import (
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.copilot.turn_halt import (
- blocker_signal_is_genuinely_terminal,
stash_repair_ceiling_turn_halt,
stash_turn_halt_from_blocker_signal,
)
@@ -1220,43 +1223,6 @@ def _is_unresolved_symbol_repair_context(repair_context: CodeAuthoringRepairCont
return repair_context is not None and repair_context.reason_code == SANDBOX_UNRESOLVED_NAME_REASON_CODE
-_CHURN_REASON_CODES = frozenset({"code_authoring_guardrail_churn", "credential_priority_authoring_churn"})
-
-
-def _record_code_authoring_guardrail_reject(
- ctx: AgentContext, *, defer_churn_stop: bool = False, frontier_unchanged: bool = False
-) -> None:
- # Callers record the current build-test outcome first so repeat detection compares that key to history.
- repeated_outcome = latest_recorded_build_test_outcome_repeated(ctx)
- # A frontier-unchanged reject is churn even when sibling edits move the whole-signature key each
- # turn (which reads as a non-repeat); it must accumulate toward the churn stop, not reset.
- if repeated_outcome is False and not frontier_unchanged:
- ctx.code_authoring_guardrail_reject_count = 0
- ctx.code_authoring_guardrail_reject_count += 1
- ctx.last_code_authoring_reject_was_credential_priority = defer_churn_stop
- LOG.info(
- "copilot code-authoring guardrail reject recorded",
- reject_count=ctx.code_authoring_guardrail_reject_count,
- credential_priority=defer_churn_stop,
- )
- if defer_churn_stop:
- if ctx.code_authoring_guardrail_reject_count < MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS:
- return
- signal: CopilotToolBlockerSignal = credential_priority_authoring_churn_stop_signal(ctx)
- elif ctx.code_authoring_guardrail_reject_count < MAX_CODE_AUTHORING_GUARDRAIL_REJECTS:
- return
- else:
- signal = code_authoring_churn_stop_signal(ctx)
- # A genuinely-terminal held blocker keeps both the rendered reply and the
- # halt kind, so the churn stop defers to it rather than overriding.
- if blocker_signal_is_genuinely_terminal(ctx.blocker_signal):
- return
- if output_contract_ladder_unresolved(ctx):
- return
- stash_blocker_signal(ctx, signal)
- ctx.blocker_signal = signal
-
-
def _record_author_time_reject_outcome(
ctx: AgentContext,
*,
@@ -5522,6 +5488,24 @@ def _author_time_reject_reopens_synthesized_imposition(ctx: AgentContext) -> boo
)
+def _log_imposition_skipped_after_update(ctx: AgentContext) -> None:
+ scout_trajectory = ctx.scout_trajectory
+ if not scout_trajectory:
+ return
+ target = ctx.reached_download_target
+ LOG.info(
+ "copilot_imposition_skipped_after_update",
+ trajectory_length=len(scout_trajectory),
+ reopen_download_target=(
+ isinstance(target, ReachedDownloadTarget)
+ and not target.already_registered
+ and bool(target.selector.strip())
+ ),
+ reopen_persistence_after_failed_run=synthesized_persistence_reopened_after_failed_run(ctx),
+ reopen_author_time_reject=_author_time_reject_reopens_synthesized_imposition(ctx),
+ )
+
+
def _recorded_outcome_imposition_block_labels(ctx: AgentContext) -> frozenset[str]:
constraint = ctx.recorded_outcome_binding_constraint
if isinstance(constraint, RecordedOutcomeBindingConstraint):
@@ -5669,16 +5653,35 @@ _VALUE_BEARING_READ_METHODS = frozenset(
)
-class _BrowserSurfaceValidation(NamedTuple):
- violations: list[str]
-
-
class _BrowserMutationSignature(NamedTuple):
method: str
receiver: str
call_shape: str
+_BrowserSurfaceProvenanceKind = Literal["never_captured", "shape_diverged", "ambiguous", "suffix_disallowed"]
+_BrowserSurfaceDivergenceSource = Literal["synthesized", "trajectory_dropped"]
+_BrowserSurfaceProvenanceSite = Literal["whole_trajectory", "extraction_suffix"]
+
+_BROWSER_SURFACE_PROVENANCE_EVENT = "copilot_browser_surface_rejection_provenance"
+
+
+class _BrowserSurfaceRejectionProvenance(NamedTuple):
+ kind: _BrowserSurfaceProvenanceKind
+ action: str
+ site: _BrowserSurfaceProvenanceSite
+ block_label: str
+ nearest_method: str | None = None
+ nearest_receiver: str | None = None
+ nearest_selector: str | None = None
+ divergence_source: _BrowserSurfaceDivergenceSource | None = None
+
+
+class _BrowserSurfaceValidation(NamedTuple):
+ violations: list[str]
+ provenance: list[_BrowserSurfaceRejectionProvenance]
+
+
class _BrowserBindings(NamedTuple):
page_aliases: set[str]
locator_aliases: set[str]
@@ -5866,8 +5869,16 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignature
locator_aliases = _collect_locator_aliases(tree)
method_aliases = _collect_browser_method_aliases(tree, page_aliases, locator_aliases)
bindings = _BrowserBindings(page_aliases, locator_aliases, method_aliases)
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call):
+ # ast.walk yields breadth-first; the coverage scan (uncovered_required_emitted_interactions)
+ # matches draft calls as an ordered subsequence, so enumerate Call nodes in source order or a
+ # nested-but-present rung is falsely reported uncovered.
+ call_nodes = sorted(
+ (node for node in ast.walk(tree) if isinstance(node, ast.Call)),
+ key=lambda node: (node.lineno, node.col_offset),
+ )
+ for node in call_nodes:
+ if isinstance(node.func, ast.Name) and node.func.id in bindings.method_aliases:
+ ambiguous.append(_call_name(node))
continue
if isinstance(node.func, ast.Name) and node.func.id in bindings.method_aliases:
ambiguous.append(_call_name(node))
@@ -5895,6 +5906,162 @@ def _browser_surface_for_code(code: str) -> tuple[list[_BrowserMutationSignature
return direct_mutations, sorted(unscouted), sorted(set(ambiguous))
+_SCOUT_TOOL_EMITTED_METHODS = {
+ "click": "click",
+ "type_text": "fill",
+ "press_key": "press",
+ "select_option": "select_option",
+ CREDENTIAL_FILL_TOOL_NAME: "fill",
+}
+
+
+def _captured_selector_for_signature(
+ signature: _BrowserMutationSignature, diagnostics: SynthesisDiagnostics | None
+) -> str | None:
+ if diagnostics is None:
+ return None
+ # Signature receivers come from ast.unparse while emitted locators are synthesizer text, so both
+ # sides are normalized before comparison.
+ receiver = normalized_locator_expr(signature.receiver)
+ for record in diagnostics.emitted_interactions:
+ if (
+ str(record.get("method") or "") == signature.method
+ and normalized_locator_expr(str(record.get("locator") or "")) == receiver
+ ):
+ selector = str(record.get("selector") or "")
+ if selector:
+ return selector
+ return None
+
+
+def _classify_unscouted_mutation(
+ mutation: _BrowserMutationSignature,
+ *,
+ scouted_mutations: list[_BrowserMutationSignature],
+ diagnostics: SynthesisDiagnostics | None,
+ site: _BrowserSurfaceProvenanceSite,
+ block_label: str,
+) -> _BrowserSurfaceRejectionProvenance:
+ action = f"{mutation.receiver}.{mutation.method}"
+ exact = next(
+ (
+ signature
+ for signature in scouted_mutations
+ if signature.method == mutation.method and signature.receiver == mutation.receiver
+ ),
+ None,
+ )
+ if exact is not None:
+ return _BrowserSurfaceRejectionProvenance(
+ kind="shape_diverged",
+ action=action,
+ site=site,
+ block_label=block_label,
+ nearest_method=exact.method,
+ nearest_receiver=exact.receiver,
+ nearest_selector=_captured_selector_for_signature(exact, diagnostics),
+ divergence_source="synthesized",
+ )
+ receiver_literals = locator_selector_literals(mutation.receiver)
+ emitted_records = diagnostics.emitted_interactions if diagnostics is not None else []
+ for record in emitted_records:
+ selector = str(record.get("selector") or "")
+ if str(record.get("method") or "") == mutation.method and selector and selector in receiver_literals:
+ return _BrowserSurfaceRejectionProvenance(
+ kind="shape_diverged",
+ action=action,
+ site=site,
+ block_label=block_label,
+ nearest_method=mutation.method,
+ nearest_receiver=str(record.get("locator") or "") or None,
+ nearest_selector=selector,
+ divergence_source="synthesized",
+ )
+ dropped_records = diagnostics.dropped_interactions if diagnostics is not None else []
+ for record in dropped_records:
+ if _SCOUT_TOOL_EMITTED_METHODS.get(str(record.get("tool_name") or "")) != mutation.method:
+ continue
+ selector = str(record.get("selector") or "")
+ if selector and selector not in receiver_literals:
+ continue
+ return _BrowserSurfaceRejectionProvenance(
+ kind="shape_diverged",
+ action=action,
+ site=site,
+ block_label=block_label,
+ nearest_method=mutation.method,
+ nearest_selector=selector or None,
+ divergence_source="trajectory_dropped",
+ )
+ nearest = next((signature for signature in scouted_mutations if signature.method == mutation.method), None) or next(
+ (signature for signature in scouted_mutations if signature.receiver == mutation.receiver), None
+ )
+ return _BrowserSurfaceRejectionProvenance(
+ kind="never_captured",
+ action=action,
+ site=site,
+ block_label=block_label,
+ nearest_method=nearest.method if nearest is not None else None,
+ nearest_receiver=nearest.receiver if nearest is not None else None,
+ nearest_selector=_captured_selector_for_signature(nearest, diagnostics) if nearest is not None else None,
+ )
+
+
+def _ambiguous_browser_action_provenance(
+ action: str, *, site: _BrowserSurfaceProvenanceSite, block_label: str
+) -> _BrowserSurfaceRejectionProvenance:
+ return _BrowserSurfaceRejectionProvenance(kind="ambiguous", action=action, site=site, block_label=block_label)
+
+
+def _provenance_reject_clause(provenance: _BrowserSurfaceRejectionProvenance) -> str:
+ if provenance.kind == "never_captured":
+ clause = f"`{provenance.action}`: never_captured — the scout never captured this action; re-scout that step"
+ if provenance.nearest_receiver is not None and provenance.nearest_method is not None:
+ clause += f" (nearest scouted signature: `{provenance.nearest_receiver}.{provenance.nearest_method}`)"
+ return clause
+ if provenance.kind == "shape_diverged":
+ clause = f"`{provenance.action}`: shape_diverged ({provenance.divergence_source})"
+ if provenance.nearest_receiver is not None and provenance.nearest_method is not None:
+ clause += (
+ f" — reuse the exact synthesized call on `{provenance.nearest_receiver}.{provenance.nearest_method}`"
+ )
+ else:
+ clause += " — reuse the exact synthesized call for this captured step"
+ if provenance.nearest_selector is not None:
+ clause += f" (captured selector {provenance.nearest_selector!r})"
+ return clause
+ if provenance.kind == "suffix_disallowed":
+ return (
+ f"`{provenance.action}`: suffix_disallowed — the synthesized spine already performs this action; "
+ "remove the duplicate from the extraction suffix"
+ )
+ return (
+ f"`{provenance.action}`: ambiguous — rewrite it as a direct page/locator call "
+ "or reuse the exact synthesized call"
+ )
+
+
+def _log_browser_surface_rejection_provenance(provenance: list[_BrowserSurfaceRejectionProvenance]) -> None:
+ for record in provenance:
+ LOG.info(
+ _BROWSER_SURFACE_PROVENANCE_EVENT,
+ kind=record.kind,
+ action=record.action,
+ site=record.site,
+ block_label=record.block_label,
+ nearest_method=record.nearest_method,
+ nearest_receiver=record.nearest_receiver,
+ nearest_selector=record.nearest_selector,
+ divergence_source=record.divergence_source,
+ )
+
+
+def _provenance_suffix_text(provenance: list[_BrowserSurfaceRejectionProvenance]) -> str:
+ if not provenance:
+ return ""
+ return " Provenance: " + "; ".join(_provenance_reject_clause(record) for record in provenance) + "."
+
+
def _whole_trajectory_browser_surface_violations(
*,
code_blocks: list[dict[str, Any]],
@@ -5902,8 +6069,10 @@ def _whole_trajectory_browser_surface_violations(
submitted_selected_code: str,
synthesized_code: str,
prior_yaml: str | None = None,
+ synthesized_diagnostics: SynthesisDiagnostics | None = None,
) -> _BrowserSurfaceValidation:
violations: list[str] = []
+ provenance: list[_BrowserSurfaceRejectionProvenance] = []
scouted_mutations, _, _ = _browser_surface_for_code(synthesized_code)
scouted_signatures = set(scouted_mutations)
for block in code_blocks:
@@ -5919,16 +6088,35 @@ def _whole_trajectory_browser_surface_violations(
action_text = ", ".join(
f"{mutation.receiver}.{mutation.method}" for mutation in sorted(unscouted_mutations)
)
+ block_provenance = [
+ _classify_unscouted_mutation(
+ mutation,
+ scouted_mutations=scouted_mutations,
+ diagnostics=synthesized_diagnostics,
+ site="whole_trajectory",
+ block_label=label,
+ )
+ for mutation in sorted(unscouted_mutations)
+ ]
+ provenance.extend(block_provenance)
violations.append(
- f"Unable to impose synthesized code block: `{label}` contains unscouted browser action(s): {action_text}."
+ f"Unable to impose synthesized code block: `{label}` contains unscouted browser action(s): "
+ f"{action_text}.{_provenance_suffix_text(block_provenance)}"
)
if block_ambiguous:
+ ambiguous_provenance = [
+ _ambiguous_browser_action_provenance(action, site="whole_trajectory", block_label=label)
+ for action in block_ambiguous
+ ]
+ provenance.extend(ambiguous_provenance)
violations.append(
f"Unable to impose synthesized code block: `{label}` contains ambiguous browser action(s): "
+ ", ".join(block_ambiguous)
+ "."
+ + _provenance_suffix_text(ambiguous_provenance)
)
- return _BrowserSurfaceValidation(violations)
+ _log_browser_surface_rejection_provenance(provenance)
+ return _BrowserSurfaceValidation(violations, provenance)
def _separated_spine_already_imposed(
@@ -5948,7 +6136,16 @@ def _separated_spine_already_imposed(
continue
block_mutations, _, _ = _browser_surface_for_code(str(block.get("code") or ""))
sibling_signatures.update(block_mutations)
- return sibling_signatures == set(scouted_mutations)
+ already_imposed = sibling_signatures == set(scouted_mutations)
+ if already_imposed:
+ LOG.info(
+ "copilot_separated_spine_fast_path",
+ spine_coverage="set_equality",
+ synthesized_mutation_count=len(scouted_mutations),
+ sibling_signature_count=len(sibling_signatures),
+ duplicate_rungs_lost=len(scouted_mutations) != len(set(scouted_mutations)),
+ )
+ return already_imposed
def _browser_surface_contains_full_action_spine(submitted_code: str, synthesized_code: str) -> bool:
@@ -5962,6 +6159,154 @@ def _browser_surface_contains_full_action_spine(submitted_code: str, synthesized
return all(any(candidate == synthesized for candidate in submitted_iter) for synthesized in synthesized_mutations)
+_SCOUTED_SPINE_UNDER_BUILD_REASON_CODE = SCOUTED_SPINE_UNDER_BUILD_REASON_CODE
+
+
+def _synthesized_resubmission_credential_scout_requirements(
+ ctx: AgentContext, synthesized: SynthesizedCodeBlock, *, block_label: str
+) -> list[str]:
+ # Runs the credential-scout gate against the verbatim resubmission the under-build reject asks
+ # for, so the reject names every step of a route that is admissible end-to-end.
+ credential_parameters = [
+ parameter for parameter in synthesized.parameters if str(parameter.get("credential_id") or "").strip()
+ ]
+ if not credential_parameters:
+ return []
+ probe_yaml = yaml.safe_dump(
+ {
+ "workflow_definition": {
+ "parameters": [
+ {
+ "parameter_type": "credential",
+ "key": str(parameter.get("key") or ""),
+ "credential_id": str(parameter.get("credential_id") or ""),
+ }
+ for parameter in credential_parameters
+ ],
+ "blocks": [{"block_type": "code", "label": block_label, "code": synthesized.code}],
+ }
+ },
+ sort_keys=False,
+ )
+ return _credentialed_code_block_scout_gate_errors(probe_yaml, ctx)
+
+
+def _scouted_spine_under_build_result(
+ workflow_yaml: str,
+ *,
+ ctx: AgentContext,
+ synthesized: SynthesizedCodeBlock,
+ draft_codes: list[str],
+ block_label: str,
+ site: str = "imposition",
+) -> _SynthesizedCodeImpositionResult | None:
+ diagnostics = synthesized.diagnostics
+ # Lane-flagged emissions (optional dismissals, readonly verifies, entry recovery) are conditional
+ # or read-only replays, not load-bearing rungs.
+ required = [record for record in diagnostics.emitted_interactions if not str(record.get("lane") or "")]
+ if not required:
+ return None
+ draft_calls = [
+ (mutation.method, mutation.receiver) for code in draft_codes for mutation in _browser_surface_for_code(code)[0]
+ ]
+ uncovered = uncovered_required_emitted_interactions(diagnostics.emitted_interactions, draft_calls)
+ if not uncovered:
+ return None
+ credential_scout_requirements = _synthesized_resubmission_credential_scout_requirements(
+ ctx, synthesized, block_label=block_label
+ )
+ missing_text = missing_rung_text(uncovered)
+ LOG.info(
+ "copilot_scouted_spine_under_build",
+ block_label=block_label,
+ site=site,
+ required_rung_count=len(required),
+ covered_rung_count=len(required) - len(uncovered),
+ missing_rungs=missing_text,
+ credential_scout_precondition_pending=bool(credential_scout_requirements),
+ )
+ first_uncovered = uncovered[0]
+ pass_route = render_missing_rung_call_sources(uncovered)
+ violation = (
+ f"Unable to impose synthesized code block: `{block_label}` under-builds the scouted spine "
+ f"({_SCOUTED_SPINE_UNDER_BUILD_REASON_CODE}): the draft covers "
+ f"{len(required) - len(uncovered)} of {len(required)} scouted rung(s); missing: {missing_text}. "
+ )
+ if credential_scout_requirements:
+ violation += (
+ "Two steps are required, in order. Step 1 — satisfy the credential-scout precondition: "
+ + " ".join(credential_scout_requirements)
+ + " Step 2 — resubmit the code block, reusing the synthesized rung source verbatim so every "
+ "scouted rung is replayed."
+ )
+ else:
+ violation += (
+ "Author the remaining synthesized rungs — reuse the synthesized code block verbatim so every "
+ "scouted rung is replayed."
+ )
+ if pass_route:
+ violation += "\n" + pass_route
+ return _SynthesizedCodeImpositionResult(
+ workflow_yaml=workflow_yaml,
+ violations=[violation],
+ repair_context=CodeAuthoringRepairContext(
+ block_label=block_label,
+ reason_code=_SCOUTED_SPINE_UNDER_BUILD_REASON_CODE,
+ selector=str(first_uncovered.get("selector") or "") or None,
+ ),
+ )
+
+
+def _persist_seam_spine_under_build_result(
+ workflow_yaml: str, ctx: AgentContext
+) -> _SynthesizedCodeImpositionResult | None:
+ if not ctx.impose_synthesized_code_block:
+ return None
+ # First-persist drafts stay imposition's concern; this seam guards later persists in a turn that
+ # already committed one, and the turn-end checkpoint owns coverage for everything else.
+ if not ctx.update_workflow_called and ctx.persisted_draft_browser_calls is None:
+ return None
+ if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ return None
+ scout_trajectory = ctx.scout_trajectory
+ if not scout_trajectory:
+ return None
+ if not str(scout_trajectory[0].get("source_url") or "").strip():
+ return None
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return None
+ code_blocks = _workflow_code_blocks(parsed)
+ synthesized = synthesize_code_block(
+ scout_trajectory,
+ strict_selectors=True,
+ reached_download_target=ctx.reached_download_target,
+ )
+ if synthesized is None:
+ return None
+ # A submission with zero code blocks still holds the open spine obligation: empty draft calls
+ # leave every required rung uncovered rather than slipping the seam.
+ return _scouted_spine_under_build_result(
+ workflow_yaml,
+ ctx=ctx,
+ synthesized=synthesized,
+ draft_codes=[str(block.get("code") or "") for block in code_blocks],
+ block_label=", ".join(_code_block_label(block) for block in code_blocks) or _SYNTHESIZED_BLOCK_LABEL,
+ site="persist_seam",
+ )
+
+
+def _workflow_yaml_browser_call_pairs(workflow_yaml: str) -> list[tuple[str, str]]:
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return []
+ return [
+ (mutation.method, mutation.receiver)
+ for block in _workflow_code_blocks(parsed)
+ for mutation in _browser_surface_for_code(str(block.get("code") or ""))[0]
+ ]
+
+
_IDENTITY_QUALIFIER_BOUNDARY = ("[", "#", ".")
_FILTERING_PSEUDO_CLASSES = (
":visible",
@@ -7180,6 +7525,7 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
if _copilot_block_authoring_policy(ctx) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
if ctx.update_workflow_called and not _should_impose_after_update_attempt(ctx):
+ _log_imposition_skipped_after_update(ctx)
return _SynthesizedCodeImpositionResult(workflow_yaml=workflow_yaml)
scout_trajectory = getattr(ctx, "scout_trajectory", None)
@@ -7259,23 +7605,57 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
submitted_selected_code=submitted_code,
synthesized_code=synthesized.code,
prior_yaml=prior_yaml,
+ synthesized_diagnostics=diagnostics,
)
violations.extend(surface_validation.violations)
+ ambiguous_reject_present = any(record.kind == "ambiguous" for record in surface_validation.provenance)
extraction_suffix = _submitted_suffix_after_synthesized_code(submitted_code, synthesized.code)
if extraction_suffix:
suffix_mutations, _, suffix_ambiguous = _browser_surface_for_code(extraction_suffix)
+ selected_label = _code_block_label(code_block)
if suffix_mutations:
action_text = ", ".join(f"{mutation.receiver}.{mutation.method}" for mutation in sorted(suffix_mutations))
+ synthesized_mutations, _, _ = _browser_surface_for_code(synthesized.code)
+ synthesized_signatures = set(synthesized_mutations)
+ suffix_provenance = [
+ _BrowserSurfaceRejectionProvenance(
+ kind="suffix_disallowed",
+ action=f"{mutation.receiver}.{mutation.method}",
+ site="extraction_suffix",
+ block_label=selected_label,
+ nearest_method=mutation.method,
+ nearest_receiver=mutation.receiver,
+ nearest_selector=_captured_selector_for_signature(mutation, diagnostics),
+ )
+ if mutation in synthesized_signatures
+ else _classify_unscouted_mutation(
+ mutation,
+ scouted_mutations=synthesized_mutations,
+ diagnostics=diagnostics,
+ site="extraction_suffix",
+ block_label=selected_label,
+ )
+ for mutation in sorted(suffix_mutations)
+ ]
+ _log_browser_surface_rejection_provenance(suffix_provenance)
violations.append(
"Unable to impose synthesized code block: extraction suffix contains unscouted browser action(s): "
+ action_text
+ "."
+ + _provenance_suffix_text(suffix_provenance)
)
if suffix_ambiguous:
+ ambiguous_reject_present = True
+ suffix_ambiguous_provenance = [
+ _ambiguous_browser_action_provenance(action, site="extraction_suffix", block_label=selected_label)
+ for action in suffix_ambiguous
+ ]
+ _log_browser_surface_rejection_provenance(suffix_ambiguous_provenance)
violations.append(
"Unable to impose synthesized code block: extraction suffix contains ambiguous browser action(s): "
+ ", ".join(suffix_ambiguous)
+ "."
+ + _provenance_suffix_text(suffix_ambiguous_provenance)
)
parameter_reconciliation = _reconcile_synthesized_parameters(
@@ -7290,6 +7670,12 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
if repair_context is None:
repair_context = parameter_reconciliation.repair_context
if violations:
+ persisted_calls = ctx.persisted_draft_browser_calls
+ if ambiguous_reject_present and ctx.update_workflow_called and persisted_calls is not None:
+ open_obligation = uncovered_required_emitted_interactions(diagnostics.emitted_interactions, persisted_calls)
+ artifact = render_missing_rung_call_sources(open_obligation)
+ if artifact:
+ violations.append(_missing_scouted_rung_violation_text(artifact))
return _SynthesizedCodeImpositionResult(
workflow_yaml=workflow_yaml,
violations=violations,
@@ -7410,6 +7796,16 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
violations=split_violations,
repair_context=repair_context,
)
+ split_under_build = _scouted_spine_under_build_result(
+ workflow_yaml,
+ ctx=ctx,
+ synthesized=synthesized,
+ draft_codes=[*durable_stage_codes, split_extraction_code],
+ block_label=str(code_block.get("label") or ""),
+ site="separated_split",
+ )
+ if split_under_build is not None:
+ return split_under_build
substitutions["separated_browser_stage_count"] = len(durable_stage_codes)
else:
imposed_code = textwrap.dedent(submitted_code if preserve_submitted_extraction else synthesized.code).lstrip(
@@ -7420,6 +7816,15 @@ def _maybe_impose_synthesized_code_block(workflow_yaml: str, ctx: AgentContext)
elif append_selected_extraction:
imposed_code = imposed_code.rstrip() + "\n" + textwrap.dedent(submitted_code).strip() + "\n"
imposed_code = _apply_parameter_reconciliation_to_code(imposed_code, parameter_reconciliation)
+ under_build = _scouted_spine_under_build_result(
+ workflow_yaml,
+ ctx=ctx,
+ synthesized=synthesized,
+ draft_codes=[imposed_code],
+ block_label=str(code_block.get("label") or ""),
+ )
+ if under_build is not None:
+ return under_build
code_block["code"] = imposed_code
if parameter_reconciliation.aliases:
substitutions["parameter_aliases"] = parameter_reconciliation.aliases
@@ -8441,6 +8846,30 @@ def _credentialed_code_block_scout_gate_errors(
return errors
+def _missing_scouted_rung_violation_text(artifact: str) -> str:
+ return "The persisted draft is missing scouted rung(s). " + artifact
+
+
+def _open_scouted_spine_obligation_artifact(ctx: AgentContext) -> str:
+ try:
+ uncovered = _scouted_spine_open_obligation(ctx)
+ except Exception:
+ LOG.warning("copilot_scouted_spine_obligation_read_failed", exc_info=True)
+ return ""
+ artifact = render_missing_rung_call_sources(uncovered)
+ if not artifact:
+ return ""
+ return _missing_scouted_rung_violation_text(artifact)
+
+
+def _credential_scout_reject_error_text(ctx: AgentContext, credential_scout_errors: list[str]) -> str:
+ error_text = "\n".join(credential_scout_errors)
+ obligation_artifact = _open_scouted_spine_obligation_artifact(ctx)
+ if obligation_artifact:
+ error_text += "\n" + obligation_artifact
+ return error_text
+
+
def _reject_schema_incompatibility(
ctx: AgentContext,
incompatibility: SchemaIncompatibility,
@@ -8529,6 +8958,19 @@ async def _update_workflow(
required_paths,
reject_family=_SYNTHESIZED_PARAMETER_BINDING_AMBIGUOUS_REASON_CODE,
)
+ if (
+ imposition.repair_context is not None
+ and imposition.repair_context.reason_code == _SCOUTED_SPINE_UNDER_BUILD_REASON_CODE
+ ):
+ _set_code_authoring_repair_context(ctx, imposition.repair_context)
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error="\n".join(imposition.violations),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(imposition.repair_context),
+ repair_context=imposition.repair_context,
+ record_repair_context_outcome=False,
+ )
return reject(
error="\n".join(imposition.violations),
user_facing_summary=_compiled_authoring_user_summary(),
@@ -8536,6 +8978,18 @@ async def _update_workflow(
repair_context=imposition.repair_context,
)
workflow_yaml = imposition.workflow_yaml
+ if imposition.substitutions is None:
+ seam_under_build = _persist_seam_spine_under_build_result(workflow_yaml, ctx)
+ if seam_under_build is not None:
+ _set_code_authoring_repair_context(ctx, seam_under_build.repair_context)
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error="\n".join(seam_under_build.violations),
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data=_code_repair_progress_data(seam_under_build.repair_context),
+ repair_context=seam_under_build.repair_context,
+ record_repair_context_outcome=False,
+ )
stripped_sandbox_imports: list[str] = []
if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
workflow_yaml, stripped_sandbox_imports = _strip_redundant_sandbox_imports_in_yaml(workflow_yaml)
@@ -8815,7 +9269,7 @@ async def _update_workflow(
)
if credential_scout_errors and code_safety_errors and code_artifact_metadata_error is None:
return reject(
- error="\n".join(credential_scout_errors),
+ error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
data={
"failure_type": "missing_credential_or_init",
@@ -8855,7 +9309,7 @@ async def _update_workflow(
)
_record_code_authoring_guardrail_reject(ctx, defer_churn_stop=True)
return reject(
- error="\n".join(credential_scout_errors),
+ error=_credential_scout_reject_error_text(ctx, credential_scout_errors),
user_facing_summary=CREDENTIAL_SCOUT_VERIFY_REPLY,
data={"failure_type": "missing_credential_or_init"},
)
@@ -9174,6 +9628,7 @@ async def _update_workflow(
ctx.staged_workflow = workflow
ctx.has_staged_proposal = True
ctx.workflow_yaml = workflow_yaml
+ ctx.persisted_draft_browser_calls = _workflow_yaml_browser_call_pairs(workflow_yaml)
ctx.code_authoring_guardrail_reject_count = 0
ctx.last_code_authoring_reject_was_credential_priority = False
_clear_code_authoring_repair_context(ctx)
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 2c48b6176..55a5858e1 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -13,8 +13,10 @@ from unittest.mock import AsyncMock
import pytest
import yaml
+from structlog.testing import capture_logs
from skyvern.forge.sdk.copilot import agent as agent_module
+from skyvern.forge.sdk.copilot import enforcement as enforcement_module
from skyvern.forge.sdk.copilot.blocker_signal import (
CREDENTIAL_SCOUT_VERIFY_REPLY,
CopilotToolBlockerSignal,
@@ -27,7 +29,9 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
authored_block_signatures_from_workflow,
authored_structure_signature_from_workflow,
latest_recorded_build_test_outcome_repeated,
+ record_build_test_outcome,
recorded_outcome_from_author_time_reject,
+ recorded_outcome_from_authoring_repair_context,
recorded_outcome_from_run_blocks_result,
)
from skyvern.forge.sdk.copilot.code_block_preflight import (
@@ -37,13 +41,14 @@ from skyvern.forge.sdk.copilot.code_block_preflight import (
)
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError
from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ SynthesisDiagnostics,
SynthesizedCodeBlock,
_get_by_role_expr,
_get_by_role_expr_strict,
)
from skyvern.forge.sdk.copilot.completion_verification import CompletionVerificationResult, CriterionVerdict
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext
+from skyvern.forge.sdk.copilot.context import CodeAuthoringRepairContext, CopilotContext, FillCarry
from skyvern.forge.sdk.copilot.enforcement import (
MAX_CODE_AUTHORING_GUARDRAIL_REJECTS,
MAX_CREDENTIAL_PRIORITY_AUTHORING_REJECTS,
@@ -70,7 +75,14 @@ from skyvern.forge.sdk.copilot.tools.workflow_update import (
_OutputContractEvaluation,
_strip_redundant_sandbox_imports_in_yaml,
)
-from skyvern.forge.sdk.copilot.turn_halt import CopilotTurnHalt, TurnHaltKind, _kind_for_blocker_signal
+from skyvern.forge.sdk.copilot.turn_halt import (
+ CopilotTurnHalt,
+ TurnHalt,
+ TurnHaltKind,
+ TurnHaltVerdict,
+ _kind_for_blocker_signal,
+ stash_repair_ceiling_turn_halt,
+)
from skyvern.forge.sdk.copilot.workflow_credential_utils import parse_workflow_yaml, workflow_blocks
from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
from skyvern.forge.sdk.workflow.models.block import CodeBlock
@@ -6702,11 +6714,35 @@ def _fake_spine_synthesized(
parameters: list[dict[str, str]] | None = None,
steps: list[dict[str, int]] | None = None,
code: str | None = None,
+ diagnostics: SynthesisDiagnostics | None = None,
) -> SynthesizedCodeBlock:
return SynthesizedCodeBlock(
code=code if code is not None else _SPINE_SYNTH_CODE,
parameters=parameters if parameters is not None else [],
steps=steps if steps is not None else [{"line_start": 1, "line_end": 1}, {"line_start": 2, "line_end": 2}],
+ diagnostics=diagnostics if diagnostics is not None else SynthesisDiagnostics(),
+ )
+
+
+def _spine_emission_diagnostics() -> SynthesisDiagnostics:
+ return SynthesisDiagnostics(
+ emitted_interaction_count=2,
+ emitted_interactions=[
+ {
+ "trajectory_index": 0,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-a",
+ "locator": 'page.locator("#stage-a")',
+ },
+ {
+ "trajectory_index": 1,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-b",
+ "locator": 'page.locator("#stage-b")',
+ },
+ ],
)
@@ -7143,7 +7179,8 @@ class TestSeparatedSpineImpositionRunEligibility:
assert any("unscouted browser action" in violation for violation in result.violations)
assert any("#hallucinated" in violation for violation in result.violations)
- assert not any("#stage-a" in violation for violation in result.violations)
+ flagged_actions = [violation.split(" Provenance: ")[0] for violation in result.violations]
+ assert not any("#stage-a" in flagged for flagged in flagged_actions)
def test_whole_trajectory_validation_exempts_spine_covered_sibling_mutations(self) -> None:
parsed = parse_workflow_yaml(_already_split_spine_yaml())
@@ -12699,3 +12736,1072 @@ class TestDeclarationContractStamp:
assert applied is False
assert new_yaml == workflow_yaml
+
+
+def _gate_blocks(sibling_code: str) -> list[dict[str, object]]:
+ return [
+ {"block_type": "code", "label": "sibling_stage", "code": sibling_code},
+ {"block_type": "code", "label": "extract_record", "code": 'return {"output": {}}\n'},
+ ]
+
+
+def _gate_validation(sibling_code: str, diagnostics: SynthesisDiagnostics, synthesized_code: str | None = None):
+ blocks = _gate_blocks(sibling_code)
+ return workflow_update_module._whole_trajectory_browser_surface_violations(
+ code_blocks=blocks,
+ selected_code_block=blocks[1],
+ submitted_selected_code=str(blocks[1]["code"]),
+ synthesized_code=synthesized_code if synthesized_code is not None else _SPINE_SYNTH_CODE,
+ synthesized_diagnostics=diagnostics,
+ )
+
+
+class TestBrowserSurfaceRejectionProvenance:
+ def test_never_captured_mutation_rejected_with_rescout_move(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ workflow_update_module,
+ "synthesize_code_block",
+ lambda *a, **k: _fake_spine_synthesized(diagnostics=_spine_emission_diagnostics()),
+ )
+ ctx = _imposition_split_ctx()
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(
+ _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
+ )
+
+ assert any("unscouted browser action" in violation for violation in result.violations)
+ assert any(
+ "never_captured" in violation and "re-scout that step" in violation for violation in result.violations
+ )
+ events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
+ assert len(events) == 1
+ assert events[0]["kind"] == "never_captured"
+ assert events[0]["site"] == "whole_trajectory"
+ assert "#hallucinated" in events[0]["action"]
+
+ def test_same_receiver_divergent_call_shape_is_shape_diverged_with_nearest(self) -> None:
+ validation = _gate_validation(
+ 'await page.locator("#stage-a").click(timeout=5000)\n', _spine_emission_diagnostics()
+ )
+
+ assert len(validation.provenance) == 1
+ provenance = validation.provenance[0]
+ assert provenance.kind == "shape_diverged"
+ assert provenance.divergence_source == "synthesized"
+ assert provenance.nearest_receiver == "page.locator('#stage-a')"
+ assert provenance.nearest_method == "click"
+ assert provenance.nearest_selector == "#stage-a"
+ assert any(
+ "shape_diverged (synthesized)" in violation and "captured selector '#stage-a'" in violation
+ for violation in validation.violations
+ )
+
+ def test_trajectory_dropped_rung_is_not_never_captured(self) -> None:
+ diagnostics = _spine_emission_diagnostics()
+ diagnostics.dropped_interactions.append(
+ {"trajectory_index": 2, "tool_name": "click", "selector": "#gone", "reason_code": "ambiguous_bare_selector"}
+ )
+
+ validation = _gate_validation('await page.locator("#gone").click()\n', diagnostics)
+
+ assert len(validation.provenance) == 1
+ provenance = validation.provenance[0]
+ assert provenance.kind == "shape_diverged"
+ assert provenance.divergence_source == "trajectory_dropped"
+ assert provenance.nearest_selector == "#gone"
+ assert not any("never_captured" in violation for violation in validation.violations)
+
+ def test_locator_form_divergence_matches_emitted_record(self) -> None:
+ diagnostics = SynthesisDiagnostics(
+ emitted_interaction_count=1,
+ emitted_interactions=[
+ {
+ "trajectory_index": 0,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#go",
+ "locator": 'page.get_by_role("button", name="Go")',
+ }
+ ],
+ )
+
+ validation = _gate_validation(
+ 'await page.locator("#go").click()\n',
+ diagnostics,
+ synthesized_code='await page.get_by_role("button", name="Go").click()',
+ )
+
+ assert len(validation.provenance) == 1
+ provenance = validation.provenance[0]
+ assert provenance.kind == "shape_diverged"
+ assert provenance.divergence_source == "synthesized"
+ assert provenance.nearest_receiver == 'page.get_by_role("button", name="Go")'
+ assert provenance.nearest_selector == "#go"
+ assert not any("never_captured" in violation for violation in validation.violations)
+
+ def test_ambiguous_alias_mutation_names_rewrite_move(self) -> None:
+ validation = _gate_validation(
+ 'do_click = page.locator("#x").click\nawait do_click()\n', _spine_emission_diagnostics()
+ )
+
+ assert len(validation.provenance) == 1
+ provenance = validation.provenance[0]
+ assert provenance.kind == "ambiguous"
+ assert provenance.nearest_method is None
+ assert provenance.nearest_receiver is None
+ assert provenance.nearest_selector is None
+ assert provenance.divergence_source is None
+ assert any(
+ "ambiguous browser action" in violation and "rewrite it as a direct page/locator call" in violation
+ for violation in validation.violations
+ )
+
+ def test_extraction_suffix_unscouted_mutation_carries_provenance(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = (
+ textwrap.dedent(synthesized.code).rstrip() + '\nawait page.locator("#electricDate").fill("2026-07-01")\n'
+ )
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any(
+ "extraction suffix contains unscouted browser action" in violation for violation in result.violations
+ )
+ events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
+ assert len(events) == 1
+ assert events[0]["site"] == "extraction_suffix"
+ assert events[0]["kind"] == "never_captured"
+
+ def test_extraction_suffix_exact_duplicate_is_suffix_disallowed(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted_code = textwrap.dedent(synthesized.code).rstrip() + '\nawait page.locator("#coverage-next").click()\n'
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(submitted_code, " " * 18)}
+ """
+ )
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any(
+ "suffix_disallowed" in violation and "remove the duplicate" in violation for violation in result.violations
+ )
+ events = [log for log in logs if log["event"] == "copilot_browser_surface_rejection_provenance"]
+ assert len(events) == 1
+ assert events[0]["kind"] == "suffix_disallowed"
+ assert events[0]["divergence_source"] is None
+
+ def test_never_captured_still_rejects_with_empty_diagnostics(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(workflow_update_module, "synthesize_code_block", lambda *a, **k: _fake_spine_synthesized())
+ ctx = _imposition_split_ctx()
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(
+ _already_split_spine_yaml(extra_sibling_code='await page.locator("#hallucinated").click()'), ctx
+ )
+
+ assert any("unscouted browser action" in violation for violation in result.violations)
+ assert any("never_captured" in violation for violation in result.violations)
+ assert result.substitutions is None
+
+
+class TestSeparatedSpineFastPathRecord:
+ def test_set_equality_pass_unchanged_and_duplicate_loss_recorded(self) -> None:
+ blocks = [
+ {"block_type": "code", "label": "s1", "code": 'await page.locator("#a").click()\n'},
+ {"block_type": "code", "label": "s2", "code": 'await page.locator("#b").click()\n'},
+ {"block_type": "code", "label": "extract", "code": 'return {"output": {}}\n'},
+ ]
+ synthesized_code = (
+ 'await page.locator("#a").click()\nawait page.locator("#b").click()\nawait page.locator("#a").click()'
+ )
+
+ with capture_logs() as logs:
+ already_imposed = workflow_update_module._separated_spine_already_imposed(
+ blocks, blocks[2], synthesized_code
+ )
+
+ assert already_imposed is True
+ events = [log for log in logs if log["event"] == "copilot_separated_spine_fast_path"]
+ assert len(events) == 1
+ assert events[0]["spine_coverage"] == "set_equality"
+ assert events[0]["synthesized_mutation_count"] == 3
+ assert events[0]["sibling_signature_count"] == 2
+ assert events[0]["duplicate_rungs_lost"] is True
+
+
+class TestImpositionSkippedAfterUpdateRecord:
+ def test_post_update_early_return_emits_skip_record(self) -> None:
+ ctx = _imposition_split_ctx()
+ ctx.update_workflow_called = True
+ workflow_yaml = _already_split_spine_yaml()
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
+
+ assert result.violations == []
+ assert result.workflow_yaml == workflow_yaml
+ events = [log for log in logs if log["event"] == "copilot_imposition_skipped_after_update"]
+ assert len(events) == 1
+ assert events[0]["trajectory_length"] == len(ctx.scout_trajectory)
+ assert events[0]["reopen_download_target"] is False
+ assert events[0]["reopen_persistence_after_failed_run"] is False
+ assert events[0]["reopen_author_time_reject"] is False
+
+
+class TestScoutCaptureParityAccounting:
+ def test_unresolvable_selector_bail_emits_capture_loss(self) -> None:
+ ctx = _code_only_ctx()
+ before = list(ctx.scouted_interactions)
+
+ with capture_logs() as logs:
+ scouting_module._record_scouted_interaction(
+ ctx, tool_name="click", selector="", source_url="https://example.com/step"
+ )
+
+ assert ctx.scouted_interactions == before
+ events = [log for log in logs if log["event"] == "copilot_scout_capture_loss"]
+ assert len(events) == 1
+ assert events[0]["tool_name"] == "click"
+ assert events[0]["reason"] == "unresolvable_selector"
+ assert events[0]["url"] == "https://example.com/step"
+
+ def test_cap_eviction_emits_per_collection_records(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scouted_interactions = []
+ ctx.scout_trajectory = []
+ for index in range(scouting_module._MAX_SCOUTED_INTERACTIONS):
+ scouting_module._record_scouted_interaction(
+ ctx, tool_name="click", selector=f"#item-{index}", source_url="https://example.com/list"
+ )
+
+ with capture_logs() as logs:
+ scouting_module._record_scouted_interaction(
+ ctx, tool_name="click", selector="#item-overflow", source_url="https://example.com/list"
+ )
+
+ events = [log for log in logs if log["event"] == "copilot_scout_interaction_evicted"]
+ by_collection = {event["collection"]: event for event in events}
+ assert set(by_collection) == {"scout_trajectory", "scouted_interactions"}
+ assert by_collection["scout_trajectory"]["trajectory_index"] == 0
+ assert "trajectory_index" not in by_collection["scouted_interactions"]
+ assert by_collection["scouted_interactions"]["selector"] == "#item-0"
+ assert len(ctx.scouted_interactions) == scouting_module._MAX_SCOUTED_INTERACTIONS
+ assert len(ctx.scout_trajectory) == scouting_module._MAX_SCOUTED_INTERACTIONS
+ assert ctx.scouted_interactions[0]["selector"] == "#item-1"
+
+ def test_dedup_replacement_is_not_an_eviction(self) -> None:
+ ctx = _code_only_ctx()
+ ctx.scouted_interactions = []
+ ctx.scout_trajectory = []
+ for index in range(scouting_module._MAX_SCOUTED_INTERACTIONS):
+ scouting_module._record_scouted_interaction(
+ ctx, tool_name="click", selector=f"#item-{index}", source_url="https://example.com/list"
+ )
+
+ with capture_logs() as logs:
+ scouting_module._record_scouted_interaction(
+ ctx, tool_name="click", selector="#item-5", source_url="https://example.com/list"
+ )
+
+ events = [log for log in logs if log["event"] == "copilot_scout_interaction_evicted"]
+ assert all(event["collection"] == "scout_trajectory" for event in events)
+ assert len(ctx.scouted_interactions) == scouting_module._MAX_SCOUTED_INTERACTIONS
+
+ @pytest.mark.asyncio
+ async def test_fill_carry_rebind_eviction_goes_through_shared_accounting(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ monkeypatch.setattr(scouting_module, "_fill_carry_validation_failure", AsyncMock(return_value=None))
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = [
+ {"tool_name": "click", "selector": f"#item-{index}", "trajectory_index": index}
+ for index in range(scouting_module._MAX_SCOUTED_INTERACTIONS)
+ ]
+ ctx.prior_fill_carry = [
+ FillCarry(
+ tool_name="type_text",
+ selector="#carried",
+ source_url="https://example.com/form",
+ typed_value="abc",
+ ).model_dump()
+ ]
+
+ with capture_logs() as logs:
+ await scouting_module._maybe_rebind_prior_fill_carry(
+ ctx, page_evidence={"current_url": "https://example.com/form"}, url="https://example.com/form"
+ )
+
+ events = [log for log in logs if log["event"] == "copilot_scout_interaction_evicted"]
+ assert len(events) == 1
+ assert events[0]["collection"] == "scout_trajectory"
+ assert events[0]["trajectory_index"] == 0
+ assert len(ctx.scout_trajectory) == scouting_module._MAX_SCOUTED_INTERACTIONS
+ assert ctx.scout_trajectory[-1]["selector"] == "#carried"
+ assert ctx.scout_trajectory[-1]["carried"] is True
+
+
+def _under_build_draft_yaml() -> str:
+ return _yaml(
+ f"""
+ title: Entry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#stage-a").click()
+ """
+ )
+
+
+def _drifted_spine_synthesized(diagnostics: SynthesisDiagnostics | None = None) -> SynthesizedCodeBlock:
+ return _fake_spine_synthesized(
+ code='await page.locator("#stage-a").click()',
+ diagnostics=diagnostics if diagnostics is not None else _spine_emission_diagnostics(),
+ )
+
+
+class TestScoutedSpineUnderBuild:
+ def test_browser_surface_mutations_are_source_ordered_across_nesting(self) -> None:
+ # A rung nested inside an `if` appears earlier in source than a later top-level rung.
+ # ast.walk enumerates breadth-first, so without a source-order sort the nested call would
+ # be reported after the top-level one and the ordered-subsequence coverage scan would
+ # falsely flag a present rung as uncovered.
+ code = textwrap.dedent(
+ """
+ page = ctx.page
+ if ctx.needs_consent:
+ page.get_by_role("button", name="Alpha").click()
+ page.get_by_role("link", name="Bravo").click()
+ """
+ )
+ direct_mutations, _unscouted, _ambiguous = workflow_update_module._browser_surface_for_code(code)
+ shapes = [mutation.call_shape for mutation in direct_mutations]
+ alpha_index = next(i for i, shape in enumerate(shapes) if "Alpha" in shape)
+ bravo_index = next(i for i, shape in enumerate(shapes) if "Bravo" in shape)
+ assert alpha_index < bravo_index
+
+ def test_under_build_draft_rejected_with_pass_route(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ workflow_update_module, "synthesize_code_block", lambda *a, **k: _drifted_spine_synthesized()
+ )
+ ctx = _quote_ctx()
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_under_build_draft_yaml(), ctx)
+
+ assert any("scouted_spine_under_build" in violation for violation in result.violations)
+ assert any("#stage-b" in violation for violation in result.violations)
+ assert any("remaining synthesized rungs" in violation for violation in result.violations)
+ assert all("fill_credential_field" not in violation for violation in result.violations)
+ assert result.repair_context is not None
+ assert result.repair_context.reason_code == "scouted_spine_under_build"
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["required_rung_count"] == 2
+ assert events[0]["covered_rung_count"] == 1
+
+ def test_lane_flagged_emissions_do_not_trigger_under_build(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ diagnostics = SynthesisDiagnostics(
+ emitted_interaction_count=4,
+ emitted_interactions=[
+ {
+ "trajectory_index": 0,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-a",
+ "locator": 'page.locator("#stage-a")',
+ },
+ {
+ "trajectory_index": 1,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#dismiss",
+ "locator": 'page.locator("#dismiss")',
+ "lane": "optional_dismissal",
+ },
+ {
+ "trajectory_index": 2,
+ "tool_name": "type_text",
+ "method": "input_value",
+ "selector": "#readonly-field",
+ "locator": 'page.locator("#readonly-field")',
+ "lane": "readonly_skip",
+ },
+ {
+ "trajectory_index": 3,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#opener",
+ "locator": 'page.locator("#opener")',
+ "lane": "entry_recovery",
+ },
+ ],
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "synthesize_code_block",
+ lambda *a, **k: _drifted_spine_synthesized(diagnostics),
+ )
+ ctx = _quote_ctx()
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_under_build_draft_yaml(), ctx)
+
+ assert result.violations == []
+ assert result.substitutions is not None
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+
+ def test_forgiven_prefix_interactions_do_not_trigger_under_build(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ diagnostics = SynthesisDiagnostics(
+ emitted_interaction_count=1,
+ emitted_interactions=[
+ {
+ "trajectory_index": 1,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-a",
+ "locator": 'page.locator("#stage-a")',
+ }
+ ],
+ forgiven_interactions=[{"trajectory_index": 0, "tool_name": "click", "lane": "entry_replay_prefix"}],
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "synthesize_code_block",
+ lambda *a, **k: _drifted_spine_synthesized(diagnostics),
+ )
+ ctx = _quote_ctx()
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(_under_build_draft_yaml(), ctx)
+
+ assert result.violations == []
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+
+ def test_full_spine_draft_from_real_generator_does_not_fire(self) -> None:
+ ctx = _quote_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+ submitted = _yaml(
+ f"""
+ title: Quote
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{textwrap.indent(textwrap.dedent(synthesized.code).strip(), " " * 18)}
+ """
+ )
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert result.violations == []
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+
+ @pytest.mark.asyncio
+ async def test_under_build_rejects_climb_churn_counter_to_stop(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ workflow_update_module, "synthesize_code_block", lambda *a, **k: _drifted_spine_synthesized()
+ )
+ ctx = _quote_ctx()
+ workflow_yaml = _under_build_draft_yaml()
+
+ for index in range(MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1):
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
+ assert result["ok"] is False
+ assert "scouted_spine_under_build" in result["error"]
+ assert ctx.code_authoring_guardrail_reject_count == index + 1
+ assert ctx.blocker_signal is None
+
+ final = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx)
+
+ assert final["ok"] is False
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ held = ctx.blocker_signal
+ assert isinstance(held, CopilotToolBlockerSignal)
+ assert held.internal_reason_code == "code_authoring_guardrail_churn"
+
+
+def _records_spine_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#stage-a",
+ "source_url": "https://example.com/records",
+ "trajectory_index": 0,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#stage-b",
+ "source_url": "https://example.com/records",
+ "trajectory_index": 1,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#stage-c",
+ "source_url": "https://example.com/records",
+ "trajectory_index": 2,
+ },
+ ]
+ return ctx
+
+
+def _records_block_yaml(code_body: str) -> str:
+ indented = textwrap.indent(textwrap.dedent(code_body).strip(), " " * 14)
+ return _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+{indented}
+ """
+ )
+
+
+def _checkpoint_eligible_ctx() -> CopilotContext:
+ ctx = _records_spine_ctx()
+ ctx.update_workflow_called = True
+ ctx.persisted_draft_browser_calls = [("click", 'page.locator("#stage-a")')]
+ return ctx
+
+
+def _credential_spine_ctx() -> CopilotContext:
+ ctx = _code_only_ctx()
+ _enable_imposition(ctx)
+ ctx.update_workflow_called = True
+ ctx.persisted_draft_browser_calls = [("click", 'page.locator("#stage-a")')]
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#stage-a",
+ "source_url": "https://example.com/records",
+ "trajectory_index": 0,
+ },
+ _credential_fill_interaction(
+ "username", credential_id="cred_records", source_url="https://example.com/records"
+ ),
+ _credential_fill_interaction(
+ "password", credential_id="cred_records", source_url="https://example.com/records"
+ ),
+ ]
+ return ctx
+
+
+def _credential_spine_block_yaml(synthesized: SynthesizedCodeBlock) -> str:
+ credential_parameter = next(
+ parameter for parameter in synthesized.parameters if str(parameter.get("credential_id") or "")
+ )
+ indented = textwrap.indent(textwrap.dedent(synthesized.code).strip(), " " * 14)
+ return _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: {credential_parameter["key"]}
+ default_value: {credential_parameter["credential_id"]}
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ parameter_keys:
+ - {credential_parameter["key"]}
+ code: |
+{indented}
+ """
+ )
+
+
+class TestScoutedSpinePersistSeamCoverage:
+ @pytest.mark.asyncio
+ async def test_early_partial_persist_then_stub_resubmission_rejected_and_turn_end_fires(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _records_spine_ctx()
+ full_trajectory = list(ctx.scout_trajectory)
+ ctx.scout_trajectory = full_trajectory[:2]
+ partial_synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert partial_synthesized is not None
+
+ persisted = await _update_workflow({"workflow_yaml": _records_block_yaml(partial_synthesized.code)}, ctx)
+
+ assert persisted["ok"] is True
+ assert ctx.persisted_draft_browser_calls is not None
+ assert [pair for pair in ctx.persisted_draft_browser_calls if pair[0] == "click"] == [
+ ("click", "page.locator('#stage-a')"),
+ ("click", "page.locator('#stage-b')"),
+ ]
+
+ ctx.scout_trajectory = full_trajectory
+
+ with capture_logs() as logs:
+ rejected = await _update_workflow({"workflow_yaml": _records_block_yaml(partial_synthesized.code)}, ctx)
+
+ assert rejected["ok"] is False
+ assert "scouted_spine_under_build" in rejected["error"]
+ assert 'await page.locator("#stage-c").click()' in rejected["error"]
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["site"] == "persist_seam"
+
+ with capture_logs() as logs:
+ nudge = enforcement_module._scouted_spine_turn_end_nudge(ctx)
+
+ assert nudge is not None
+ assert "scouted_spine_under_build" in nudge
+ assert 'await page.locator("#stage-c").click()' in nudge
+ turn_end_events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(turn_end_events) == 1
+ assert turn_end_events[0]["site"] == "turn_end"
+
+ @pytest.mark.asyncio
+ async def test_post_update_skip_path_persist_under_coverage_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _records_spine_ctx()
+ ctx.update_workflow_called = True
+
+ with capture_logs() as logs:
+ result = await _update_workflow(
+ {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
+ )
+
+ assert result["ok"] is False
+ assert "scouted_spine_under_build" in result["error"]
+ assert 'await page.locator("#stage-b").click()' in result["error"]
+ assert 'await page.locator("#stage-c").click()' in result["error"]
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ assert [log for log in logs if log["event"] == "copilot_imposition_skipped_after_update"]
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["site"] == "persist_seam"
+
+ @pytest.mark.asyncio
+ async def test_second_persist_dropping_all_code_blocks_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _checkpoint_eligible_ctx()
+ no_code_yaml = _yaml(
+ """
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: send_email
+ label: notify
+ recipients:
+ subject: Records ready
+ body: Records run finished
+ """
+ )
+
+ with capture_logs() as logs:
+ result = await _update_workflow({"workflow_yaml": no_code_yaml}, ctx)
+
+ assert result["ok"] is False
+ assert "scouted_spine_under_build" in result["error"]
+ assert 'await page.locator("#stage-a").click()' in result["error"]
+ assert 'await page.locator("#stage-c").click()' in result["error"]
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["site"] == "persist_seam"
+
+ def test_separated_split_branch_under_coverage_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ diagnostics = _spine_emission_diagnostics()
+ diagnostics.emitted_interactions.append(
+ {
+ "trajectory_index": 2,
+ "tool_name": "click",
+ "method": "click",
+ "selector": "#stage-c",
+ "locator": 'page.locator("#stage-c")',
+ "call_source": 'await page.locator("#stage-c").click()',
+ }
+ )
+ monkeypatch.setattr(
+ workflow_update_module,
+ "synthesize_code_block",
+ lambda *a, **k: _fake_spine_synthesized(diagnostics=diagnostics),
+ )
+ ctx = _imposition_split_ctx()
+ workflow_yaml = _records_block_yaml(
+ _SPINE_SYNTH_CODE
+ + '\nvalue = await page.locator("#result").inner_text()\nreturn {"output": {"record_id": value}}'
+ )
+
+ with capture_logs() as logs:
+ result = workflow_update_module._maybe_impose_synthesized_code_block(workflow_yaml, ctx)
+
+ assert any("scouted_spine_under_build" in violation for violation in result.violations)
+ assert any('await page.locator("#stage-c").click()' in violation for violation in result.violations)
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["site"] == "separated_split"
+
+ @pytest.mark.asyncio
+ async def test_verbatim_synthesized_resubmission_clears_coverage_and_persists(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _records_spine_ctx()
+ ctx.update_workflow_called = True
+ ctx.persisted_draft_browser_calls = [("click", 'page.locator("#stage-a")')]
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+
+ with capture_logs() as logs:
+ result = await _update_workflow({"workflow_yaml": _records_block_yaml(synthesized.code)}, ctx)
+
+ assert result["ok"] is True
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert ctx.persisted_draft_browser_calls is not None
+ assert [pair for pair in ctx.persisted_draft_browser_calls if pair[0] == "click"] == [
+ ("click", "page.locator('#stage-a')"),
+ ("click", "page.locator('#stage-b')"),
+ ("click", "page.locator('#stage-c')"),
+ ]
+ assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None
+
+ @pytest.mark.asyncio
+ async def test_composite_route_resubmission_passes_credential_gate_and_persists(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ monkeypatch.setattr(
+ workflow_update_module, "_credential_reference_validation_error", AsyncMock(return_value=None)
+ )
+ ctx = _credential_spine_ctx()
+
+ with capture_logs() as logs:
+ rejected = await _update_workflow(
+ {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
+ )
+
+ assert rejected["ok"] is False
+ assert "scouted_spine_under_build" in rejected["error"]
+ assert "fill_credential_field" in rejected["error"]
+ assert "click the submit control or press Enter" in rejected["error"]
+ assert "Missing rung source to reuse verbatim" in rejected["error"]
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["site"] == "persist_seam"
+ assert events[0]["credential_scout_precondition_pending"] is True
+
+ ctx.scout_trajectory.append(_submit_interaction(source_url="https://example.com/records"))
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+
+ with capture_logs() as logs:
+ accepted = await _update_workflow({"workflow_yaml": _credential_spine_block_yaml(synthesized)}, ctx)
+
+ assert accepted["ok"] is True
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None
+
+ @pytest.mark.asyncio
+ async def test_satisfied_credential_precondition_keeps_single_step_route(self) -> None:
+ ctx = _credential_spine_ctx()
+ ctx.scout_trajectory.append(_submit_interaction(source_url="https://example.com/records"))
+
+ with capture_logs() as logs:
+ rejected = await _update_workflow(
+ {"workflow_yaml": _records_block_yaml('await page.locator("#stage-a").click()')}, ctx
+ )
+
+ assert rejected["ok"] is False
+ assert "scouted_spine_under_build" in rejected["error"]
+ assert "fill_credential_field" not in rejected["error"]
+ assert "Missing rung source to reuse verbatim" in rejected["error"]
+ events = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build"]
+ assert len(events) == 1
+ assert events[0]["credential_scout_precondition_pending"] is False
+
+ @pytest.mark.asyncio
+ async def test_credential_scout_reject_carries_open_obligation_artifact(self) -> None:
+ ctx = _credential_spine_ctx()
+ synthesized = workflow_update_module.synthesize_code_block(ctx.scout_trajectory, strict_selectors=True)
+ assert synthesized is not None
+
+ result = await _update_workflow({"workflow_yaml": _credential_spine_block_yaml(synthesized)}, ctx)
+
+ assert result["ok"] is False
+ assert result["data"]["failure_type"] == "missing_credential_or_init"
+ assert "later submit action on the same page" in result["error"]
+ assert "The persisted draft is missing scouted rung(s)." in result["error"]
+ assert "Missing rung source to reuse verbatim" in result["error"]
+ assert ".password" in result["error"]
+
+
+class TestScoutedSpineTurnEndCheckpoint:
+ def test_checkpoint_is_single_shot_and_emits_unresolved_when_spent(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+
+ first = enforcement_module._scouted_spine_turn_end_nudge(ctx)
+ assert first is not None
+ assert "scouted_spine_under_build" in first
+ assert 'await page.locator("#stage-b").click()' in first
+ assert ctx.code_authoring_guardrail_reject_count == 1
+
+ with capture_logs() as logs:
+ second = enforcement_module._scouted_spine_turn_end_nudge(ctx)
+
+ assert second is None
+ assert ctx.code_authoring_guardrail_reject_count == 1
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "turn_end"
+
+ def test_checkpoint_reject_threads_churn_and_ceiling_emits_unresolved(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ seed_repair = CodeAuthoringRepairContext(
+ block_label="persisted_draft",
+ reason_code="scouted_spine_under_build",
+ selector="#stage-b",
+ )
+ record_build_test_outcome(ctx, recorded_outcome_from_authoring_repair_context(seed_repair))
+ ctx.code_authoring_guardrail_reject_count = MAX_CODE_AUTHORING_GUARDRAIL_REJECTS - 1
+
+ with capture_logs() as logs:
+ nudge = enforcement_module._scouted_spine_turn_end_nudge(ctx)
+
+ assert nudge is not None
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+ held = ctx.blocker_signal
+ assert isinstance(held, CopilotToolBlockerSignal)
+ assert held.internal_reason_code == "code_authoring_guardrail_churn"
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "churn_stop"
+
+ assert enforcement_module._scouted_spine_turn_end_nudge(ctx) is None
+ assert ctx.code_authoring_guardrail_reject_count == MAX_CODE_AUTHORING_GUARDRAIL_REJECTS
+
+ def test_no_spurious_checkpoint_on_full_coverage_or_lane_only_remainder(self) -> None:
+ full = _checkpoint_eligible_ctx()
+ full.persisted_draft_browser_calls = [
+ ("click", 'page.locator("#stage-a")'),
+ ("click", 'page.locator("#stage-b")'),
+ ("click", 'page.locator("#stage-c")'),
+ ]
+ with capture_logs() as logs:
+ assert enforcement_module._scouted_spine_turn_end_nudge(full) is None
+ assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]
+ assert full.code_authoring_guardrail_reject_count == 0
+
+ lane_only = _checkpoint_eligible_ctx()
+ lane_only.scout_trajectory = lane_only.scout_trajectory[:1] + [
+ {
+ "tool_name": "click",
+ "selector": "#cookie-accept",
+ "role": "button",
+ "accessible_name": "Accept all cookies",
+ "source_url": "https://example.com/records",
+ "trajectory_index": 1,
+ }
+ ]
+ with capture_logs() as logs:
+ assert enforcement_module._scouted_spine_turn_end_nudge(lane_only) is None
+ assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]
+
+ no_persist = _records_spine_ctx()
+ assert enforcement_module._scouted_spine_turn_end_nudge(no_persist) is None
+
+ standard = _checkpoint_eligible_ctx()
+ standard.block_authoring_policy = BlockAuthoringPolicy.STANDARD
+ assert enforcement_module._scouted_spine_turn_end_nudge(standard) is None
+
+
+def _full_coverage_calls() -> list[tuple[str, str]]:
+ return [
+ ("click", 'page.locator("#stage-a")'),
+ ("click", 'page.locator("#stage-b")'),
+ ("click", 'page.locator("#stage-c")'),
+ ]
+
+
+class TestScoutedSpineTurnHaltExit:
+ def test_repair_ceiling_halt_with_open_obligation_emits_unresolved_and_reframes_reply(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ ctx.last_test_anti_bot = "challenge_detected"
+ signal = enforcement_module.repair_ceiling_stop_signal(ctx, None)
+ assert "verification challenge" in signal.user_facing_reason
+ ctx.blocker_signal = signal
+ halt = TurnHalt(kind=TurnHaltKind.REPAIR_CEILING_REACHED, blocker_signal=signal)
+
+ with capture_logs() as logs:
+ result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "turn_halt"
+ assert result.user_response == enforcement_module.SCOUTED_SPINE_TURN_HALT_USER_REASON
+ assert "verification challenge" not in result.user_response
+
+ def test_site_block_halt_with_open_obligation_emits_unresolved_and_keeps_site_block_reply(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ signal = enforcement_module._probable_site_block_stop_signal(ctx)
+ ctx.blocker_signal = signal
+ halt = TurnHalt(kind=TurnHaltKind.PROBABLE_SITE_BLOCK, blocker_signal=signal)
+
+ with capture_logs() as logs:
+ result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "turn_halt"
+ assert result.user_response == signal.user_facing_reason
+
+ def test_delivered_unverified_halt_with_open_obligation_emits_unresolved(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ halt = TurnHalt(
+ kind=TurnHaltKind.DELIVERED_UNVERIFIED,
+ verdict=TurnHaltVerdict.DELIVERED_UNVERIFIED,
+ )
+
+ with capture_logs() as logs:
+ agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "turn_halt"
+
+ def test_halt_without_open_obligation_emits_nothing_and_keeps_reply(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ ctx.persisted_draft_browser_calls = _full_coverage_calls()
+ signal = enforcement_module.repair_ceiling_stop_signal(ctx, None)
+ ctx.blocker_signal = signal
+ halt = TurnHalt(kind=TurnHaltKind.REPAIR_CEILING_REACHED, blocker_signal=signal)
+
+ with capture_logs() as logs:
+ result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ assert not [log for log in logs if "scouted_spine" in str(log.get("event"))]
+ assert result.user_response == signal.user_facing_reason
+
+ @pytest.mark.asyncio
+ async def test_wrapped_exception_exit_with_stashed_repair_ceiling_halt_emits_unresolved(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ signal = enforcement_module.repair_ceiling_stop_signal(ctx, None)
+ stash_repair_ceiling_turn_halt(ctx, signal, consecutive_identical_repair_count=3)
+ ctx.blocker_signal = signal
+
+ with capture_logs() as logs:
+ result = await agent_module._resolve_wrapped_exception_exit_result(
+ ctx,
+ None,
+ goal_satisfied=False,
+ error=RuntimeError("wrapped"),
+ workflow_permanent_id="wp",
+ )
+
+ unresolved = [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert len(unresolved) == 1
+ assert unresolved[0]["site"] == "turn_halt"
+ assert result.user_response == enforcement_module.SCOUTED_SPINE_TURN_HALT_USER_REASON
+
+ def test_obligation_check_failure_never_blocks_the_halt_reply(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ signal = enforcement_module.repair_ceiling_stop_signal(ctx, None)
+ ctx.blocker_signal = signal
+ halt = TurnHalt(kind=TurnHaltKind.REPAIR_CEILING_REACHED, blocker_signal=signal)
+
+ def _boom(*args: object, **kwargs: object) -> None:
+ raise RuntimeError("synthesis unavailable")
+
+ monkeypatch.setattr(enforcement_module, "synthesize_code_block", _boom)
+
+ with capture_logs() as logs:
+ result = agent_module._build_turn_halt_exit_result(ctx, global_llm_context=None, halt=halt)
+
+ assert not [log for log in logs if log["event"] == "copilot_scouted_spine_under_build_unresolved"]
+ assert result.user_response == signal.user_facing_reason
+
+
+class TestAmbiguousRejectCarriesOpenObligationArtifact:
+ def test_ambiguous_sibling_reject_with_open_obligation_carries_call_source(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ record_build_test_outcome(ctx, _author_time_reject_outcome("metadata_reject"))
+ submitted = _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#stage-a").click()
+ - block_type: code
+ label: helper_stage
+ code: |
+ opener = page.locator("#stage-b").click
+ await opener()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any("ambiguous browser action" in violation for violation in result.violations)
+ assert any('await page.locator("#stage-b").click()' in violation for violation in result.violations)
+ assert any('await page.locator("#stage-c").click()' in violation for violation in result.violations)
+
+ def test_ambiguous_reject_without_open_obligation_stays_bare(self) -> None:
+ ctx = _checkpoint_eligible_ctx()
+ ctx.persisted_draft_browser_calls = [
+ ("click", 'page.locator("#stage-a")'),
+ ("click", 'page.locator("#stage-b")'),
+ ("click", 'page.locator("#stage-c")'),
+ ]
+ record_build_test_outcome(ctx, _author_time_reject_outcome("metadata_reject"))
+ submitted = _yaml(
+ f"""
+ title: Records
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {workflow_update_module._SYNTHESIZED_BLOCK_LABEL}
+ code: |
+ await page.locator("#stage-a").click()
+ - block_type: code
+ label: helper_stage
+ code: |
+ opener = page.locator("#stage-b").click
+ await opener()
+ """
+ )
+
+ result = workflow_update_module._maybe_impose_synthesized_code_block(submitted, ctx)
+
+ assert any("ambiguous browser action" in violation for violation in result.violations)
+ assert not any("reuse verbatim" in violation for violation in result.violations)
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 1a4c8c708..520de8b65 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -33,6 +33,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
is_optional_dismissal_only_trajectory,
render_synthesized_offer_text,
synthesize_code_block,
+ uncovered_required_emitted_interactions,
)
from skyvern.forge.sdk.copilot.context import (
FillCarry,
@@ -2615,3 +2616,191 @@ class TestReadonlyControlStateCarry:
persisted = StructuredContext(fill_carry=carry).to_json_str()
assert '"control_readonly": true' in persisted
assert '"control_value_satisfied": false' in persisted
+
+
+class TestEmittedInteractionPartition:
+ def _mixed_trajectory(self) -> list[dict[str, Any]]:
+ return [
+ {"tool_name": "click", "selector": "#open", "source_url": "https://example.com/start"},
+ {
+ "tool_name": "type_text",
+ "selector": "#name",
+ "typed_value": "Ada",
+ "role": "textbox",
+ "accessible_name": "Name",
+ },
+ {"tool_name": "hover", "selector": "#menu"},
+ {"tool_name": "select_option", "selector": "#state"},
+ {
+ "tool_name": "type_text",
+ "selector": "#locked",
+ "typed_value": "fixed",
+ "control_readonly": True,
+ "control_value_satisfied": True,
+ },
+ {
+ "tool_name": "click",
+ "selector": "#accept",
+ "role": "button",
+ "accessible_name": "Accept all cookies",
+ },
+ {"tool_name": "press_key", "selector": "#name", "key": "Enter"},
+ ]
+
+ def test_every_retained_index_in_exactly_one_partition(self) -> None:
+ trajectory = self._mixed_trajectory()
+
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert result is not None
+ diagnostics = result.diagnostics
+ assert diagnostics.truncated is False
+ emitted = {record["trajectory_index"] for record in diagnostics.emitted_interactions}
+ dropped = {record["trajectory_index"] for record in diagnostics.dropped_interactions}
+ forgiven = {record["trajectory_index"] for record in diagnostics.forgiven_interactions}
+ assert emitted | dropped | forgiven == set(range(len(trajectory)))
+ assert emitted & dropped == set()
+ assert emitted & forgiven == set()
+ assert dropped & forgiven == set()
+
+ def test_emitted_records_carry_method_selector_and_lane_flags(self) -> None:
+ trajectory = self._mixed_trajectory()
+
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert result is not None
+ by_index = {record["trajectory_index"]: record for record in result.diagnostics.emitted_interactions}
+ assert by_index[0]["method"] == "click"
+ assert by_index[0]["selector"] == "#open"
+ assert "lane" not in by_index[0]
+ assert by_index[1]["method"] == "fill"
+ assert by_index[4]["method"] == "input_value"
+ assert by_index[4]["lane"] == "readonly_skip"
+ assert by_index[5]["method"] == "click"
+ assert by_index[5]["lane"] == "optional_dismissal"
+ assert by_index[6]["method"] == "press"
+ dropped_reasons = {
+ record["trajectory_index"]: record["reason_code"] for record in result.diagnostics.dropped_interactions
+ }
+ assert dropped_reasons[2] == "unsupported_tool"
+ assert dropped_reasons[3] == "missing_value"
+
+ def test_every_emitted_record_carries_verbatim_call_source(self) -> None:
+ trajectory = self._mixed_trajectory()
+
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert result is not None
+ assert result.diagnostics.emitted_interactions
+ for record in result.diagnostics.emitted_interactions:
+ call_source = str(record.get("call_source") or "")
+ assert call_source.strip()
+ for line in call_source.splitlines():
+ assert line.strip() in result.code
+ by_index = {record["trajectory_index"]: record for record in result.diagnostics.emitted_interactions}
+ assert 'await page.locator("#open").click()' in by_index[0]["call_source"]
+
+ def test_entry_replay_prefix_indices_are_forgiven_not_lost(self) -> None:
+ trajectory = [
+ {"tool_name": "click", "selector": "button", "source_url": "https://example.com/start"},
+ {"tool_name": "click", "selector": "#promo"},
+ {
+ "tool_name": "type_text",
+ "selector": "#name",
+ "typed_value": "Ada",
+ "role": "textbox",
+ "accessible_name": "Name",
+ },
+ {"tool_name": "click", "selector": "#submit"},
+ ]
+
+ result = synthesize_code_block(trajectory, strict_selectors=True)
+
+ assert result is not None
+ diagnostics = result.diagnostics
+ emitted = {record["trajectory_index"] for record in diagnostics.emitted_interactions}
+ dropped = {record["trajectory_index"] for record in diagnostics.dropped_interactions}
+ forgiven = {record["trajectory_index"] for record in diagnostics.forgiven_interactions}
+ assert emitted | dropped | forgiven == set(range(len(trajectory)))
+ assert emitted == {2, 3}
+ assert dropped == {0}
+ assert diagnostics.forgiven_interactions == [
+ {"trajectory_index": 1, "tool_name": "click", "lane": "entry_replay_prefix"}
+ ]
+
+
+class TestUncoveredRequiredEmittedInteractions:
+ def _emitted(self) -> list[dict[str, Any]]:
+ return [
+ {"method": "click", "locator": 'page.locator("#a")', "selector": "#a"},
+ {"method": "fill", "locator": 'page.locator("#b")', "selector": "#b"},
+ {"method": "click", "locator": 'page.locator("#c")', "selector": "#c"},
+ ]
+
+ def test_verbatim_in_order_resubmission_clears(self) -> None:
+ draft_calls = [
+ ("click", 'page.locator("#a")'),
+ ("fill", 'page.locator("#b")'),
+ ("click", 'page.locator("#c")'),
+ ]
+
+ assert uncovered_required_emitted_interactions(self._emitted(), draft_calls) == []
+
+ def test_reordered_but_complete_draft_reports_under_build(self) -> None:
+ draft_calls = [
+ ("fill", 'page.locator("#b")'),
+ ("click", 'page.locator("#a")'),
+ ("click", 'page.locator("#c")'),
+ ]
+
+ uncovered = uncovered_required_emitted_interactions(self._emitted(), draft_calls)
+
+ assert [record["selector"] for record in uncovered] == ["#b", "#c"]
+
+ def test_first_miss_over_reports_later_rungs_as_missing(self) -> None:
+ draft_calls = [
+ ("click", 'page.locator("#a")'),
+ ("click", 'page.locator("#c")'),
+ ]
+
+ uncovered = uncovered_required_emitted_interactions(self._emitted(), draft_calls)
+
+ assert [record["selector"] for record in uncovered] == ["#b", "#c"]
+
+ def test_shared_name_literal_on_different_element_does_not_cover(self) -> None:
+ emitted = [
+ {"method": "click", "locator": 'page.get_by_role("button", name="Submit")', "selector": "Submit"},
+ ]
+ draft_calls = [("click", 'page.get_by_role("link", name="Submit")')]
+
+ uncovered = uncovered_required_emitted_interactions(emitted, draft_calls)
+
+ assert [record["selector"] for record in uncovered] == ["Submit"]
+
+ def test_bare_locator_call_with_exact_full_selector_covers(self) -> None:
+ emitted = [
+ {"method": "click", "locator": 'page.locator("#a").first', "selector": "#a"},
+ ]
+ draft_calls = [("click", 'page.locator("#a")')]
+
+ assert uncovered_required_emitted_interactions(emitted, draft_calls) == []
+
+ def test_receiver_quoting_selector_among_other_literals_does_not_cover(self) -> None:
+ emitted = [
+ {"method": "click", "locator": 'page.locator("#a")', "selector": "#a"},
+ ]
+ draft_calls = [("click", 'page.locator("#wrapper").locator("#a", has_text="Go")')]
+
+ uncovered = uncovered_required_emitted_interactions(emitted, draft_calls)
+
+ assert [record["selector"] for record in uncovered] == ["#a"]
+
+ def test_lane_flagged_records_are_not_required(self) -> None:
+ emitted = self._emitted()
+ emitted[1]["lane"] = "optional_dismissal"
+ draft_calls = [
+ ("click", 'page.locator("#a")'),
+ ("click", 'page.locator("#c")'),
+ ]
+
+ assert uncovered_required_emitted_interactions(emitted, draft_calls) == []
diff --git a/tests/unit/test_copilot_enforcement_pruning.py b/tests/unit/test_copilot_enforcement_pruning.py
index 8c227dee8..1ebfd75b0 100644
--- a/tests/unit/test_copilot_enforcement_pruning.py
+++ b/tests/unit/test_copilot_enforcement_pruning.py
@@ -98,6 +98,8 @@ class _Ctx:
self.observation_after_navigate = False
self.navigate_enforcement_done = False
self.update_workflow_called = False
+ self.persisted_draft_browser_calls = None
+ self.scouted_spine_checkpoint_fired = False
self.test_after_update_done = False
self.post_update_nudge_count = 0
self.coverage_nudge_count = 0