Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10825: impose code-artifact metadata at the persist seam + adjudicate terminal challenge halts (#6496)
Andrew Neilsoned95f65c5e84
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex d7ab2f595..5541ff25e 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -35,6 +35,7 @@ from skyvern.forge.prompts import prompt_enginefrom skyvern.forge.sdk.copilot.blocker_signal import (CopilotToolBlockerSignal,assert_clean_user_facing_text,+ contains_internal_machinery_leak,)from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_datafrom skyvern.forge.sdk.copilot.build_phase import initial_build_phase@@ -53,6 +54,7 @@ from skyvern.forge.sdk.copilot.context import (finalize_discovery_counter_in_global_llm_context,)from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified+from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORYfrom skyvern.forge.sdk.copilot.outcome_verification_trace import (finalize_outcome_verification_trace,record_gate_decision,@@ -1194,6 +1196,11 @@ _UNBACKED_WORKFLOW_DELIVERY_REPLY = ("from this turn. Please retry with the target site, page, or workflow requirement.")_UNBACKED_WORKFLOW_DELIVERY_PREFIX = "I wasn't able to produce a workflow proposal in this turn."++_INLINE_REJECT_NOTE_FALLBACK = (+ "This draft didn't pass validation against the live page, so I haven't saved it. "+ "I'll revise it before proposing again."+)_GENERIC_MISSING_CONTEXT_PHRASES = ("missing details","one more detail",@@ -1396,6 +1403,36 @@ def _workflow_block_count(ctx: CopilotContext) -> int | None:return len(blocks) if isinstance(blocks, list) and blocks else None+def _observed_page_sentence(ctx: CopilotContext) -> str:+ evidence = getattr(ctx, "workflow_verification_evidence", None)+ url = getattr(evidence, "current_url", None)+ if not isinstance(url, str) or not url.strip():+ return ""+ sentence = f" The last page I observed was {url.strip()[:140]}."+ return "" if contains_internal_machinery_leak(sentence) else sentence+++def _observed_facts_halt_reply(ctx: CopilotContext) -> str:+ block_count = _workflow_block_count(ctx)+ block_phrase = f"a {block_count}-block draft" if block_count else "a draft"+ observed = _observed_page_sentence(ctx)+ if getattr(ctx, "last_workflow", None) is not None:+ return (+ f"I built {block_phrase} and was still testing it when the turn ran out of time."+ f"{observed} I haven't verified the results, so I'm not claiming them."+ )+ return (+ f"The turn ran out of time before I could finish.{observed}"+ " I haven't verified any results, so I'm not claiming them."+ )+++def _halted_mid_progress(ctx: CopilotContext, internal_tool_instruction_failure: bool) -> bool:+ if internal_tool_instruction_failure:+ return True+ return getattr(ctx, "last_failure_category_top", None) == PER_TOOL_BUDGET_FAILURE_CATEGORY++def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:# Caller-owned sentence templates add punctuation around these fragments.text = clean_recorded_failure_text(value, max_chars=max_chars).rstrip(".")@@ -1438,10 +1475,13 @@ def _recorded_failure_is_internal_tool_instruction(ctx: CopilotContext) -> bool:for value in candidates:if not isinstance(value, str) or not value.strip():continue+ # Evaluate the redacted form at the same truncation the reply embeds:+ # standard redaction already neutralizes browser-session references,+ # so flag only what would still leak.verdict = evaluate_output_policy(request_policy=None,response_type="REPLY",- user_response=value,+ user_response=clean_recorded_failure_text(value, max_chars=240),output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,)if OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes:@@ -1561,6 +1601,12 @@ def _recorded_failure_reply(reason, status_sentence = _recorded_failure_summary(ctx)if not reason:return None+ if internal_tool_instruction_failure is None:+ internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)+ # A guard-halted or budget-paced run was interrupted, not disproven; render+ # observed facts instead of a failure verdict built from internal text.+ if _halted_mid_progress(ctx, internal_tool_instruction_failure):+ return _observed_facts_halt_reply(ctx)run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80).lower()block_count = _workflow_block_count(ctx)@@ -1570,9 +1616,7 @@ def _recorded_failure_reply(or getattr(ctx, "last_test_ok", None) is not Noneor getattr(diagnosis_input, "workflow_run_id", None))- if internal_tool_instruction_failure is None:- internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)- test_failed = (ctx.last_test_ok is False or run_status == "failed") and not internal_tool_instruction_failure+ test_failed = ctx.last_test_ok is False or run_status == "failed"unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"if getattr(ctx, "last_workflow", None) is not None:@@ -1609,12 +1653,19 @@ def _build_wip_exit_result() -> AgentResult:"""Selected non-success exits surface the most recent successfully parsed workflow."""internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)+ halted_mid_progress = _halted_mid_progress(ctx, internal_tool_instruction_failure)recorded_failure_reply = _recorded_failure_reply(ctx, cancelled=cancelled, internal_tool_instruction_failure=internal_tool_instruction_failure)effective_terminal = terminal_reason or ("cancel" if cancelled else None)def _guard(text: str) -> tuple[str, TurnOutcome]:+ if contains_internal_machinery_leak(text):+ LOG.warning(+ "copilot terminal output invariant replaced leaked text",+ terminal_reason=effective_terminal,+ )+ text = _observed_facts_halt_reply(ctx)return apply_repeated_reply_guard(final_text=text,attempted_kind=ResponseKind.CLARIFY,@@ -1663,14 +1714,14 @@ def _build_wip_exit_result(if (ctx.last_workflow is not Noneand ctx.last_workflow_yaml- and (ctx.last_test_ok is not False or internal_tool_instruction_failure)+ and (ctx.last_test_ok is not False or halted_mid_progress)and not ctx.last_test_suspicious_success):full_test_ok = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is Trueunvalidated = not full_test_okif unvalidated and recorded_failure_reply:reply = recorded_failure_reply- if internal_tool_instruction_failure:+ if halted_mid_progress:reply = _ensure_unvalidated_proposal_affordance(reply)else:reply = unvalidated_reply if unvalidated else tested_reply@@ -1849,6 +1900,10 @@ async def _translate_to_agent_result(last_workflow = ctx.last_workflowlast_workflow_yaml = ctx.last_workflow_yaml+ def _with_inline_reject_note(response: Any, detail: str) -> str:+ note = detail if not contains_internal_machinery_leak(detail) else _INLINE_REJECT_NOTE_FALLBACK+ return f"{response}\n\n(Note: {note})"+if resp_type == "REPLACE_WORKFLOW":LOG.warning("Agent used inline REPLACE_WORKFLOW instead of update_workflow tool")workflow_yaml = action_data.get("workflow_yaml", "")@@ -1889,17 +1944,17 @@ async def _translate_to_agent_result(wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)if wait_block_error:- user_response = f"{user_response}\n\n(Note: {wait_block_error})"+ user_response = _with_inline_reject_note(user_response, wait_block_error)ctx.last_test_ok = Noneworkflow_yaml = ""banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)if banned_items:_record_banned_block_reject_span("replace_workflow_inline", banned_items)- user_response = f"{user_response}\n\n(Note: {_banned_block_reject_message(banned_items)})"+ user_response = _with_inline_reject_note(user_response, _banned_block_reject_message(banned_items))workflow_yaml = ""stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)if stale_metadata:- user_response = f"{user_response}\n\n(Note: {_stale_block_metadata_message(stale_metadata)})"+ user_response = _with_inline_reject_note(user_response, _stale_block_metadata_message(stale_metadata))ctx.last_test_ok = Noneworkflow_yaml = ""composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)@@ -1909,7 +1964,7 @@ async def _translate_to_agent_result(workflow_permanent_id=ctx.workflow_permanent_id,target_url=workflow_target_url(workflow_yaml),)- user_response = f"{user_response}\n\n(Note: {composition_evidence_error})"+ user_response = _with_inline_reject_note(user_response, composition_evidence_error)ctx.last_test_ok = Noneworkflow_yaml = ""if workflow_yaml:diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 6c88fc2d0..54728c4f7 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -2,6 +2,7 @@from __future__ import annotations+import refrom collections.abc import Mappingfrom types import MappingProxyTypefrom typing import Any, Literal, Protocol@@ -57,14 +58,53 @@ _LEAK_DENY_TOKENS: tuple[str, ...] = ("normal instruction","like 'continue",'like "continue',+ "per-tool-call budget",)+# Raw workflow-run and browser-session identifiers are internal; user-facing+# text must reference runs by what they did, never by id.+_RUN_ID_LEAK_RE = re.compile(r"\b(?:wr|pbs)_[a-z0-9_]+", re.IGNORECASE)++_INTERNAL_GUARD_TOKENS: tuple[str, ...] = (+ "per_tool_budget",+ "per-tool-call budget",+ "active_run_terminal_evidence",+ "block-running tool",+ "block running tool",+)++_INTERNAL_TOOL_NAME_TOKENS: tuple[str, ...] = (+ "update_workflow",+ "update_and_run_blocks",+ "run_blocks_and_collect_debug",+ "get_run_results",+ "inspect_page_for_composition",+ "discover_workflow_entrypoint",+ "get_browser_screenshot",+ "list_credentials",+)+++def contains_internal_machinery_leak(value: str | None) -> bool:+ """String-level terminal-output invariant: user-facing text carries no raw+ run ids, internal guard tokens, or agent-directed tool references."""+ if not isinstance(value, str) or not value:+ return False+ if _RUN_ID_LEAK_RE.search(value):+ return True+ lowered = value.lower()+ if any(token in lowered for token in _INTERNAL_GUARD_TOKENS):+ return True+ return any(token in lowered for token in _INTERNAL_TOOL_NAME_TOKENS)+def assert_clean_user_facing_text(value: str, *, blocked_tool: str | None = None) -> None:lowered = value.lower()for token in _LEAK_DENY_TOKENS:if token.lower() in lowered:raise ValueError(f"blocker user-facing text leaked token {token!r}: {value!r}")+ if contains_internal_machinery_leak(value):+ raise ValueError(f"blocker user-facing text leaked internal machinery: {value!r}")if blocked_tool and blocked_tool.lower() in lowered:raise ValueError(f"blocker user-facing text leaked tool name {blocked_tool!r}: {value!r}")diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pyindex c90a9e0e6..4a8124853 100644--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -271,9 +271,6 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesizereturn SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes)-_DEPENDENCY_ID = f"dependency:{_SYNTHESIZED_BLOCK_LABEL}_reached"-_OBSERVATION_REF_ID = f"observation:{_SYNTHESIZED_BLOCK_LABEL}_scout"-# Model-owned slots the synthesizer cannot prove; the model fills these._FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"_FILL_CLAIM_ID = "claim:<fill>"@@ -282,9 +279,23 @@ _FILL_CRITERION_ID = "criterion:<fill>"_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"-def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+def artifact_dependency_id(block_label: str) -> str:+ return f"dependency:{_slug(block_label)}_reached"+++def artifact_observation_ref_id(block_label: str) -> str:+ return f"observation:{_slug(block_label)}_scout"+++def build_artifact_metadata_skeleton(+ trajectory: Sequence[Mapping[str, Any]],+ *,+ block_label: str,+) -> dict[str, Any]:"""Fill only the scout-proven evidence shape (`observed_not_verified`); the terminal goal andoutcomes stay as `<fill>` placeholders the model owns."""+ dependency_id = artifact_dependency_id(block_label)+ observation_ref_id = artifact_observation_ref_id(block_label)# First recorded source_url is the page the synthesized block's leading `page.goto` lands on.# The trajectory carries only pre-action source pages, not post-action reached URLs, so this is# an advisory entry-page hint, never a reached-page identity (which SPAs would mis-key anyway).@@ -294,23 +305,23 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]))page_dependency: dict[str, Any] = {- "id": _DEPENDENCY_ID,+ "id": dependency_id,"scope": "page","status": "observed_not_verified",- "observation_refs": [_OBSERVATION_REF_ID],+ "observation_refs": [observation_ref_id],}if entry_url_hint:page_dependency["url_hint"] = entry_url_hintobservation_ref: dict[str, Any] = {- "observation_ref": _OBSERVATION_REF_ID,- "dependency_id": _DEPENDENCY_ID,+ "observation_ref": observation_ref_id,+ "dependency_id": dependency_id,"status": "observed_not_verified","source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,}return {- "block_label": _SYNTHESIZED_BLOCK_LABEL,+ "block_label": block_label,"declared_goal": _FILL_DECLARED_GOAL,"page_dependencies": [page_dependency],"observation_refs": [observation_ref],@@ -320,9 +331,9 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]])"scope": "outcome","text": _FILL_CLAIM_TEXT,"status": "observed_not_verified",- "depends_on": [_DEPENDENCY_ID],+ "depends_on": [dependency_id],"covered_criteria": [_FILL_CRITERION_ID],- "observation_refs": [_OBSERVATION_REF_ID],+ "observation_refs": [observation_ref_id],}],"completion_criteria": [@@ -343,6 +354,10 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]])}+def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL)++def _render_artifact_metadata_block(metadata: Mapping[str, Any]) -> str:return json.dumps(metadata, indent=2, sort_keys=True)diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.pyindex b1a89a93a..ea9bd557f 100644--- a/skyvern/forge/sdk/copilot/composition_evidence.py+++ b/skyvern/forge/sdk/copilot/composition_evidence.py@@ -111,22 +111,41 @@ def _challenge_kind(indicators: list[str]) -> str:return "unknown" if indicators else "none"+# Tags that carry challenge-vendor markup without rendering a widget. A passive+# script/meta tag ships on every page behind some CDNs, so it can trigger the+# visual fallback but never assert human verification by itself.+_PASSIVE_CHALLENGE_TAGS: frozenset[str] = frozenset({"script", "noscript", "style", "link", "meta"})+++def interactive_challenge_controls(challenge_controls: list[dict[str, Any]] | None) -> list[dict[str, Any]]:+ return [+ control+ for control in challenge_controls or []+ if isinstance(control, dict) and str(control.get("tag") or "").lower() not in _PASSIVE_CHALLENGE_TAGS+ ]++def _challenge_state(indicators: list[str],*,source: str = DOM_EVIDENCE_SOURCE,gated_submit_controls: list[dict[str, Any]] | None = None,+ challenge_controls: list[dict[str, Any]] | None = None,) -> dict[str, Any]:detected = bool(indicators)gated_controls = gated_submit_controls or []+ # Raw-HTML token hits only mark `detected` (triggering the visual fallback);+ # asserting human verification requires a rendered challenge control or a+ # later vision confirmation.+ semantic_challenge = bool(interactive_challenge_controls(challenge_controls))return {"detected": detected,"kind": _challenge_kind(indicators),"source": source if detected else "","indicators": indicators[:8],- "requires_human_verification": detected,+ "requires_human_verification": semantic_challenge,"visual_location": "",- "gates_submit_controls": bool(detected and gated_controls),+ "gates_submit_controls": bool(semantic_challenge and gated_controls),"gated_submit_controls": gated_controls[:5] if detected else [],}@@ -165,6 +184,7 @@ def _evidence_metadata(indicators: list[str] | None = None,*,forms: list[dict[str, Any]] | None = None,+ challenge_controls: list[dict[str, Any]] | None = None,) -> dict[str, Any]:gated_controls = _gated_submit_controls(forms or [])return {@@ -176,6 +196,7 @@ def _evidence_metadata("challenge_state": _challenge_state(indicators or [],gated_submit_controls=gated_controls,+ challenge_controls=challenge_controls,),}@@ -223,6 +244,20 @@ def page_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:return bool(evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"))+# The vision classifier's typed non-challenge obstruction kind: a consent dialog+# is dismissed, not solved, so it must never promote challenge state.+CONSENT_OBSTRUCTION_KIND = "cookie_consent"+++def _confirmed_visual_challenge(evidence: dict[str, Any], visual_summary: dict[str, Any]) -> bool:+ if visual_summary.get("challenge_detected") is not True:+ return False+ if interactive_challenge_controls(evidence.get("challenge_controls")):+ return True+ obstruction_kind = str(visual_summary.get("obstruction_kind") or "").strip().lower()+ return obstruction_kind != CONSENT_OBSTRUCTION_KIND++def merge_visual_composition_evidence(evidence: dict[str, Any],*,@@ -261,28 +296,34 @@ def merge_visual_composition_evidence(if bounded:omissions.append(bounded)challenge_state = dict(merged.get("challenge_state") or {})- if visual_summary.get("challenge_detected") is True:+ challenge_confirmed = _confirmed_visual_challenge(evidence, visual_summary)+ if challenge_confirmed:challenge_state["detected"] = Truechallenge_state["requires_human_verification"] = Truechallenge_state["source"] = ("dom+screenshot" if challenge_state.get("source") else SCREENSHOT_EVIDENCE_SOURCE)- challenge_kind = _bounded_string(visual_summary.get("challenge_kind"), 80)- if challenge_kind:- challenge_state["kind"] = challenge_kind- challenge_location = _bounded_string(visual_summary.get("challenge_location"), 180)- if challenge_location:- challenge_state["visual_location"] = challenge_location- if visual_summary.get("submit_blocked") is True:+ if challenge_confirmed or challenge_state.get("detected") is True:+ challenge_kind = _bounded_string(visual_summary.get("challenge_kind"), 80)+ if challenge_kind:+ challenge_state["kind"] = challenge_kind+ challenge_location = _bounded_string(visual_summary.get("challenge_location"), 180)+ if challenge_location:+ challenge_state["visual_location"] = challenge_location+ if visual_summary.get("submit_blocked") is True and challenge_confirmed:challenge_state["gates_submit_controls"] = True- visual_blocked_controls = [- {- "text": _bounded_string(item, 120),- "disabled": True,- }- for item in visual_summary.get("blocked_submit_controls") or []- if _bounded_string(item, 120)- ]+ visual_blocked_controls = (+ [+ {+ "text": _bounded_string(item, 120),+ "disabled": True,+ }+ for item in visual_summary.get("blocked_submit_controls") or []+ if _bounded_string(item, 120)+ ]+ if challenge_confirmed+ else []+ )if visual_blocked_controls:existing_controls = [item for item in challenge_state.get("gated_submit_controls") or [] if isinstance(item, dict)@@ -1664,7 +1705,7 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -"empty_page_visual_state": None,"evidence_confidence": confidence,"source_tool": "inspect_page_for_composition",- **_evidence_metadata(anti_bot_indicators, forms=forms),+ **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),}@@ -1940,5 +1981,5 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:"empty_page_visual_state": None,"evidence_confidence": confidence,"source_tool": "inspect_page_for_composition",- **_evidence_metadata(anti_bot_indicators, forms=forms),+ **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),}diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex 707b2b091..b6044da6a 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -5,6 +5,7 @@ from dataclasses import dataclass, fieldfrom enum import StrEnumfrom typing import Any, cast+from skyvern.forge.sdk.copilot.blocker_signal import contains_internal_machinery_leakfrom skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, ResponseTypefrom skyvern.forge.sdk.copilot.output_utils import (looks_like_workflow_delivery_claim,@@ -376,6 +377,8 @@ def _contains_raw_secret(value: Any) -> bool:def _contains_internal_tool_instruction(user_response: str | None) -> bool:if not isinstance(user_response, str):return False+ if contains_internal_machinery_leak(user_response):+ return Truenormalized = " ".join(user_response.lower().translate(_INTERNAL_TOOL_INSTRUCTION_TRANSLATION).split())if any(marker in normalized for marker in _INTERNAL_TOOL_INSTRUCTION_MARKERS):return Truediff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 2ce073605..cc3d5292c 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -70,6 +70,8 @@ from .blockers import _analyze_run_blocks as _analyze_run_blocksfrom .blockers import _build_loop_blocker_signal as _build_loop_blocker_signalfrom .blockers import _last_run_has_terminal_anti_bot_blocker as _last_run_has_terminal_anti_bot_blockerfrom .blockers import _per_tool_budget_problem_rerun_signal as _per_tool_budget_problem_rerun_signal+from .blockers import _post_budget_terminal_challenge_signal as _post_budget_terminal_challenge_signal+from .blockers import _post_run_terminal_challenge_reason as _post_run_terminal_challenge_reasonfrom .blockers import (_record_per_tool_budget_problem_blocks_from_results,_tool_loop_error,@@ -226,6 +228,7 @@ from .scouting import _role_name_from_selector as _role_name_from_selectorfrom .workflow_update import BlockObservationRef as BlockObservationReffrom .workflow_update import CodeArtifactMetadata as CodeArtifactMetadatafrom .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argument+from .workflow_update import _code_block_safety_errors as _code_block_safety_errorsfrom .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadatafrom .workflow_update import _pre_run_workflow_coverage_error as _pre_run_workflow_coverage_errorfrom .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_spandiff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.pyindex eee55372d..870e9903c 100644--- a/skyvern/forge/sdk/copilot/tools/blockers.py+++ b/skyvern/forge/sdk/copilot/tools/blockers.py@@ -8,7 +8,12 @@ import structlogfrom skyvern.forge import appfrom skyvern.forge.failure_classifier import classify_from_failure_reason-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, build_loop_blocker_signal+from skyvern.forge.sdk.copilot.blocker_signal import (+ CopilotToolBlockerSignal,+ assert_clean_user_facing_text,+ build_loop_blocker_signal,+)+from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controlsfrom skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDSfrom skyvern.forge.sdk.copilot.failure_tracking import (ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,@@ -361,7 +366,9 @@ def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:indicators = evidence.get("anti_bot_indicators")has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)- if has_indicators:+ # Raw-HTML token hits alone never justify a terminal claim; a rendered+ # challenge control must corroborate them.+ if has_indicators and interactive_challenge_controls(evidence.get("challenge_controls")):disabled_controls = _disabled_submit_controls_from_evidence(evidence)if disabled_controls:labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])@@ -369,6 +376,42 @@ def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:return None+def _post_run_result_evidence_summary(evidence: dict[str, Any]) -> str | None:+ containers = [container for container in evidence.get("result_containers") or [] if isinstance(container, dict)]+ if not containers:+ return None+ hints: list[str] = []+ for container in containers[:3]:+ for key in ("selector", "id", "tag"):+ value = container.get(key)+ if isinstance(value, str) and value.strip():+ hints.append(value.strip()[:80])+ break+ hint_text = f" ({', '.join(hints)})" if hints else ""+ return f"{len(containers)} result container(s){hint_text}"+++_TERMINAL_CHALLENGE_FALLBACK_USER_FACING = (+ "A page barrier was still blocking the workflow's next step after the last run, "+ "so I stopped without claiming results."+)+++def _terminal_challenge_user_facing_reason(reason: str) -> str:+ # The narrative names the observed evidence (challenge kind, control labels)+ # instead of asserting a fixed mechanism; page-derived text falls back to a+ # pre-validated template if it ever trips the leak deny list.+ candidate = (+ f"I stopped because the page I inspected after the last run still shows: {reason}. "+ "I haven't verified any results, so I'm not claiming them."+ )+ try:+ assert_clean_user_facing_text(candidate)+ except ValueError:+ return _TERMINAL_CHALLENGE_FALLBACK_USER_FACING+ return candidate++def _post_budget_terminal_challenge_signal(ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str) -> CopilotToolBlockerSignal | None:@@ -391,23 +434,45 @@ def _post_budget_terminal_challenge_signal(if not reason:return None+ # Same-packet adjudication: when the observed page also carries result+ # evidence, the answer-from-observed-page path outranks a terminal claim.+ result_evidence = _post_run_result_evidence_summary(evidence)+ if result_evidence:+ agent_steering = (+ "The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "+ f"shows: {reason} — but the SAME observed page also contains result evidence: {result_evidence}. "+ f"Do NOT claim the page is blocked and do NOT rerun {tool_name} unchanged. "+ "Answer from the observed page if it contains the requested result/no-result evidence; "+ "otherwise inspect the live page again before deciding."+ )+ return CopilotToolBlockerSignal(+ blocker_kind="tool_error",+ agent_steering_text=agent_steering,+ user_facing_reason=(+ "The last page I checked already shows results. I'll work from that evidence instead of retrying."+ ),+ recovery_hint="retry_with_different_tool",+ cleared_by_tools=PAGE_INSPECTION_TOOLS+ | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),+ preserves_workflow_draft=True,+ renders_final_reply=False,+ internal_reason_code="tool_error_post_budget_challenge_result_evidence",+ blocked_tool=tool_name,+ )+requested_labels = _requested_block_label_set(arguments)labels_text = f" Requested labels: {', '.join(sorted(requested_labels))}." if requested_labels else ""agent_steering = ("The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "f"now shows: {reason}.{labels_text} Do NOT call "f"{tool_name} again in this turn, do NOT try another proxy/location from this evidence state, and "- "do NOT claim registry results or no-results were verified. REPLY now with a blocker explanation "+ "do NOT claim results or no-results were verified. REPLY now with a blocker explanation ""that names the observed challenge/disabled control and summarizes the tested workflow state.")- user_facing = (- "The site's verification challenge is still blocking the submit/search control after live-page inspection, "- "so I stopped without claiming results."- )return CopilotToolBlockerSignal(blocker_kind="tool_error",agent_steering_text=agent_steering,- user_facing_reason=user_facing,+ user_facing_reason=_terminal_challenge_user_facing_reason(reason),recovery_hint="report_blocker_to_user",cleared_by_tools=frozenset(),preserves_workflow_draft=True,diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex 31d01fe1f..4e8e50729 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,)from skyvern.forge.sdk.copilot.composition_evidence import (+ CONSENT_OBSTRUCTION_KIND,has_bounded_page_schema,merge_visual_composition_evidence,page_evidence_needs_visual_fallback,@@ -247,11 +248,12 @@ def _composition_extract_screenshot_b64(result: dict[str, Any]) -> str:def _composition_visual_prompt(evidence: dict[str, Any]) -> str:+ # The DOM challenge_state / anti-bot token hits are deliberately NOT fed in:+ # the vision pass classifies obstruction-vs-challenge from the screenshot+ # alone instead of confirming the detector's anchor.context = {"page_title": evidence.get("page_title") or "","current_url": evidence.get("current_url") or "",- "anti_bot_indicators": evidence.get("anti_bot_indicators") or [],- "challenge_state": evidence.get("challenge_state") or {},"form_count": len(evidence.get("forms") or []),"result_container_count": len(evidence.get("result_containers") or []),"page_obstruction_count": len(evidence.get("page_obstructions") or []),@@ -267,9 +269,18 @@ def _composition_visual_prompt(evidence: dict[str, Any]) -> str:"In summary, include the visible page state that would help verify an end-state outcome, ""such as cart items, ""record rows, visible identifiers, quantities, statuses, prices, confirmations, search results, "- "or selected values when legible. Also note visible anti-bot or human-verification state, "- "whether it appears to gate a submit/search control, whether any visible artificial barrier "- "mechanically blocks the underlying page, and where it appears relative to the page controls. "+ "or selected values when legible. "+ "Classify any visible artificial barrier from the screenshot alone: a verification challenge is a "+ "captcha or human-verification widget asking the visitor to prove they are human; a page "+ "obstruction is a dismissible layer such as a cookie/privacy consent dialog, promo or newsletter "+ "modal, chat widget, or loading overlay. "+ f"Use obstruction_kind values: {CONSENT_OBSTRUCTION_KIND}, promo_modal, chat_widget, "+ "loading_overlay, other. A cookie/privacy consent dialog is always a page obstruction, never a "+ "challenge: report it with page_obstruction_detected true and obstruction_kind "+ f"{CONSENT_OBSTRUCTION_KIND}, and do not set challenge_detected or submit_blocked for it. "+ "Set challenge_detected to true only for a visible verification challenge, set submit_blocked to "+ "true only when that challenge visibly gates a submit/search control, and note where the barrier "+ "appears relative to the page controls. ""Do not include raw DOM, code, selectors, personal data, or workflow instructions. ""If no challenge is visible, set challenge_detected to false and submit_blocked to false. ""If no page obstruction is visible, set page_obstruction_detected to false. "diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 6417b4be8..ed86894dd 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -10,11 +10,14 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationErrorfrom skyvern.forge import appfrom skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp+from skyvern.forge.sdk.copilot.code_block_synthesis import artifact_dependency_id, artifact_observation_ref_idfrom skyvern.forge.sdk.copilot.composition_evidence import (+ SCOUT_INTERACTION_EVIDENCE_TOOL,composition_page_evidence_error,normalize_block_observation_refs,workflow_target_url,)+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicyfrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (POST_INTERMEDIATE_SUCCESS_NUDGE,@@ -26,11 +29,12 @@ from skyvern.forge.sdk.copilot.output_policy import (format_output_policy_tool_error,output_policy_verdict_to_trace_data,)-from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteractionfrom skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_endfrom skyvern.forge.sdk.copilot.tracing_setup import copilot_spanfrom skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml-from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected+from skyvern.forge.sdk.workflow.models.block import CodeBlockfrom skyvern.forge.sdk.workflow.models.workflow import Workflowfrom skyvern.schemas.proxy_location import ProxyLocationfrom skyvern.schemas.workflows import BlockType@@ -47,6 +51,7 @@ from .banned_blocks import (_banned_block_reject_message,_challenge_http_request_reject_message,_copilot_banned_block_types,+ _copilot_block_authoring_policy,_detect_new_banned_blocks,_record_banned_block_reject_span,_timing_only_challenge_wait_reject_message,@@ -78,12 +83,12 @@ ArtifactEvidenceStatus = Literal["satisfied", "missing", "diagnostic_only", "obsclass CodeArtifactClaimedOutcome(BaseModel):- id: str- scope: str- text: str- status: ArtifactEvidenceStatus- depends_on: list[str] = Field(default_factory=list)- covered_criteria: list[str] = Field(default_factory=list)+ id: str = ""+ scope: str = ""+ text: str = ""+ status: ArtifactEvidenceStatus = "observed_not_verified"+ depends_on: list[str] = Field(default_factory=list, description="Page-dependency ids this claim relies on.")+ covered_criteria: list[str] = Field(default_factory=list, description="Completion-criterion ids this claim covers.")criteria_ids: list[str] = Field(default_factory=list)evidence_refs: list[str] = Field(default_factory=list)observation_refs: list[str] = Field(default_factory=list)@@ -92,9 +97,9 @@ class CodeArtifactClaimedOutcome(BaseModel):class CodeArtifactPageDependency(BaseModel):- id: str- scope: str- status: ArtifactEvidenceStatus+ id: str = ""+ scope: str = ""+ status: ArtifactEvidenceStatus = "observed_not_verified"url_hint: str | None = Nonepage_state_hint: str | None = Nonerequired_affordances: list[str] = Field(default_factory=list)@@ -104,9 +109,9 @@ class CodeArtifactPageDependency(BaseModel):class CodeArtifactCompletionCriterion(BaseModel):- id: str- text: str- level: Literal["terminal", "outcome", "prefix", "method"]+ id: str = ""+ text: str = ""+ level: Literal["terminal", "outcome", "prefix", "method"] = "terminal"outcome: str | None = Noneterminal: bool | None = None@@ -117,7 +122,9 @@ class CodeArtifactScopedRef(BaseModel):criterion_id: str | None = Noneevidence_ref: str | None = Noneobservation_ref: str | None = None- status: ArtifactEvidenceStatus = Field(validation_alias=AliasChoices("status", "evidence_status"))+ status: ArtifactEvidenceStatus = Field(+ default="observed_not_verified", validation_alias=AliasChoices("status", "evidence_status")+ )source_tool: str | None = Noneobservation_step: Annotated[int, Field(ge=0)] | None = Nonerun_sample_id: str | None = None@@ -127,15 +134,15 @@ class CodeArtifactScopedRef(BaseModel):class CodeArtifactTerminalVerifierExpectation(BaseModel):- id: str- text: str+ id: str = ""+ text: str = ""criteria_ids: list[str] = Field(default_factory=list)claimed_outcome_ids: list[str] = Field(default_factory=list)class CodeArtifactExplorationObservation(BaseModel):- id: str- text: str+ id: str = ""+ text: str = ""status: Literal["observed_not_verified"] = Field(default="observed_not_verified",validation_alias=AliasChoices("status", "evidence_status"),@@ -145,21 +152,59 @@ class CodeArtifactExplorationObservation(BaseModel):observation_step: Annotated[int, Field(ge=0)] | None = Nonecurrent_url: str | None = Nonesource_label: str | None = None- checkpoint_next_mode: Literal["stop"] | None = None+ checkpoint_next_mode: Literal["advance", "stop"] | None = Noneclass CodeArtifactMetadata(BaseModel):- artifact_id: str | None = None- block_label: str | None = None+ artifact_id: str | None = Field(+ default=None, description="Server-owned id; defaults to `code_artifact:<block_label>` when omitted."+ )+ block_label: str | None = Field(+ default=None, description="Label of the authored `code` block this artifact describes."+ )block_id: str | None = None- declared_goal: str- claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(default_factory=list)- page_dependencies: list[CodeArtifactPageDependency] = Field(default_factory=list)- completion_criteria: list[CodeArtifactCompletionCriterion] = Field(default_factory=list)- evidence_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)- observation_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)- terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(default_factory=list)- exploration_observations: list[CodeArtifactExplorationObservation] = Field(default_factory=list)+ declared_goal: str = Field(default="", description="The durable goal this block accomplishes; model-owned.")+ claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(+ default_factory=list,+ description=(+ "Outcomes this block claims. Each claim links `depends_on` page-dependency ids, covered "+ "criterion ids, and claim-scoped observation/evidence refs; a `satisfied` claim additionally "+ "requires claim-scoped `evidence_refs`. Mechanical links are server-defaulted at the "+ "persist seam in code-block authoring mode."+ ),+ )+ page_dependencies: list[CodeArtifactPageDependency] = Field(+ default_factory=list,+ description=(+ "Pages or states the code depends on; non-`missing` rows carry scoped evidence or observation "+ "refs. Server-defaulted when omitted."+ ),+ )+ completion_criteria: list[CodeArtifactCompletionCriterion] = Field(+ default_factory=list,+ description="Completion criteria; include at least one `terminal` criterion.",+ )+ evidence_refs: list[CodeArtifactScopedRef] = Field(+ default_factory=list,+ description=(+ "Artifact-level refs: each entry carries its ref id, a scoped id (claim/dependency/criterion), "+ "and `source_tool` unless status is `missing`."+ ),+ )+ observation_refs: list[CodeArtifactScopedRef] = Field(+ default_factory=list,+ description=(+ "Artifact-level observation refs; same shape rules as `evidence_refs`. Server-defaulted when omitted."+ ),+ )+ terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(+ default_factory=list,+ description="What terminal verification must observe; link `criteria_ids` or `claimed_outcome_ids`.",+ )+ exploration_observations: list[CodeArtifactExplorationObservation] = Field(+ default_factory=list,+ description="Scout-time observations; status stays `observed_not_verified` until verification passes.",+ )_CODE_ARTIFACT_REQUIRED_LIST_FIELDS = (@@ -190,13 +235,24 @@ def _format_code_artifact_violations(violations: list[str]) -> str:def _normalize_code_artifact_metadata(raw_metadata: Any,workflow_yaml: str,+ *,+ impose_defaults: bool = False,+ scout_trajectory: list[ScoutedInteraction] | None = None,) -> tuple[dict[str, dict[str, Any]], str | None]:+ """Normalize submitted artifact metadata at the persist seam.++ Entries keyed to a missing code-block label are re-keyed to the single+ uncovered block or dropped, never rejected. When imposing, mechanical graph+ fields are server-defaulted (and uncovered labels get a deterministic+ skeleton) so only contradictions reject."""if raw_metadata in (None, [], {}):return {}, Noneitems = _code_artifact_metadata_items(raw_metadata)code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)- normalized: dict[str, dict[str, Any]] = {}+ trajectory: list[ScoutedInteraction] = scout_trajectory or []violations: list[str] = []+ anchored: list[dict[str, Any]] = []+ unanchored: list[dict[str, Any]] = []seen_labels: set[str] = set()for raw_item in items:try:@@ -210,32 +266,55 @@ def _normalize_code_artifact_metadata(continuedumped = metadata.model_dump(mode="json", exclude_none=True)label = str(dumped.get("block_label") or "").strip()- if not label:- violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")- continue- if label not in code_blocks:- violations.append(- f"Artifact metadata for `{label}` does not reference an existing code block label. "- "Attach metadata only to authored code blocks."- )+ if not label or label not in code_blocks:+ unanchored.append(dumped)continueif label in seen_labels:- violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")+ LOG.info("copilot_code_artifact_metadata_duplicate_dropped", block_label=label)continueseen_labels.add(label)dumped["block_label"] = label- item_violations: list[str] = []+ anchored.append(dumped)++ uncovered = [label for label in code_blocks if label not in seen_labels]+ if len(unanchored) == 1 and len(uncovered) == 1:+ # The only metadata-less code block is the only plausible owner of the+ # only unanchored entry; re-key instead of dropping.+ rekeyed = unanchored.pop()+ LOG.info(+ "copilot_code_artifact_metadata_rekeyed",+ stale_label=str(rekeyed.get("block_label") or "") or None,+ block_label=uncovered[0],+ )+ rekeyed["block_label"] = uncovered[0]+ rekeyed.pop("artifact_id", None)+ rekeyed.pop("block_id", None)+ seen_labels.add(uncovered[0])+ anchored.append(rekeyed)+ uncovered = []+ if unanchored:+ LOG.info(+ "copilot_code_artifact_metadata_stale_entries_dropped",+ stale_labels=[str(item.get("block_label") or "") or None for item in unanchored],+ )++ normalized: dict[str, dict[str, Any]] = {}+ for dumped in anchored:+ label = str(dumped["block_label"])artifact_id = str(dumped.get("artifact_id") or "").strip()- if artifact_id and not artifact_id.startswith("code_artifact:"):- item_violations.append(- f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."- )- dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)+ if not artifact_id.startswith("code_artifact:"):+ artifact_id = _artifact_id_for_block_label(label)+ dumped["artifact_id"] = artifact_idblock_id = str(dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or "").strip()if block_id:dumped["block_id"] = block_id+ if impose_defaults:+ if not str(dumped.get("declared_goal") or "").strip():+ dumped["declared_goal"] = _block_goal_fallback(label, code_blocks[label])+ _impose_code_artifact_metadata_defaults(label, dumped, trajectory)+ item_violations: list[str] = []declared_goal = str(dumped.get("declared_goal") or "").strip()if not declared_goal:item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")@@ -250,11 +329,221 @@ def _normalize_code_artifact_metadata(violations.extend(item_violations)continuenormalized[label] = dumped+ if impose_defaults:+ for label in uncovered:+ normalized[label] = _imposed_artifact_skeleton(label, code_blocks[label], trajectory)if violations:- return {}, _format_code_artifact_violations(violations)+ return normalized, _format_code_artifact_violations(violations)return normalized, None+def _artifact_label_fragment(label: str) -> str:+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"+++def _artifact_mutable_rows(value: Any) -> list[dict[str, Any]]:+ return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else []+++def _drop_contradictory_checkpoint_mode(ref: dict[str, Any]) -> None:+ status = str(ref.get("status") or "").strip()+ mode = ref.get("checkpoint_next_mode")+ if mode == "advance" and status != "diagnostic_only":+ ref.pop("checkpoint_next_mode", None)+ elif mode == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:+ ref.pop("checkpoint_next_mode", None)+++def _impose_code_artifact_metadata_defaults(+ label: str,+ artifact: dict[str, Any],+ scout_trajectory: list[ScoutedInteraction],+) -> None:+ """Fill the mechanical evidence-graph fields the persist seam owns; semantic+ content (declared_goal, claim/criterion text) stays model-authored, and+ contradictions (e.g. `satisfied` without evidence) are left for validation."""+ fragment = _artifact_label_fragment(label)+ dependency_id = artifact_dependency_id(label)+ observation_ref_id = artifact_observation_ref_id(label)+ declared_goal = str(artifact.get("declared_goal") or "").strip()++ dependencies = _artifact_mutable_rows(artifact.get("page_dependencies"))+ for index, dependency in enumerate(dependencies):+ if not str(dependency.get("id") or "").strip():+ dependency["id"] = f"dependency:{fragment}_{index}"+ if not str(dependency.get("scope") or "").strip():+ dependency["scope"] = "page"+ if not dependencies:+ default_dependency: dict[str, Any] = {+ "id": dependency_id,+ "scope": "page",+ "status": "observed_not_verified",+ "observation_refs": [observation_ref_id],+ }+ entry_url_hint = next(+ (url for url in (str(item.get("source_url") or "").strip() for item in scout_trajectory) if url),+ None,+ )+ if entry_url_hint:+ default_dependency["url_hint"] = entry_url_hint+ artifact["page_dependencies"] = [default_dependency]+ dependencies = _artifact_mutable_rows(artifact.get("page_dependencies"))+ primary_dependency_id = str(dependencies[0].get("id") or "").strip() or dependency_id++ claims = _artifact_mutable_rows(artifact.get("claimed_outcomes"))+ for index, claim in enumerate(claims):+ if not str(claim.get("id") or "").strip():+ claim["id"] = f"claim:{fragment}_{index}"+ if not str(claim.get("scope") or "").strip():+ claim["scope"] = "outcome"+ if not str(claim.get("text") or "").strip():+ claim["text"] = declared_goal or f"outcome of `{label}`"+ if not claims and declared_goal:+ artifact["claimed_outcomes"] = [+ {+ "id": f"claim:{fragment}_goal",+ "scope": "outcome",+ "text": declared_goal,+ "status": "observed_not_verified",+ "depends_on": [primary_dependency_id],+ "observation_refs": [observation_ref_id],+ }+ ]+ claims = _artifact_mutable_rows(artifact.get("claimed_outcomes"))++ criteria = _artifact_mutable_rows(artifact.get("completion_criteria"))+ for index, criterion in enumerate(criteria):+ if not str(criterion.get("id") or "").strip():+ criterion["id"] = f"criterion:{fragment}_goal_{index}"+ if not str(criterion.get("text") or "").strip():+ criterion["text"] = declared_goal or f"criterion for `{label}`"+ if not criteria and claims:+ artifact["completion_criteria"] = [+ {+ "id": f"criterion:{fragment}_goal_{index}",+ "text": str(claim.get("text") or declared_goal or "").strip() or f"criterion for `{label}`",+ "level": "terminal",+ "terminal": True,+ }+ for index, claim in enumerate(claims)+ ]+ criteria = _artifact_mutable_rows(artifact.get("completion_criteria"))+ criterion_ids = [str(criterion.get("id") or "").strip() for criterion in criteria]+ criterion_ids = [criterion_id for criterion_id in criterion_ids if criterion_id]++ for claim in claims:+ if not isinstance(claim, dict):+ continue+ if not _artifact_string_list(claim.get("depends_on")):+ claim["depends_on"] = [primary_dependency_id]+ if (+ criterion_ids+ and not _artifact_string_list(claim.get("covered_criteria"))+ and not _artifact_string_list(claim.get("criteria_ids"))+ ):+ claim["covered_criteria"] = list(criterion_ids)+ status = str(claim.get("status") or "").strip()+ if (+ status not in {"missing", "satisfied"}+ and not _artifact_string_list(claim.get("evidence_refs"))+ and not _artifact_string_list(claim.get("observation_refs"))+ ):+ claim["observation_refs"] = [observation_ref_id]++ for dependency in dependencies:+ if not isinstance(dependency, dict):+ continue+ status = str(dependency.get("status") or "").strip()+ if (+ status not in {"missing", "satisfied"}+ and not _artifact_string_list(dependency.get("evidence_refs"))+ and not _artifact_string_list(dependency.get("observation_refs"))+ ):+ dependency["observation_refs"] = [observation_ref_id]++ expectations = _artifact_mutable_rows(artifact.get("terminal_verifier_expectations"))+ if not expectations and (criterion_ids or declared_goal):+ artifact["terminal_verifier_expectations"] = [+ {+ "id": f"expectation:{fragment}_terminal",+ "text": f"Terminal verification observes: {declared_goal or label}",+ "criteria_ids": list(criterion_ids),+ }+ ]+ expectations = _artifact_mutable_rows(artifact.get("terminal_verifier_expectations"))+ for index, expectation in enumerate(expectations):+ if not isinstance(expectation, dict):+ continue+ if not str(expectation.get("id") or "").strip():+ expectation["id"] = f"expectation:{fragment}_{index}"+ if not str(expectation.get("text") or "").strip():+ expectation["text"] = f"Terminal verification observes: {declared_goal or label}"+ if (+ criterion_ids+ and not _artifact_string_list(expectation.get("criteria_ids"))+ and not _artifact_string_list(expectation.get("claimed_outcome_ids"))+ ):+ expectation["criteria_ids"] = list(criterion_ids)++ if not _artifact_mutable_rows(artifact.get("evidence_refs")) and not _artifact_mutable_rows(+ artifact.get("observation_refs")+ ):+ artifact["observation_refs"] = [+ {+ "observation_ref": observation_ref_id,+ "dependency_id": primary_dependency_id,+ "status": "observed_not_verified",+ "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,+ }+ ]++ for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):+ for index, ref in enumerate(_artifact_mutable_rows(artifact.get(field_name))):+ if not str(ref.get(ref_key) or "").strip():+ prefix = "evidence" if ref_key == "evidence_ref" else "observation"+ ref[ref_key] = f"{prefix}:{fragment}_{index}"+ if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):+ ref["dependency_id"] = primary_dependency_id+ status = str(ref.get("status") or "").strip()+ if status != "missing" and not str(ref.get("source_tool") or "").strip():+ ref["source_tool"] = SCOUT_INTERACTION_EVIDENCE_TOOL+ _drop_contradictory_checkpoint_mode(ref)++ if isinstance(artifact.get("exploration_observations"), list):+ observations: list[dict[str, Any]] = []+ for index, observation in enumerate(_artifact_mutable_rows(artifact["exploration_observations"])):+ if not str(observation.get("text") or "").strip():+ continue+ if not str(observation.get("id") or "").strip():+ observation["id"] = f"exploration:{fragment}_{index}"+ if observation.get("checkpoint_next_mode") == "advance":+ observation.pop("checkpoint_next_mode", None)+ observations.append(observation)+ artifact["exploration_observations"] = observations+++def _block_goal_fallback(label: str, block: Mapping[str, Any]) -> str:+ title = str(block.get("title") or "").strip()+ return title or label.replace("_", " ").strip() or label+++def _imposed_artifact_skeleton(+ label: str,+ block: Mapping[str, Any],+ scout_trajectory: list[ScoutedInteraction],+) -> dict[str, Any]:+ artifact: dict[str, Any] = {+ "block_label": label,+ "artifact_id": _artifact_id_for_block_label(label),+ "declared_goal": _block_goal_fallback(label, block),+ }+ block_id = str(block.get("block_id") or block.get("id") or "").strip()+ if block_id:+ artifact["block_id"] = block_id+ _impose_code_artifact_metadata_defaults(label, artifact, scout_trajectory)+ return artifact++def _code_artifact_metadata_items(raw_metadata: Any) -> list[Any]:if isinstance(raw_metadata, Mapping):items: list[Any] = []@@ -278,8 +567,42 @@ def _workflow_yaml_code_blocks_by_label(workflow_yaml: str | None) -> dict[str,def _artifact_id_for_block_label(label: str) -> str:- fragment = re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"- return f"code_artifact:{fragment}"+ return f"code_artifact:{_artifact_label_fragment(label)}"+++def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None) -> list[str]:+ """Run the sandbox's static safety rule on new/changed code blocks before any run.++ Label-scoped diff so legacy code blocks the model did not touch cannot wedge+ every subsequent update."""+ prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)+ errors: list[str] = []+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():+ code = str(block.get("code") or "")+ if not code.strip():+ continue+ prior_block = prior_blocks.get(label)+ if prior_block is not None and str(prior_block.get("code") or "") == code:+ continue+ try:+ CodeBlock.is_safe_code(code)+ except SyntaxError as exc:+ errors.append(f"Code block `{label}` is not valid Python: {exc}")+ except InsecureCodeDetected as exc:+ errors.append(+ f"Code block `{label}` failed the sandbox safety check: {exc}. Rewrite without import "+ "statements, dunder access, or private attributes; the sandbox already provides `page`, "+ "`json`, `re`, `html`, `asyncio.sleep`, and common builtins."+ )+ return errors+++def _code_seam_rejection_user_summary(*, metadata_rejected: bool, code_rejected: bool) -> str:+ if metadata_rejected and code_rejected:+ return "I need to adjust the workflow's code and its verification details before testing."+ if code_rejected:+ return "I need to adjust the workflow's code so it can run safely before testing."+ return "I need to adjust how the workflow verifies its results before testing."def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:@@ -392,16 +715,42 @@ async def _update_workflow(ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))+ scout_trajectory = getattr(ctx, "scout_trajectory", None)code_artifact_metadata, code_artifact_metadata_error = _normalize_code_artifact_metadata(params.get("code_artifact_metadata"),workflow_yaml,+ impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,+ scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,)- if code_artifact_metadata_error is not None:- return {"ok": False, "error": code_artifact_metadata_error}- if code_artifact_metadata:- ctx.code_artifact_metadata = code_artifact_metadata- ctx.workflow_verification_evidence.code_artifact_metadata = code_artifact_metadata- params["code_artifact_metadata"] = code_artifact_metadata+ prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)+ code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)+ # Per-label salvage keeps conforming metadata across a rejection; a+ # rejected code block keeps nothing, since its yaml never becomes the+ # draft. Prior-draft labels survive every rejection gate below — the+ # accept path prunes to the submitted blocks once the draft switches.+ if code_artifact_metadata and not code_safety_errors:+ existing_metadata = getattr(ctx, "code_artifact_metadata", None)+ merged_metadata = {+ **(existing_metadata if isinstance(existing_metadata, dict) else {}),+ **code_artifact_metadata,+ }+ retained_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml)) | set(+ _workflow_yaml_code_blocks_by_label(prior_workflow_yaml)+ )+ merged_metadata = {block: row for block, row in merged_metadata.items() if block in retained_labels}+ ctx.code_artifact_metadata = merged_metadata+ ctx.workflow_verification_evidence.code_artifact_metadata = merged_metadata+ params["code_artifact_metadata"] = merged_metadata+ seam_errors = [error for error in (code_artifact_metadata_error, *code_safety_errors) if error]+ if seam_errors:+ return {+ "ok": False,+ "error": "\n".join(seam_errors),+ "user_facing_summary": _code_seam_rejection_user_summary(+ metadata_rejected=code_artifact_metadata_error is not None,+ code_rejected=bool(code_safety_errors),+ ),+ }if allow_missing_credentials is None:allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is Trueif not allow_missing_credentials:@@ -521,6 +870,13 @@ async def _update_workflow(ctx.staged_workflow = workflowctx.has_staged_proposal = Truectx.workflow_yaml = workflow_yaml+ accepted_metadata = getattr(ctx, "code_artifact_metadata", None)+ if isinstance(accepted_metadata, dict) and accepted_metadata:+ accepted_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))+ pruned_metadata = {block: row for block, row in accepted_metadata.items() if block in accepted_labels}+ if pruned_metadata != accepted_metadata:+ ctx.code_artifact_metadata = pruned_metadata+ ctx.workflow_verification_evidence.code_artifact_metadata = pruned_metadata# Best-effort — narrative emit failures must never abort an# otherwise-successful update_workflow tool call. ``isinstance``# narrows the parameter's declared ``AgentContext`` to thediff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex 76a3e1875..bce793bac 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -1182,7 +1182,10 @@ workflow_definition:)process_mock.assert_not_called()- assert "inspect_page_for_composition" in agent_result.user_response+ # The reject note must be product language, never the gate's+ # agent-directed tool instruction.+ assert "(Note:" in agent_result.user_response+ assert "inspect_page_for_composition" not in agent_result.user_responseassert agent_result.updated_workflow is Noneassert agent_result.workflow_yaml is Nonediff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.pyindex fcf278f41..f6e159259 100644--- a/tests/unit/test_copilot_blocker_signal.py+++ b/tests/unit/test_copilot_blocker_signal.py@@ -91,6 +91,20 @@ def test_assert_clean_passes_normal_product_language() -> None:assert_clean_user_facing_text("I couldn't complete that on this turn.")+def test_assert_clean_raises_on_internal_budget_vocabulary() -> None:+ with pytest.raises(ValueError):+ assert_clean_user_facing_text("The run exceeded the 6s per-tool-call budget while still making progress.")+++def test_assert_clean_raises_on_raw_run_id() -> None:+ with pytest.raises(ValueError):+ assert_clean_user_facing_text("Run ID: wr_538438176486379954. Outcome is uncertain.")+++def test_assert_clean_allows_prose_mentioning_runs_without_ids() -> None:+ assert_clean_user_facing_text("The last run didn't finish; I stopped without claiming results.")++class _Ctx:blocker_signal: CopilotToolBlockerSignal | None = Nonelatest_tool_blocker_signal: CopilotToolBlockerSignal | None = Nonediff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.pyindex 10892f6f8..a10d9de81 100644--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py@@ -28,6 +28,38 @@ def _violation_count(error: str) -> int:return len(re.findall(r"^\d+\.", error, flags=re.M))+def _valid_metadata(label: str) -> dict:+ return {+ "block_label": label,+ "artifact_id": f"code_artifact:{label}",+ "declared_goal": "g",+ "claimed_outcomes": [+ {+ "id": "claim:x",+ "scope": "outcome",+ "text": "x",+ "status": "observed_not_verified",+ "depends_on": ["dependency:p"],+ "covered_criteria": ["criterion:c"],+ "observation_refs": ["obs1"],+ }+ ],+ "page_dependencies": [+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}+ ],+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],+ "observation_refs": [+ {+ "observation_ref": "obs1",+ "dependency_id": "dependency:p",+ "status": "observed_not_verified",+ "source_tool": "scout_interaction",+ }+ ],+ }++def _broken_metadata(label: str) -> dict:return {"block_label": label,@@ -52,46 +84,25 @@ class TestAccumulateAllViolations:assert _violation_count(error) >= 5assert error.startswith("Artifact metadata has ")assert "fix all of them in one update" in error- assert "`artifact_id` to start with `code_artifact:`" in errorassert "requires `source_tool`" in errorassert "requires `depends_on`" in errorassert "is `satisfied` but has no" in error+ def test_non_conforming_artifact_id_is_imposed_not_rejected(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata["artifact_id"] = "not-prefixed"+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))+ assert error is None+ assert normalized["my_block"]["artifact_id"] == "code_artifact:my_block"+def test_single_violation_is_not_numbered(self) -> None:- metadata = {- "block_label": "my_block",- "artifact_id": "not-prefixed",- "declared_goal": "g",- "claimed_outcomes": [- {- "id": "claim:x",- "scope": "outcome",- "text": "x",- "status": "observed_not_verified",- "depends_on": ["dependency:p"],- "covered_criteria": ["criterion:c"],- "observation_refs": ["obs1"],- }- ],- "page_dependencies": [- {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}- ],- "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],- "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],- "observation_refs": [- {- "observation_ref": "obs1",- "dependency_id": "dependency:p",- "status": "observed_not_verified",- "source_tool": "scout_interaction",- }- ],- }+ metadata = _valid_metadata("my_block")+ metadata["claimed_outcomes"][0].pop("depends_on")normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))assert normalized == {}assert error is not Noneassert _violation_count(error) == 0- assert error == "Artifact metadata for `my_block` requires `artifact_id` to start with `code_artifact:`."+ assert error == "Artifact metadata claim `claim:x` for `my_block` requires `depends_on`."def test_violations_aggregate_across_multiple_artifacts(self) -> None:yaml = textwrap.dedent(@@ -116,50 +127,302 @@ class TestAccumulateAllViolations:assert "block_one" in errorassert "block_two" in error- def test_unknown_label_does_not_short_circuit_other_artifacts(self) -> None:+ def test_unknown_label_is_dropped_and_other_artifacts_still_validated(self) -> None:normalized, error = _normalize_code_artifact_metadata([_broken_metadata("ghost_label"), _broken_metadata("my_block")], _code_block_yaml("my_block"))assert normalized == {}assert error is not None- assert "does not reference an existing code block label" in error- # The second artifact's shape violations are still surfaced.+ # The stale entry is pruned, never rejected; the anchored artifact's+ # shape violations are still surfaced.+ assert "ghost_label" not in errorassert "requires `source_tool`" in errordef test_valid_metadata_passes(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("my_block")], _code_block_yaml("my_block")+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_empty_metadata_is_noop(self) -> None:+ assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)+ assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)+++_SCOUT_TRAJECTORY = [+ {+ "tool_name": "click",+ "selector": "#search-submit",+ "source_url": "https://registry.example.com/search",+ "role": "button",+ "accessible_name": "Search",+ "trajectory_index": 0,+ },+]+++def _assert_passes_full_validator(row: dict) -> None:+ """The imposed row must conform without any trajectory-driven defaulting."""+ renormalized, error = _normalize_code_artifact_metadata([row], _code_block_yaml(row["block_label"]))+ assert error is None, error+ assert list(renormalized.keys()) == [row["block_label"]]+++class TestSeamImposition:+ def test_minimal_metadata_is_fully_defaulted(self) -> None:+ metadata = {"block_label": "my_block", "declared_goal": "Search the registry and expand result rows"}+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ row = normalized["my_block"]+ assert row["artifact_id"] == "code_artifact:my_block"+ assert row["page_dependencies"]+ assert row["claimed_outcomes"]+ assert row["completion_criteria"]+ assert row["terminal_verifier_expectations"]+ assert row["observation_refs"]+ _assert_passes_full_validator(row)++ def test_omitted_page_dependencies_filled_from_trajectory(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("page_dependencies")+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ dependency = normalized["my_block"]["page_dependencies"][0]+ assert dependency["status"] == "observed_not_verified"+ assert dependency["url_hint"] == "https://registry.example.com/search"+ _assert_passes_full_validator(normalized["my_block"])++ def test_omitted_claims_derived_from_declared_goal(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("claimed_outcomes")+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ claim = normalized["my_block"]["claimed_outcomes"][0]+ assert claim["text"] == "g"+ assert claim["status"] == "observed_not_verified"+ assert claim["depends_on"] == ["dependency:p"]+ _assert_passes_full_validator(normalized["my_block"])++ def test_omitted_criteria_derived_from_claims(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("completion_criteria")+ metadata["claimed_outcomes"][0].pop("covered_criteria")+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ row = normalized["my_block"]+ criterion = row["completion_criteria"][0]+ assert criterion["level"] == "terminal"+ assert criterion["text"] == "x"+ assert row["claimed_outcomes"][0]["covered_criteria"] == [criterion["id"]]+ _assert_passes_full_validator(row)++ def test_omitted_expectations_linked_to_criteria(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("terminal_verifier_expectations")+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ expectation = normalized["my_block"]["terminal_verifier_expectations"][0]+ assert expectation["criteria_ids"] == ["criterion:c"]+ _assert_passes_full_validator(normalized["my_block"])++ def test_omitted_artifact_refs_default_to_scout_observation(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("observation_refs")+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ ref = normalized["my_block"]["observation_refs"][0]+ assert ref["source_tool"] == "scout_interaction"+ assert ref["status"] == "observed_not_verified"+ _assert_passes_full_validator(normalized["my_block"])++ def test_missing_ref_source_tool_and_scope_filled(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata["observation_refs"] = [{"observation_ref": "obs1", "status": "observed_not_verified"}]+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ ref = normalized["my_block"]["observation_refs"][0]+ assert ref["source_tool"] == "scout_interaction"+ assert ref["dependency_id"] == "dependency:p"+ _assert_passes_full_validator(normalized["my_block"])++ def test_contradictory_checkpoint_advance_dropped(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata["observation_refs"][0]["checkpoint_next_mode"] = "advance"+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ assert "checkpoint_next_mode" not in normalized["my_block"]["observation_refs"][0]++ def test_satisfied_claim_without_evidence_is_undefaultable(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata["claimed_outcomes"][0]["status"] = "satisfied"+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert normalized == {}+ assert error is not None+ assert "is `satisfied` but has no" in error++ def test_no_imposition_when_not_imposing(self) -> None:+ metadata = {"block_label": "my_block", "declared_goal": "g"}+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))+ assert normalized == {}+ assert error is not None+ assert "requires non-empty" in error++ def test_imposition_works_without_scout_trajectory(self) -> None:+ # The run-3 class: model-authored per-entry refs missing the scoped id+ # and source_tool must be normalized, not rejected, even when no scout+ # interaction was recorded before authoring.+ metadata = {+ "block_label": "my_block",+ "declared_goal": "g",+ "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],+ }+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True+ )+ assert error is None+ ref = normalized["my_block"]["observation_refs"][0]+ assert ref["dependency_id"]+ assert ref["source_tool"] == "scout_interaction"+ assert "url_hint" not in normalized["my_block"]["page_dependencies"][0]+ _assert_passes_full_validator(normalized["my_block"])++ def test_imposition_fills_missing_mechanical_ids(self) -> None:metadata = {"block_label": "my_block",- "artifact_id": "code_artifact:my_block","declared_goal": "g",- "claimed_outcomes": [- {- "id": "claim:x",- "scope": "outcome",- "text": "x",- "status": "observed_not_verified",- "depends_on": ["dependency:p"],- "covered_criteria": ["criterion:c"],- "observation_refs": ["obs1"],- }- ],- "page_dependencies": [- {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}- ],- "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],- "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],- "observation_refs": [- {- "observation_ref": "obs1",- "dependency_id": "dependency:p",- "status": "observed_not_verified",- "source_tool": "scout_interaction",- }- ],+ "claimed_outcomes": [{"text": "rows visible"}],+ "completion_criteria": [{"text": "rows shown"}],}+ normalized, error = _normalize_code_artifact_metadata(+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY+ )+ assert error is None+ row = normalized["my_block"]+ assert row["claimed_outcomes"][0]["id"]+ assert row["claimed_outcomes"][0]["scope"] == "outcome"+ assert row["completion_criteria"][0]["id"]+ _assert_passes_full_validator(row)+++class TestPerLabelSalvage:+ def test_conforming_label_survives_offending_label(self) -> None:+ yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: block_one+ code: |+ await page.goto("https://example.com/")+ - block_type: code+ label: block_two+ code: |+ await page.goto("https://example.com/")+ """+ ).strip()+ bad = _valid_metadata("block_two")+ bad["claimed_outcomes"][0]["status"] = "satisfied"+ normalized, error = _normalize_code_artifact_metadata([_valid_metadata("block_one"), bad], yaml)+ assert list(normalized.keys()) == ["block_one"]+ assert error is not None+ assert "block_two" in error+ assert "block_one" not in error++ def test_unknown_label_dropped_alone(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("ghost_label"), _valid_metadata("my_block")], _code_block_yaml("my_block")+ )+ assert list(normalized.keys()) == ["my_block"]+ assert error is None+++class TestStaleLabelRekey:+ def test_single_stale_entry_rekeys_to_single_uncovered_label(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("stale_label")], _code_block_yaml("my_block")+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]+ assert normalized["my_block"]["block_label"] == "my_block"+ assert normalized["my_block"]["artifact_id"] == "code_artifact:my_block"++ def test_label_less_entry_rekeys_to_single_uncovered_label(self) -> None:+ metadata = _valid_metadata("my_block")+ metadata.pop("block_label")normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))assert error is Noneassert list(normalized.keys()) == ["my_block"]- def test_empty_metadata_is_noop(self) -> None:- assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)- assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)+ def test_multiple_stale_entries_are_dropped_not_rekeyed(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("ghost_a"), _valid_metadata("ghost_b")], _code_block_yaml("my_block")+ )+ assert normalized == {}+ assert error is None++ def test_dropped_stale_entry_leaves_skeleton_for_uncovered_label_when_imposing(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("ghost_a"), _valid_metadata("ghost_b")],+ _code_block_yaml("my_block"),+ impose_defaults=True,+ scout_trajectory=_SCOUT_TRAJECTORY,+ )+ assert error is None+ assert list(normalized.keys()) == ["my_block"]+ row = normalized["my_block"]+ assert row["declared_goal"]+ assert row["page_dependencies"]+ assert row["observation_refs"]+ _assert_passes_full_validator(row)++ def test_partial_coverage_gets_skeleton_for_uncovered_label_when_imposing(self) -> None:+ yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: block_one+ code: |+ await page.goto("https://example.com/")+ - block_type: code+ label: block_two+ code: |+ await page.goto("https://example.com/")+ """+ ).strip()+ normalized, error = _normalize_code_artifact_metadata(+ [_valid_metadata("block_one")],+ yaml,+ impose_defaults=True,+ scout_trajectory=_SCOUT_TRAJECTORY,+ )+ assert error is None+ assert sorted(normalized.keys()) == ["block_one", "block_two"]+ _assert_passes_full_validator(normalized["block_two"])++ def test_duplicate_label_keeps_first_entry(self) -> None:+ first = _valid_metadata("my_block")+ second = _valid_metadata("my_block")+ second["declared_goal"] = "different"+ normalized, error = _normalize_code_artifact_metadata([first, second], _code_block_yaml("my_block"))+ assert error is None+ assert normalized["my_block"]["declared_goal"] == "g"diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pynew file mode 100644index 000000000..7d0995437--- /dev/null+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -0,0 +1,268 @@+"""Tests for the code-block persist seam in `_update_workflow`.++OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.+"""++from __future__ import annotations++import textwrap+from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.copilot.blocker_signal import assert_clean_user_facing_text+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.tools import (+ _code_block_safety_errors,+ _detect_stale_block_metadata,+ _update_workflow,+)+++def _yaml(body: str) -> str:+ return textwrap.dedent(body).strip() + "\n"+++_IMPORTING_CODE_YAML = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: search_registry+ code: |+ import asyncio+ await page.goto("https://example.com/search")+ """+)++_SAFE_CODE_YAML = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: search_registry+ code: |+ await page.goto("https://example.com/search")+ """+)+++def _code_only_ctx() -> CopilotContext:+ ctx = CopilotContext(+ organization_id="o",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_yaml="",+ browser_session_id=None,+ stream=SimpleNamespace(), # type: ignore[arg-type]+ )+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ctx.scout_trajectory = [+ {+ "tool_name": "click",+ "selector": "#search-submit",+ "source_url": "https://example.com/search",+ "trajectory_index": 0,+ }+ ]+ return ctx+++class TestCodeSafetySeam:+ def test_import_in_new_code_block_is_a_seam_error(self) -> None:+ errors = _code_block_safety_errors(_IMPORTING_CODE_YAML, None)+ assert len(errors) == 1+ assert "search_registry" in errors[0]+ assert "Not allowed to import modules" in errors[0]++ def test_unchanged_legacy_code_block_is_not_rechecked(self) -> None:+ assert _code_block_safety_errors(_IMPORTING_CODE_YAML, _IMPORTING_CODE_YAML) == []++ def test_changed_code_block_is_rechecked(self) -> None:+ assert _code_block_safety_errors(_IMPORTING_CODE_YAML, _SAFE_CODE_YAML)++ def test_safe_code_passes(self) -> None:+ assert _code_block_safety_errors(_SAFE_CODE_YAML, None) == []++ def test_syntax_error_is_a_seam_error(self) -> None:+ broken = _SAFE_CODE_YAML.replace('await page.goto("https://example.com/search")', "await page.goto(")+ errors = _code_block_safety_errors(broken, None)+ assert len(errors) == 1+ assert "not valid Python" in errors[0]++ @pytest.mark.asyncio+ async def test_update_workflow_rejects_import_before_any_run(self) -> None:+ ctx = _code_only_ctx()+ result = await _update_workflow({"workflow_yaml": _IMPORTING_CODE_YAML}, ctx)+ assert result["ok"] is False+ assert "Not allowed to import modules" in result["error"]+ assert "import" not in result["user_facing_summary"]+ assert result["user_facing_summary"]++ @pytest.mark.asyncio+ async def test_code_rejection_does_not_salvage_metadata_into_ctx(self) -> None:+ ctx = _code_only_ctx()+ metadata = [{"block_label": "search_registry", "declared_goal": "search the registry"}]+ result = await _update_workflow(+ {"workflow_yaml": _IMPORTING_CODE_YAML, "code_artifact_metadata": metadata}, ctx+ )+ assert result["ok"] is False+ assert ctx.code_artifact_metadata == {}+++class TestSeamSalvageIntoContext:+ @pytest.mark.asyncio+ async def test_conforming_label_persists_when_sibling_label_rejects(self) -> None:+ workflow_yaml = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: block_one+ code: |+ await page.goto("https://example.com/search")+ - block_type: code+ label: block_two+ code: |+ await page.goto("https://example.com/results")+ """+ )+ metadata = [+ {"block_label": "block_one", "declared_goal": "search the registry"},+ {+ "block_label": "block_two",+ "declared_goal": "expand the result rows",+ "claimed_outcomes": [+ {"id": "claim:rows", "scope": "outcome", "text": "rows expanded", "status": "satisfied"}+ ],+ },+ ]+ ctx = _code_only_ctx()+ result = await _update_workflow({"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata}, ctx)++ assert result["ok"] is False+ assert "block_two" in result["error"]+ assert "block_one" not in result["error"]+ assert list(ctx.code_artifact_metadata.keys()) == ["block_one"]+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata+ assert "contract violation" not in result["user_facing_summary"]+ assert "`" not in result["user_facing_summary"]++ @pytest.mark.asyncio+ async def test_stale_entry_dropped_and_draft_metadata_survives_unaccepted_submission(self) -> None:+ draft_yaml = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: block_a+ code: |+ await page.goto("https://example.com/search")+ - block_type: code+ label: block_b+ code: |+ await page.goto("https://example.com/results")+ """+ )+ submitted_yaml = _yaml(+ """+ title: Registry lookup+ workflow_definition:+ blocks:+ - block_type: code+ label: block_a+ code: |+ await page.goto("https://example.com/search")+ """+ )+ ctx = _code_only_ctx()+ ctx.workflow_yaml = draft_yaml+ stored_block_b = {"block_label": "block_b", "declared_goal": "expand the result rows"}+ ctx.code_artifact_metadata = {+ "block_a": {"block_label": "block_a", "declared_goal": "search the registry"},+ "block_b": stored_block_b,+ }+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)+ metadata = [+ {"block_label": "block_a", "declared_goal": "search the registry"},+ {"block_label": "ghost", "declared_goal": "does not exist"},+ ]++ result = await _update_workflow({"workflow_yaml": submitted_yaml, "code_artifact_metadata": metadata}, ctx)++ # The stale entry is pruned at the seam; the submission proceeds and+ # only non-metadata gates may reject it, so prior-draft metadata stays.+ error_text = str(result.get("error") or "")+ assert "Artifact metadata" not in error_text+ assert "ghost" not in error_text+ assert sorted(ctx.code_artifact_metadata) == ["block_a", "block_b"]+ assert ctx.code_artifact_metadata["block_b"] == stored_block_b+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata++ @pytest.mark.asyncio+ async def test_minimal_metadata_with_trajectory_produces_no_violation_error(self) -> None:+ ctx = _code_only_ctx()+ metadata = [{"block_label": "search_registry", "declared_goal": "search the registry"}]+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)+ # The seam may reject later (credential checks need the app); the metadata+ # contract itself must not be the rejection.+ error_text = str(result.get("error") or "")+ assert "Artifact metadata" not in error_text+ assert "contract violation" not in error_text+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"+++class TestStaleLabelSeamFlow:+ @pytest.mark.asyncio+ async def test_stale_metadata_label_rekeys_without_any_stale_rejection(self) -> None:+ # Run-2 shape: metadata keyed to a label absent from the submitted+ # YAML. The seam re-keys it to the only uncovered code block, so+ # neither the metadata gate nor the stale-block-metadata validation+ # path can bounce the submission back to the model.+ ctx = _code_only_ctx()+ ctx.workflow_yaml = _SAFE_CODE_YAML+ metadata = [{"block_label": "search_certificant_stale", "declared_goal": "search the registry"}]++ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)++ error_text = str(result.get("error") or "")+ assert "Artifact metadata" not in error_text+ assert "still appears stale" not in error_text+ assert list(ctx.code_artifact_metadata.keys()) == ["search_registry"]+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"+ # The seam never rewrites YAML labels, so its output cannot trip the+ # stale-block-metadata validation that fires on label/title renames.+ assert _detect_stale_block_metadata(_SAFE_CODE_YAML, ctx.workflow_yaml) == []++ @pytest.mark.asyncio+ async def test_malformed_per_entry_refs_normalize_without_scout_interactions(self) -> None:+ # Run-3 shape: model-authored observation_refs rows missing the scoped+ # id, authored before any scout interaction was recorded.+ ctx = _code_only_ctx()+ ctx.scout_trajectory = []+ metadata = [+ {+ "block_label": "search_registry",+ "declared_goal": "search the registry",+ "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],+ }+ ]++ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)++ error_text = str(result.get("error") or "")+ assert "Artifact metadata" not in error_text+ assert "contract violation" not in error_text+ ref = ctx.code_artifact_metadata["search_registry"]["observation_refs"][0]+ assert ref["dependency_id"]+ assert ref["source_tool"]+++def test_run_id_leak_check_covers_non_numeric_ids() -> None:+ with pytest.raises(ValueError):+ assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.pyindex dd2dd66fd..3aa143a93 100644--- a/tests/unit/test_copilot_composition_evidence.py+++ b/tests/unit/test_copilot_composition_evidence.py@@ -2443,3 +2443,109 @@ async def test_capture_prefers_html_parse_over_hollow_structured_on_fallback(monassert "Welcome notice text" in evidence["visible_text_excerpt"]assert evidence["schema_empty_page"] is Trueassert server.calls.count("skyvern_get_html") == 1+++class TestSemanticChallengeSplit:+ def test_passive_vendor_markup_does_not_assert_human_verification(self) -> None:+ html = """+ <html><head><title>Registry search results</title>+ <script src="https://cdn.example/challenge-platform/api.js"></script>+ </head><body>+ <form><input name="q" /><button type="submit" disabled>Search</button></form>+ <table id="results"><tbody><tr><td>SAMPLE, PERSON</td></tr></tbody></table>+ </body></html>+ """+ parsed = parse_composition_html(+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"+ )+ state = parsed["challenge_state"]+ assert state["detected"] is True+ assert state["requires_human_verification"] is False+ assert state["gates_submit_controls"] is False+ # Substring detection still arms the visual fallback for confirmation.+ assert page_evidence_needs_visual_fallback(parsed) is True++ def test_rendered_challenge_widget_asserts_human_verification(self) -> None:+ html = """+ <html><head><title>Verify you are human</title></head><body>+ <form>+ <input name="q" />+ <div id="cf-turnstile-widget" data-sitekey="abc"></div>+ <button type="submit" disabled>Search</button>+ </form>+ </body></html>+ """+ parsed = parse_composition_html(+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"+ )+ state = parsed["challenge_state"]+ assert state["detected"] is True+ assert state["requires_human_verification"] is True+ assert state["gates_submit_controls"] is True++ def test_consent_typed_visual_obstruction_does_not_promote_challenge(self) -> None:+ html = """+ <html><head><title>Search</title>+ <script src="https://cdn.example/challenge-platform/api.js"></script>+ </head><body>+ <form><input name="q" /><button type="submit" disabled>Search</button></form>+ </body></html>+ """+ parsed = parse_composition_html(+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"+ )+ merged = merge_visual_composition_evidence(+ parsed,+ visual_summary={+ "summary": "A privacy settings dialog covers the page.",+ "challenge_detected": True,+ "challenge_kind": "",+ "challenge_location": "",+ "submit_blocked": True,+ "blocked_submit_controls": ["Search"],+ "page_obstruction_detected": True,+ "obstruction_kind": "cookie_consent",+ "obstruction_location": "center",+ "underlying_page_blocked": True,+ "visible_dismiss_controls": ["Accept all"],+ "omissions": [],+ },+ )+ state = merged["challenge_state"]+ assert state["requires_human_verification"] is False+ assert state["gates_submit_controls"] is False+ kinds = [obstruction.get("kind") for obstruction in merged["page_obstructions"]]+ assert "cookie_consent" in kinds++ def test_vision_confirmation_still_promotes_challenge(self) -> None:+ html = """+ <html><head><title>Search</title>+ <script src="https://cdn.example/challenge-platform/api.js"></script>+ </head><body>+ <form><input name="q" /><button type="submit" disabled>Search</button></form>+ </body></html>+ """+ parsed = parse_composition_html(+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"+ )+ merged = merge_visual_composition_evidence(+ parsed,+ visual_summary={+ "summary": "A human-verification widget sits above the Search button.",+ "challenge_detected": True,+ "challenge_kind": "human_verification",+ "challenge_location": "above the Search button",+ "submit_blocked": True,+ "blocked_submit_controls": ["Search"],+ "page_obstruction_detected": False,+ "obstruction_kind": "",+ "obstruction_location": "",+ "underlying_page_blocked": None,+ "visible_dismiss_controls": [],+ "omissions": [],+ },+ )+ state = merged["challenge_state"]+ assert state["requires_human_verification"] is True+ assert state["gates_submit_controls"] is True+ assert {"text": "Search", "disabled": True} in state["gated_submit_controls"]diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.pyindex 8e596a7fe..6ad1a68e5 100644--- a/tests/unit/test_copilot_per_tool_budget.py+++ b/tests/unit/test_copilot_per_tool_budget.py@@ -43,6 +43,8 @@ from skyvern.forge.sdk.copilot.tools import (_mark_page_inspected,_mark_pending_reconciliation_run,_maybe_clear_reconciliation_flag,+ _post_budget_terminal_challenge_signal,+ _post_run_terminal_challenge_reason,_record_composition_page_observation,_record_per_tool_budget_problem_blocks_from_results,_record_run_blocks_result,@@ -1069,3 +1071,103 @@ def test_nudge_counter_preserved_across_consecutive_budget_trips() -> None:assert ctx.per_tool_budget_nudge_count == 1assert ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY+++def _challenge_gated_post_run_evidence(**extra: object) -> dict:+ evidence = {+ "current_url": "https://example.com/registry/search",+ "workflow_run_id": "wr_budget",+ "observed_after_workflow_run": True,+ "anti_bot_indicators": ["human-verification"],+ "forms": [{"submit_controls": [{"text": "Search", "selector": "#search", "disabled": True}]}],+ "challenge_state": {+ "detected": True,+ "kind": "human_verification",+ "requires_human_verification": True,+ "gates_submit_controls": True,+ "gated_submit_controls": [{"text": "Search", "disabled": True}],+ },+ }+ evidence.update(extra)+ return evidence+++def _armed_post_run_context() -> CopilotContext:+ ctx = _fresh_context()+ ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY+ ctx.last_test_ok = False+ ctx.per_tool_budget_problem_block_labels = ["submit_registry_search"]+ return ctx+++def test_terminal_challenge_routes_to_answer_path_when_result_evidence_in_same_packet() -> None:+ ctx = _armed_post_run_context()+ ctx.composition_page_evidence = _challenge_gated_post_run_evidence(+ result_containers=[{"tag": "table", "selector": "#results", "row_selector": "#results tbody tr"}],+ visible_text_excerpt="SAMPLE, PERSON Anytown, XY Exampleland Certified Active",+ )++ signal = _post_budget_terminal_challenge_signal(+ ctx, {"block_labels": ["submit_registry_search"]}, "update_and_run_blocks"+ )++ assert signal is not None+ assert signal.internal_reason_code == "tool_error_post_budget_challenge_result_evidence"+ assert signal.renders_final_reply is False+ assert signal.preserves_workflow_draft is True+ assert "Answer from the observed page" in signal.agent_steering_text+ assert "result container" in signal.agent_steering_text+++def test_terminal_challenge_fires_without_result_evidence_and_names_observed_facts() -> None:+ ctx = _armed_post_run_context()+ ctx.composition_page_evidence = _challenge_gated_post_run_evidence()++ signal = _post_budget_terminal_challenge_signal(ctx, None, "update_and_run_blocks")++ assert signal is not None+ assert signal.internal_reason_code == "tool_error_post_budget_challenge_blocker"+ assert signal.renders_final_reply is True+ assert "human verification" in signal.user_facing_reason+ assert "Search" in signal.user_facing_reason+ assert "per-tool-call budget" not in signal.user_facing_reason+ assert "wr_" not in signal.user_facing_reason+++def test_bare_substring_indicators_do_not_produce_terminal_reason() -> None:+ # A passive challenge-vendor token over raw HTML plus an unrelated disabled+ # submit control must not assert a terminal blocked claim.+ evidence = _challenge_gated_post_run_evidence(+ challenge_state={+ "detected": True,+ "kind": "human_verification",+ "requires_human_verification": False,+ "gates_submit_controls": False,+ "gated_submit_controls": [],+ },+ )+ evidence["anti_bot_indicators"] = ["challenge"]+ evidence["challenge_controls"] = [{"tag": "script", "src": "https://cdn.example/challenge-platform/api.js"}]++ assert _post_run_terminal_challenge_reason(evidence) is None++ ctx = _armed_post_run_context()+ ctx.composition_page_evidence = evidence+ assert _post_budget_terminal_challenge_signal(ctx, None, "update_and_run_blocks") is None+++def test_rendered_challenge_widget_still_produces_terminal_reason() -> None:+ evidence = _challenge_gated_post_run_evidence(+ challenge_state={+ "detected": True,+ "kind": "captcha",+ "requires_human_verification": False,+ "gates_submit_controls": False,+ "gated_submit_controls": [],+ },+ )+ evidence["challenge_controls"] = [{"tag": "div", "class": "cf-turnstile", "data_sitekey": "abc"}]++ reason = _post_run_terminal_challenge_reason(evidence)+ assert reason is not None+ assert "disabled" in reasondiff --git a/tests/unit/test_copilot_terminal_halt_narrative.py b/tests/unit/test_copilot_terminal_halt_narrative.pynew file mode 100644index 000000000..1fd6843b4--- /dev/null+++ b/tests/unit/test_copilot_terminal_halt_narrative.py@@ -0,0 +1,155 @@+"""Terminal-output invariant and observed-facts halt narrative tests.++OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest++from skyvern.forge.sdk.copilot.agent import (+ _build_wip_exit_result,+ _clean_recorded_failure_text,+ _observed_facts_halt_reply,+ _recorded_failure_reply,+)+from skyvern.forge.sdk.copilot.blocker_signal import contains_internal_machinery_leak+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY+from skyvern.forge.sdk.copilot.output_policy import (+ CopilotOutputKind,+ OutputPolicyReason,+ evaluate_output_policy,+)++_ANTI_RERUN_GUARD_TEXT = (+ "The prior PER_TOOL_BUDGET run for run wr_538488327000571062 advanced the live browser at "+ "https://www.example.com/. Before another block-running tool, inspect the current browser page with "+ 'inspect_page_for_composition(target_url="current_page").'+)++_PACING_GUARD_TEXT = "The run exceeded the 3s per-tool-call budget while still making progress."+++def _ctx() -> CopilotContext:+ return CopilotContext(+ organization_id="o",+ workflow_id="w",+ workflow_permanent_id="wp",+ workflow_yaml="",+ browser_session_id=None,+ stream=SimpleNamespace(), # type: ignore[arg-type]+ )+++class TestInternalMachineryLeakPredicate:+ @pytest.mark.parametrize(+ "text",+ [+ _ANTI_RERUN_GUARD_TEXT,+ _PACING_GUARD_TEXT,+ "Outcome is uncertain for run wr_538438176486379954.",+ "Browser session pbs_538476723643067648 is live.",+ "Before another block-running tool, inspect the current browser page.",+ "Call update_and_run_blocks with the smaller chain.",+ "Use inspect_page_for_composition to confirm the page state.",+ ],+ )+ def test_internal_machinery_is_detected(self, text: str) -> None:+ assert contains_internal_machinery_leak(text) is True++ @pytest.mark.parametrize(+ "text",+ [+ "I created and tested the workflow successfully.",+ "I built a 2-block draft and was still testing it when the turn ran out of time.",+ "The site's verification page was still blocking the search control.",+ "Send me the registry URL you want to search.",+ "",+ ],+ )+ def test_product_language_is_clean(self, text: str) -> None:+ assert contains_internal_machinery_leak(text) is False++ def test_output_policy_hard_blocks_guard_text(self) -> None:+ verdict = evaluate_output_policy(+ request_policy=None,+ response_type="REPLY",+ user_response=_ANTI_RERUN_GUARD_TEXT,+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,+ )+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes+++class TestRecordedFailureReply:+ def test_guard_text_renders_observed_facts_not_test_failed(self) -> None:+ ctx = _ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))+ ctx.last_update_block_count = 1+ ctx.last_test_ok = False+ ctx.last_test_failure_reason = _ANTI_RERUN_GUARD_TEXT+ ctx.workflow_verification_evidence.current_url = "https://www.example.com/registry"++ reply = _recorded_failure_reply(ctx)++ assert reply is not None+ assert "wr_" not in reply+ assert "PER_TOOL_BUDGET" not in reply+ assert "per-tool-call budget" not in reply+ assert "block-running tool" not in reply+ assert "the test failed" not in reply+ assert "ran out of time" in reply+ assert "1-block draft" in reply+ assert "https://www.example.com/registry" in reply++ def test_budget_paced_run_does_not_render_as_test_failed(self) -> None:+ ctx = _ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))+ ctx.last_update_block_count = 1+ ctx.last_test_ok = False+ ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY+ ctx.last_test_failure_reason = "The run was canceled while still making progress."++ reply = _recorded_failure_reply(ctx)++ assert reply is not None+ assert "the test failed" not in reply+ assert "ran out of time" in reply++ def test_fragment_cleaner_substitutes_guard_text(self) -> None:+ cleaned = _clean_recorded_failure_text(_ANTI_RERUN_GUARD_TEXT)+ assert "wr_" not in cleaned+ assert "PER_TOOL_BUDGET" not in cleaned++ def test_observed_facts_reply_without_draft_states_timeout(self) -> None:+ ctx = _ctx()+ reply = _observed_facts_halt_reply(ctx)+ assert "ran out of time" in reply+ assert not contains_internal_machinery_leak(reply)+++class TestWipExitTerminalInvariant:+ def test_timeout_exit_with_guard_failure_reason_renders_clean_terminal(self) -> None:+ ctx = _ctx()+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))+ ctx.last_workflow_yaml = "title: wf\nworkflow_definition:\n blocks: []\n"+ ctx.last_update_block_count = 1+ ctx.last_test_ok = False+ ctx.last_test_failure_reason = _ANTI_RERUN_GUARD_TEXT+ ctx.workflow_verification_evidence.current_url = "https://www.example.com/registry"++ result = _build_wip_exit_result(+ ctx,+ None,+ default_reply="I ran out of time processing your request.",+ unvalidated_reply="I ran out of time; the draft is untested.",+ tested_reply="I ran out of time, but I have a tested draft for you.",+ terminal_reason="timeout",+ )++ assert not contains_internal_machinery_leak(result.user_response)+ assert "the test failed" not in result.user_response+ # The halt surfaces the draft for review instead of discarding it.+ assert result.updated_workflow is ctx.last_workflowdiff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.pyindex 484365139..a19100e72 100644--- a/tests/unit/test_copilot_timeout_wip.py+++ b/tests/unit/test_copilot_timeout_wip.py@@ -311,10 +311,9 @@ class TestBuildUnexpectedErrorExitResult:result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)assert result.user_response.startswith(- "I built a 3-block draft and tested it, but the test couldn't finish: "- "The previous workflow run did not finish before the turn budget expired. "- "Last run status: canceled."+ "I built a 3-block draft and was still testing it when the turn ran out of time.")+ assert "the test failed" not in result.user_responseassert "draft workflow proposal" in result.user_responseassert "Do NOT" not in result.user_responseassert "block-running tools" not in result.user_response