Skip to content

Automate browser based workflows with AI

SKY-10825: impose code-artifact metadata at the persist seam + adjudicate terminal challenge halts (#6496)

Andrew Neilsoned95f65c5e84
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index d7ab2f595..5541ff25e 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -35,6 +35,7 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.copilot.blocker_signal import (
CopilotToolBlockerSignal,
assert_clean_user_facing_text,
+ contains_internal_machinery_leak,
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
@@ -53,6 +54,7 @@ from skyvern.forge.sdk.copilot.context import (
finalize_discovery_counter_in_global_llm_context,
)
from skyvern.forge.sdk.copilot.enforcement import outcome_fully_verified
+from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY
from skyvern.forge.sdk.copilot.outcome_verification_trace import (
finalize_outcome_verification_trace,
record_gate_decision,
@@ -1194,6 +1196,11 @@ _UNBACKED_WORKFLOW_DELIVERY_REPLY = (
"from this turn. Please retry with the target site, page, or workflow requirement."
)
_UNBACKED_WORKFLOW_DELIVERY_PREFIX = "I wasn't able to produce a workflow proposal in this turn."
+
+_INLINE_REJECT_NOTE_FALLBACK = (
+ "This draft didn't pass validation against the live page, so I haven't saved it. "
+ "I'll revise it before proposing again."
+)
_GENERIC_MISSING_CONTEXT_PHRASES = (
"missing details",
"one more detail",
@@ -1396,6 +1403,36 @@ def _workflow_block_count(ctx: CopilotContext) -> int | None:
return len(blocks) if isinstance(blocks, list) and blocks else None
+def _observed_page_sentence(ctx: CopilotContext) -> str:
+ evidence = getattr(ctx, "workflow_verification_evidence", None)
+ url = getattr(evidence, "current_url", None)
+ if not isinstance(url, str) or not url.strip():
+ return ""
+ sentence = f" The last page I observed was {url.strip()[:140]}."
+ return "" if contains_internal_machinery_leak(sentence) else sentence
+
+
+def _observed_facts_halt_reply(ctx: CopilotContext) -> str:
+ block_count = _workflow_block_count(ctx)
+ block_phrase = f"a {block_count}-block draft" if block_count else "a draft"
+ observed = _observed_page_sentence(ctx)
+ if getattr(ctx, "last_workflow", None) is not None:
+ return (
+ f"I built {block_phrase} and was still testing it when the turn ran out of time."
+ f"{observed} I haven't verified the results, so I'm not claiming them."
+ )
+ return (
+ f"The turn ran out of time before I could finish.{observed}"
+ " I haven't verified any results, so I'm not claiming them."
+ )
+
+
+def _halted_mid_progress(ctx: CopilotContext, internal_tool_instruction_failure: bool) -> bool:
+ if internal_tool_instruction_failure:
+ return True
+ return getattr(ctx, "last_failure_category_top", None) == PER_TOOL_BUDGET_FAILURE_CATEGORY
+
+
def _clean_recorded_failure_text(value: Any, max_chars: int = 240) -> str:
# Caller-owned sentence templates add punctuation around these fragments.
text = clean_recorded_failure_text(value, max_chars=max_chars).rstrip(".")
@@ -1438,10 +1475,13 @@ def _recorded_failure_is_internal_tool_instruction(ctx: CopilotContext) -> bool:
for value in candidates:
if not isinstance(value, str) or not value.strip():
continue
+ # Evaluate the redacted form at the same truncation the reply embeds:
+ # standard redaction already neutralizes browser-session references,
+ # so flag only what would still leak.
verdict = evaluate_output_policy(
request_policy=None,
response_type="REPLY",
- user_response=value,
+ user_response=clean_recorded_failure_text(value, max_chars=240),
output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
)
if OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes:
@@ -1561,6 +1601,12 @@ def _recorded_failure_reply(
reason, status_sentence = _recorded_failure_summary(ctx)
if not reason:
return None
+ if internal_tool_instruction_failure is None:
+ internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)
+ # A guard-halted or budget-paced run was interrupted, not disproven; render
+ # observed facts instead of a failure verdict built from internal text.
+ if _halted_mid_progress(ctx, internal_tool_instruction_failure):
+ return _observed_facts_halt_reply(ctx)
run_status = _clean_recorded_failure_text(getattr(verification, "run_status", None), max_chars=80).lower()
block_count = _workflow_block_count(ctx)
@@ -1570,9 +1616,7 @@ def _recorded_failure_reply(
or getattr(ctx, "last_test_ok", None) is not None
or getattr(diagnosis_input, "workflow_run_id", None)
)
- if internal_tool_instruction_failure is None:
- internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)
- test_failed = (ctx.last_test_ok is False or run_status == "failed") and not internal_tool_instruction_failure
+ test_failed = ctx.last_test_ok is False or run_status == "failed"
unrecoverable_stop = next_action == "stop" or failure_type == "unrecoverable_tool_error"
if getattr(ctx, "last_workflow", None) is not None:
@@ -1609,12 +1653,19 @@ def _build_wip_exit_result(
) -> AgentResult:
"""Selected non-success exits surface the most recent successfully parsed workflow."""
internal_tool_instruction_failure = _recorded_failure_is_internal_tool_instruction(ctx)
+ halted_mid_progress = _halted_mid_progress(ctx, internal_tool_instruction_failure)
recorded_failure_reply = _recorded_failure_reply(
ctx, cancelled=cancelled, internal_tool_instruction_failure=internal_tool_instruction_failure
)
effective_terminal = terminal_reason or ("cancel" if cancelled else None)
def _guard(text: str) -> tuple[str, TurnOutcome]:
+ if contains_internal_machinery_leak(text):
+ LOG.warning(
+ "copilot terminal output invariant replaced leaked text",
+ terminal_reason=effective_terminal,
+ )
+ text = _observed_facts_halt_reply(ctx)
return apply_repeated_reply_guard(
final_text=text,
attempted_kind=ResponseKind.CLARIFY,
@@ -1663,14 +1714,14 @@ def _build_wip_exit_result(
if (
ctx.last_workflow is not None
and ctx.last_workflow_yaml
- and (ctx.last_test_ok is not False or internal_tool_instruction_failure)
+ and (ctx.last_test_ok is not False or halted_mid_progress)
and not ctx.last_test_suspicious_success
):
full_test_ok = ctx.last_test_ok is True and ctx.last_full_workflow_test_ok is True
unvalidated = not full_test_ok
if unvalidated and recorded_failure_reply:
reply = recorded_failure_reply
- if internal_tool_instruction_failure:
+ if halted_mid_progress:
reply = _ensure_unvalidated_proposal_affordance(reply)
else:
reply = unvalidated_reply if unvalidated else tested_reply
@@ -1849,6 +1900,10 @@ async def _translate_to_agent_result(
last_workflow = ctx.last_workflow
last_workflow_yaml = ctx.last_workflow_yaml
+ def _with_inline_reject_note(response: Any, detail: str) -> str:
+ note = detail if not contains_internal_machinery_leak(detail) else _INLINE_REJECT_NOTE_FALLBACK
+ return f"{response}\n\n(Note: {note})"
+
if resp_type == "REPLACE_WORKFLOW":
LOG.warning("Agent used inline REPLACE_WORKFLOW instead of update_workflow tool")
workflow_yaml = action_data.get("workflow_yaml", "")
@@ -1889,17 +1944,17 @@ async def _translate_to_agent_result(
wait_block_error = _timing_only_challenge_wait_reject_message(ctx, workflow_yaml)
if wait_block_error:
- user_response = f"{user_response}\n\n(Note: {wait_block_error})"
+ user_response = _with_inline_reject_note(user_response, wait_block_error)
ctx.last_test_ok = None
workflow_yaml = ""
banned_items = _detect_new_banned_blocks(workflow_yaml, ctx.last_workflow_yaml)
if banned_items:
_record_banned_block_reject_span("replace_workflow_inline", banned_items)
- user_response = f"{user_response}\n\n(Note: {_banned_block_reject_message(banned_items)})"
+ user_response = _with_inline_reject_note(user_response, _banned_block_reject_message(banned_items))
workflow_yaml = ""
stale_metadata = _detect_stale_block_metadata(workflow_yaml, ctx.last_workflow_yaml or ctx.workflow_yaml)
if stale_metadata:
- user_response = f"{user_response}\n\n(Note: {_stale_block_metadata_message(stale_metadata)})"
+ user_response = _with_inline_reject_note(user_response, _stale_block_metadata_message(stale_metadata))
ctx.last_test_ok = None
workflow_yaml = ""
composition_evidence_error = composition_page_evidence_error(ctx, workflow_yaml)
@@ -1909,7 +1964,7 @@ async def _translate_to_agent_result(
workflow_permanent_id=ctx.workflow_permanent_id,
target_url=workflow_target_url(workflow_yaml),
)
- user_response = f"{user_response}\n\n(Note: {composition_evidence_error})"
+ user_response = _with_inline_reject_note(user_response, composition_evidence_error)
ctx.last_test_ok = None
workflow_yaml = ""
if workflow_yaml:
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 6c88fc2d0..54728c4f7 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Literal, Protocol
@@ -57,14 +58,53 @@ _LEAK_DENY_TOKENS: tuple[str, ...] = (
"normal instruction",
"like 'continue",
'like "continue',
+ "per-tool-call budget",
)
+# Raw workflow-run and browser-session identifiers are internal; user-facing
+# text must reference runs by what they did, never by id.
+_RUN_ID_LEAK_RE = re.compile(r"\b(?:wr|pbs)_[a-z0-9_]+", re.IGNORECASE)
+
+_INTERNAL_GUARD_TOKENS: tuple[str, ...] = (
+ "per_tool_budget",
+ "per-tool-call budget",
+ "active_run_terminal_evidence",
+ "block-running tool",
+ "block running tool",
+)
+
+_INTERNAL_TOOL_NAME_TOKENS: tuple[str, ...] = (
+ "update_workflow",
+ "update_and_run_blocks",
+ "run_blocks_and_collect_debug",
+ "get_run_results",
+ "inspect_page_for_composition",
+ "discover_workflow_entrypoint",
+ "get_browser_screenshot",
+ "list_credentials",
+)
+
+
+def contains_internal_machinery_leak(value: str | None) -> bool:
+ """String-level terminal-output invariant: user-facing text carries no raw
+ run ids, internal guard tokens, or agent-directed tool references."""
+ if not isinstance(value, str) or not value:
+ return False
+ if _RUN_ID_LEAK_RE.search(value):
+ return True
+ lowered = value.lower()
+ if any(token in lowered for token in _INTERNAL_GUARD_TOKENS):
+ return True
+ return any(token in lowered for token in _INTERNAL_TOOL_NAME_TOKENS)
+
def assert_clean_user_facing_text(value: str, *, blocked_tool: str | None = None) -> None:
lowered = value.lower()
for token in _LEAK_DENY_TOKENS:
if token.lower() in lowered:
raise ValueError(f"blocker user-facing text leaked token {token!r}: {value!r}")
+ if contains_internal_machinery_leak(value):
+ raise ValueError(f"blocker user-facing text leaked internal machinery: {value!r}")
if blocked_tool and blocked_tool.lower() in lowered:
raise ValueError(f"blocker user-facing text leaked tool name {blocked_tool!r}: {value!r}")
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index c90a9e0e6..4a8124853 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -271,9 +271,6 @@ def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> Synthesize
return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes)
-_DEPENDENCY_ID = f"dependency:{_SYNTHESIZED_BLOCK_LABEL}_reached"
-_OBSERVATION_REF_ID = f"observation:{_SYNTHESIZED_BLOCK_LABEL}_scout"
-
# Model-owned slots the synthesizer cannot prove; the model fills these.
_FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"
_FILL_CLAIM_ID = "claim:<fill>"
@@ -282,9 +279,23 @@ _FILL_CRITERION_ID = "criterion:<fill>"
_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"
-def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+def artifact_dependency_id(block_label: str) -> str:
+ return f"dependency:{_slug(block_label)}_reached"
+
+
+def artifact_observation_ref_id(block_label: str) -> str:
+ return f"observation:{_slug(block_label)}_scout"
+
+
+def build_artifact_metadata_skeleton(
+ trajectory: Sequence[Mapping[str, Any]],
+ *,
+ block_label: str,
+) -> dict[str, Any]:
"""Fill only the scout-proven evidence shape (`observed_not_verified`); the terminal goal and
outcomes stay as `<fill>` placeholders the model owns."""
+ dependency_id = artifact_dependency_id(block_label)
+ observation_ref_id = artifact_observation_ref_id(block_label)
# First recorded source_url is the page the synthesized block's leading `page.goto` lands on.
# The trajectory carries only pre-action source pages, not post-action reached URLs, so this is
# an advisory entry-page hint, never a reached-page identity (which SPAs would mis-key anyway).
@@ -294,23 +305,23 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]])
)
page_dependency: dict[str, Any] = {
- "id": _DEPENDENCY_ID,
+ "id": dependency_id,
"scope": "page",
"status": "observed_not_verified",
- "observation_refs": [_OBSERVATION_REF_ID],
+ "observation_refs": [observation_ref_id],
}
if entry_url_hint:
page_dependency["url_hint"] = entry_url_hint
observation_ref: dict[str, Any] = {
- "observation_ref": _OBSERVATION_REF_ID,
- "dependency_id": _DEPENDENCY_ID,
+ "observation_ref": observation_ref_id,
+ "dependency_id": dependency_id,
"status": "observed_not_verified",
"source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,
}
return {
- "block_label": _SYNTHESIZED_BLOCK_LABEL,
+ "block_label": block_label,
"declared_goal": _FILL_DECLARED_GOAL,
"page_dependencies": [page_dependency],
"observation_refs": [observation_ref],
@@ -320,9 +331,9 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]])
"scope": "outcome",
"text": _FILL_CLAIM_TEXT,
"status": "observed_not_verified",
- "depends_on": [_DEPENDENCY_ID],
+ "depends_on": [dependency_id],
"covered_criteria": [_FILL_CRITERION_ID],
- "observation_refs": [_OBSERVATION_REF_ID],
+ "observation_refs": [observation_ref_id],
}
],
"completion_criteria": [
@@ -343,6 +354,10 @@ def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]])
}
+def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+ return build_artifact_metadata_skeleton(trajectory, block_label=_SYNTHESIZED_BLOCK_LABEL)
+
+
def _render_artifact_metadata_block(metadata: Mapping[str, Any]) -> str:
return json.dumps(metadata, indent=2, sort_keys=True)
diff --git a/skyvern/forge/sdk/copilot/composition_evidence.py b/skyvern/forge/sdk/copilot/composition_evidence.py
index b1a89a93a..ea9bd557f 100644
--- a/skyvern/forge/sdk/copilot/composition_evidence.py
+++ b/skyvern/forge/sdk/copilot/composition_evidence.py
@@ -111,22 +111,41 @@ def _challenge_kind(indicators: list[str]) -> str:
return "unknown" if indicators else "none"
+# Tags that carry challenge-vendor markup without rendering a widget. A passive
+# script/meta tag ships on every page behind some CDNs, so it can trigger the
+# visual fallback but never assert human verification by itself.
+_PASSIVE_CHALLENGE_TAGS: frozenset[str] = frozenset({"script", "noscript", "style", "link", "meta"})
+
+
+def interactive_challenge_controls(challenge_controls: list[dict[str, Any]] | None) -> list[dict[str, Any]]:
+ return [
+ control
+ for control in challenge_controls or []
+ if isinstance(control, dict) and str(control.get("tag") or "").lower() not in _PASSIVE_CHALLENGE_TAGS
+ ]
+
+
def _challenge_state(
indicators: list[str],
*,
source: str = DOM_EVIDENCE_SOURCE,
gated_submit_controls: list[dict[str, Any]] | None = None,
+ challenge_controls: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
detected = bool(indicators)
gated_controls = gated_submit_controls or []
+ # Raw-HTML token hits only mark `detected` (triggering the visual fallback);
+ # asserting human verification requires a rendered challenge control or a
+ # later vision confirmation.
+ semantic_challenge = bool(interactive_challenge_controls(challenge_controls))
return {
"detected": detected,
"kind": _challenge_kind(indicators),
"source": source if detected else "",
"indicators": indicators[:8],
- "requires_human_verification": detected,
+ "requires_human_verification": semantic_challenge,
"visual_location": "",
- "gates_submit_controls": bool(detected and gated_controls),
+ "gates_submit_controls": bool(semantic_challenge and gated_controls),
"gated_submit_controls": gated_controls[:5] if detected else [],
}
@@ -165,6 +184,7 @@ def _evidence_metadata(
indicators: list[str] | None = None,
*,
forms: list[dict[str, Any]] | None = None,
+ challenge_controls: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
gated_controls = _gated_submit_controls(forms or [])
return {
@@ -176,6 +196,7 @@ def _evidence_metadata(
"challenge_state": _challenge_state(
indicators or [],
gated_submit_controls=gated_controls,
+ challenge_controls=challenge_controls,
),
}
@@ -223,6 +244,20 @@ def page_evidence_needs_visual_fallback(evidence: dict[str, Any]) -> bool:
return bool(evidence.get("anti_bot_indicators") or evidence.get("challenge_controls"))
+# The vision classifier's typed non-challenge obstruction kind: a consent dialog
+# is dismissed, not solved, so it must never promote challenge state.
+CONSENT_OBSTRUCTION_KIND = "cookie_consent"
+
+
+def _confirmed_visual_challenge(evidence: dict[str, Any], visual_summary: dict[str, Any]) -> bool:
+ if visual_summary.get("challenge_detected") is not True:
+ return False
+ if interactive_challenge_controls(evidence.get("challenge_controls")):
+ return True
+ obstruction_kind = str(visual_summary.get("obstruction_kind") or "").strip().lower()
+ return obstruction_kind != CONSENT_OBSTRUCTION_KIND
+
+
def merge_visual_composition_evidence(
evidence: dict[str, Any],
*,
@@ -261,28 +296,34 @@ def merge_visual_composition_evidence(
if bounded:
omissions.append(bounded)
challenge_state = dict(merged.get("challenge_state") or {})
- if visual_summary.get("challenge_detected") is True:
+ challenge_confirmed = _confirmed_visual_challenge(evidence, visual_summary)
+ if challenge_confirmed:
challenge_state["detected"] = True
challenge_state["requires_human_verification"] = True
challenge_state["source"] = (
"dom+screenshot" if challenge_state.get("source") else SCREENSHOT_EVIDENCE_SOURCE
)
- challenge_kind = _bounded_string(visual_summary.get("challenge_kind"), 80)
- if challenge_kind:
- challenge_state["kind"] = challenge_kind
- challenge_location = _bounded_string(visual_summary.get("challenge_location"), 180)
- if challenge_location:
- challenge_state["visual_location"] = challenge_location
- if visual_summary.get("submit_blocked") is True:
+ if challenge_confirmed or challenge_state.get("detected") is True:
+ challenge_kind = _bounded_string(visual_summary.get("challenge_kind"), 80)
+ if challenge_kind:
+ challenge_state["kind"] = challenge_kind
+ challenge_location = _bounded_string(visual_summary.get("challenge_location"), 180)
+ if challenge_location:
+ challenge_state["visual_location"] = challenge_location
+ if visual_summary.get("submit_blocked") is True and challenge_confirmed:
challenge_state["gates_submit_controls"] = True
- visual_blocked_controls = [
- {
- "text": _bounded_string(item, 120),
- "disabled": True,
- }
- for item in visual_summary.get("blocked_submit_controls") or []
- if _bounded_string(item, 120)
- ]
+ visual_blocked_controls = (
+ [
+ {
+ "text": _bounded_string(item, 120),
+ "disabled": True,
+ }
+ for item in visual_summary.get("blocked_submit_controls") or []
+ if _bounded_string(item, 120)
+ ]
+ if challenge_confirmed
+ else []
+ )
if visual_blocked_controls:
existing_controls = [
item for item in challenge_state.get("gated_submit_controls") or [] if isinstance(item, dict)
@@ -1664,7 +1705,7 @@ def parse_composition_html(html: str, *, inspected_url: str, current_url: str) -
"empty_page_visual_state": None,
"evidence_confidence": confidence,
"source_tool": "inspect_page_for_composition",
- **_evidence_metadata(anti_bot_indicators, forms=forms),
+ **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),
}
@@ -1940,5 +1981,5 @@ def parse_composition_structured(data: Any, *, inspected_url: str, current_url:
"empty_page_visual_state": None,
"evidence_confidence": confidence,
"source_tool": "inspect_page_for_composition",
- **_evidence_metadata(anti_bot_indicators, forms=forms),
+ **_evidence_metadata(anti_bot_indicators, forms=forms, challenge_controls=challenge_controls),
}
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 707b2b091..b6044da6a 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
from enum import StrEnum
from typing import Any, cast
+from skyvern.forge.sdk.copilot.blocker_signal import contains_internal_machinery_leak
from skyvern.forge.sdk.copilot.context import COPILOT_RESPONSE_TYPES, ResponseType
from skyvern.forge.sdk.copilot.output_utils import (
looks_like_workflow_delivery_claim,
@@ -376,6 +377,8 @@ def _contains_raw_secret(value: Any) -> bool:
def _contains_internal_tool_instruction(user_response: str | None) -> bool:
if not isinstance(user_response, str):
return False
+ if contains_internal_machinery_leak(user_response):
+ return True
normalized = " ".join(user_response.lower().translate(_INTERNAL_TOOL_INSTRUCTION_TRANSLATION).split())
if any(marker in normalized for marker in _INTERNAL_TOOL_INSTRUCTION_MARKERS):
return True
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 2ce073605..cc3d5292c 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -70,6 +70,8 @@ from .blockers import _analyze_run_blocks as _analyze_run_blocks
from .blockers import _build_loop_blocker_signal as _build_loop_blocker_signal
from .blockers import _last_run_has_terminal_anti_bot_blocker as _last_run_has_terminal_anti_bot_blocker
from .blockers import _per_tool_budget_problem_rerun_signal as _per_tool_budget_problem_rerun_signal
+from .blockers import _post_budget_terminal_challenge_signal as _post_budget_terminal_challenge_signal
+from .blockers import _post_run_terminal_challenge_reason as _post_run_terminal_challenge_reason
from .blockers import (
_record_per_tool_budget_problem_blocks_from_results,
_tool_loop_error,
@@ -226,6 +228,7 @@ from .scouting import _role_name_from_selector as _role_name_from_selector
from .workflow_update import BlockObservationRef as BlockObservationRef
from .workflow_update import CodeArtifactMetadata as CodeArtifactMetadata
from .workflow_update import _code_artifact_metadata_as_tool_argument as _code_artifact_metadata_as_tool_argument
+from .workflow_update import _code_block_safety_errors as _code_block_safety_errors
from .workflow_update import _normalize_code_artifact_metadata as _normalize_code_artifact_metadata
from .workflow_update import _pre_run_workflow_coverage_error as _pre_run_workflow_coverage_error
from .workflow_update import _record_workflow_proxy_location_span as _record_workflow_proxy_location_span
diff --git a/skyvern/forge/sdk/copilot/tools/blockers.py b/skyvern/forge/sdk/copilot/tools/blockers.py
index eee55372d..870e9903c 100644
--- a/skyvern/forge/sdk/copilot/tools/blockers.py
+++ b/skyvern/forge/sdk/copilot/tools/blockers.py
@@ -8,7 +8,12 @@ import structlog
from skyvern.forge import app
from skyvern.forge.failure_classifier import classify_from_failure_reason
-from skyvern.forge.sdk.copilot.blocker_signal import CopilotToolBlockerSignal, build_loop_blocker_signal
+from skyvern.forge.sdk.copilot.blocker_signal import (
+ CopilotToolBlockerSignal,
+ assert_clean_user_facing_text,
+ build_loop_blocker_signal,
+)
+from skyvern.forge.sdk.copilot.composition_evidence import interactive_challenge_controls
from skyvern.forge.sdk.copilot.enforcement import TOTAL_TIMEOUT_SECONDS
from skyvern.forge.sdk.copilot.failure_tracking import (
ACTIVE_RUN_TERMINAL_EVIDENCE_FAILURE_CATEGORY,
@@ -361,7 +366,9 @@ def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:
indicators = evidence.get("anti_bot_indicators")
has_indicators = isinstance(indicators, list) and any(isinstance(item, str) and item.strip() for item in indicators)
- if has_indicators:
+ # Raw-HTML token hits alone never justify a terminal claim; a rendered
+ # challenge control must corroborate them.
+ if has_indicators and interactive_challenge_controls(evidence.get("challenge_controls")):
disabled_controls = _disabled_submit_controls_from_evidence(evidence)
if disabled_controls:
labels = ", ".join(_composition_control_label(control) for control in disabled_controls[:3])
@@ -369,6 +376,42 @@ def _post_run_terminal_challenge_reason(evidence: dict[str, Any]) -> str | None:
return None
+def _post_run_result_evidence_summary(evidence: dict[str, Any]) -> str | None:
+ containers = [container for container in evidence.get("result_containers") or [] if isinstance(container, dict)]
+ if not containers:
+ return None
+ hints: list[str] = []
+ for container in containers[:3]:
+ for key in ("selector", "id", "tag"):
+ value = container.get(key)
+ if isinstance(value, str) and value.strip():
+ hints.append(value.strip()[:80])
+ break
+ hint_text = f" ({', '.join(hints)})" if hints else ""
+ return f"{len(containers)} result container(s){hint_text}"
+
+
+_TERMINAL_CHALLENGE_FALLBACK_USER_FACING = (
+ "A page barrier was still blocking the workflow's next step after the last run, "
+ "so I stopped without claiming results."
+)
+
+
+def _terminal_challenge_user_facing_reason(reason: str) -> str:
+ # The narrative names the observed evidence (challenge kind, control labels)
+ # instead of asserting a fixed mechanism; page-derived text falls back to a
+ # pre-validated template if it ever trips the leak deny list.
+ candidate = (
+ f"I stopped because the page I inspected after the last run still shows: {reason}. "
+ "I haven't verified any results, so I'm not claiming them."
+ )
+ try:
+ assert_clean_user_facing_text(candidate)
+ except ValueError:
+ return _TERMINAL_CHALLENGE_FALLBACK_USER_FACING
+ return candidate
+
+
def _post_budget_terminal_challenge_signal(
ctx: AgentContext, arguments: dict[str, Any] | None, tool_name: str
) -> CopilotToolBlockerSignal | None:
@@ -391,23 +434,45 @@ def _post_budget_terminal_challenge_signal(
if not reason:
return None
+ # Same-packet adjudication: when the observed page also carries result
+ # evidence, the answer-from-observed-page path outranks a terminal claim.
+ result_evidence = _post_run_result_evidence_summary(evidence)
+ if result_evidence:
+ agent_steering = (
+ "The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "
+ f"shows: {reason} — but the SAME observed page also contains result evidence: {result_evidence}. "
+ f"Do NOT claim the page is blocked and do NOT rerun {tool_name} unchanged. "
+ "Answer from the observed page if it contains the requested result/no-result evidence; "
+ "otherwise inspect the live page again before deciding."
+ )
+ return CopilotToolBlockerSignal(
+ blocker_kind="tool_error",
+ agent_steering_text=agent_steering,
+ user_facing_reason=(
+ "The last page I checked already shows results. I'll work from that evidence instead of retrying."
+ ),
+ recovery_hint="retry_with_different_tool",
+ cleared_by_tools=PAGE_INSPECTION_TOOLS
+ | frozenset({"get_run_results", "update_workflow", "update_and_run_blocks"}),
+ preserves_workflow_draft=True,
+ renders_final_reply=False,
+ internal_reason_code="tool_error_post_budget_challenge_result_evidence",
+ blocked_tool=tool_name,
+ )
+
requested_labels = _requested_block_label_set(arguments)
labels_text = f" Requested labels: {', '.join(sorted(requested_labels))}." if requested_labels else ""
agent_steering = (
"The prior block-running tool hit a failed/budgeted frontier, and bounded current-page inspection "
f"now shows: {reason}.{labels_text} Do NOT call "
f"{tool_name} again in this turn, do NOT try another proxy/location from this evidence state, and "
- "do NOT claim registry results or no-results were verified. REPLY now with a blocker explanation "
+ "do NOT claim results or no-results were verified. REPLY now with a blocker explanation "
"that names the observed challenge/disabled control and summarizes the tested workflow state."
)
- user_facing = (
- "The site's verification challenge is still blocking the submit/search control after live-page inspection, "
- "so I stopped without claiming results."
- )
return CopilotToolBlockerSignal(
blocker_kind="tool_error",
agent_steering_text=agent_steering,
- user_facing_reason=user_facing,
+ user_facing_reason=_terminal_challenge_user_facing_reason(reason),
recovery_hint="report_blocker_to_user",
cleared_by_tools=frozenset(),
preserves_workflow_draft=True,
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 31d01fe1f..4e8e50729 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -25,6 +25,7 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
)
from skyvern.forge.sdk.copilot.composition_evidence import (
+ CONSENT_OBSTRUCTION_KIND,
has_bounded_page_schema,
merge_visual_composition_evidence,
page_evidence_needs_visual_fallback,
@@ -247,11 +248,12 @@ def _composition_extract_screenshot_b64(result: dict[str, Any]) -> str:
def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
+ # The DOM challenge_state / anti-bot token hits are deliberately NOT fed in:
+ # the vision pass classifies obstruction-vs-challenge from the screenshot
+ # alone instead of confirming the detector's anchor.
context = {
"page_title": evidence.get("page_title") or "",
"current_url": evidence.get("current_url") or "",
- "anti_bot_indicators": evidence.get("anti_bot_indicators") or [],
- "challenge_state": evidence.get("challenge_state") or {},
"form_count": len(evidence.get("forms") or []),
"result_container_count": len(evidence.get("result_containers") or []),
"page_obstruction_count": len(evidence.get("page_obstructions") or []),
@@ -267,9 +269,18 @@ def _composition_visual_prompt(evidence: dict[str, Any]) -> str:
"In summary, include the visible page state that would help verify an end-state outcome, "
"such as cart items, "
"record rows, visible identifiers, quantities, statuses, prices, confirmations, search results, "
- "or selected values when legible. Also note visible anti-bot or human-verification state, "
- "whether it appears to gate a submit/search control, whether any visible artificial barrier "
- "mechanically blocks the underlying page, and where it appears relative to the page controls. "
+ "or selected values when legible. "
+ "Classify any visible artificial barrier from the screenshot alone: a verification challenge is a "
+ "captcha or human-verification widget asking the visitor to prove they are human; a page "
+ "obstruction is a dismissible layer such as a cookie/privacy consent dialog, promo or newsletter "
+ "modal, chat widget, or loading overlay. "
+ f"Use obstruction_kind values: {CONSENT_OBSTRUCTION_KIND}, promo_modal, chat_widget, "
+ "loading_overlay, other. A cookie/privacy consent dialog is always a page obstruction, never a "
+ "challenge: report it with page_obstruction_detected true and obstruction_kind "
+ f"{CONSENT_OBSTRUCTION_KIND}, and do not set challenge_detected or submit_blocked for it. "
+ "Set challenge_detected to true only for a visible verification challenge, set submit_blocked to "
+ "true only when that challenge visibly gates a submit/search control, and note where the barrier "
+ "appears relative to the page controls. "
"Do not include raw DOM, code, selectors, personal data, or workflow instructions. "
"If no challenge is visible, set challenge_detected to false and submit_blocked to false. "
"If no page obstruction is visible, set page_obstruction_detected to false. "
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 6417b4be8..ed86894dd 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -10,11 +10,14 @@ from pydantic import AliasChoices, BaseModel, Field, ValidationError
from skyvern.forge import app
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
+from skyvern.forge.sdk.copilot.code_block_synthesis import artifact_dependency_id, artifact_observation_ref_id
from skyvern.forge.sdk.copilot.composition_evidence import (
+ SCOUT_INTERACTION_EVIDENCE_TOOL,
composition_page_evidence_error,
normalize_block_observation_refs,
workflow_target_url,
)
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
POST_INTERMEDIATE_SUCCESS_NUDGE,
@@ -26,11 +29,12 @@ from skyvern.forge.sdk.copilot.output_policy import (
format_output_policy_tool_error,
output_policy_verdict_to_trace_data,
)
-from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.runtime import AgentContext, ScoutedInteraction
from skyvern.forge.sdk.copilot.streaming_adapter import emit_workflow_draft, maybe_emit_design_end
from skyvern.forge.sdk.copilot.tracing_setup import copilot_span
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
-from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
+from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException, InsecureCodeDetected
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
from skyvern.forge.sdk.workflow.models.workflow import Workflow
from skyvern.schemas.proxy_location import ProxyLocation
from skyvern.schemas.workflows import BlockType
@@ -47,6 +51,7 @@ from .banned_blocks import (
_banned_block_reject_message,
_challenge_http_request_reject_message,
_copilot_banned_block_types,
+ _copilot_block_authoring_policy,
_detect_new_banned_blocks,
_record_banned_block_reject_span,
_timing_only_challenge_wait_reject_message,
@@ -78,12 +83,12 @@ ArtifactEvidenceStatus = Literal["satisfied", "missing", "diagnostic_only", "obs
class CodeArtifactClaimedOutcome(BaseModel):
- id: str
- scope: str
- text: str
- status: ArtifactEvidenceStatus
- depends_on: list[str] = Field(default_factory=list)
- covered_criteria: list[str] = Field(default_factory=list)
+ id: str = ""
+ scope: str = ""
+ text: str = ""
+ status: ArtifactEvidenceStatus = "observed_not_verified"
+ depends_on: list[str] = Field(default_factory=list, description="Page-dependency ids this claim relies on.")
+ covered_criteria: list[str] = Field(default_factory=list, description="Completion-criterion ids this claim covers.")
criteria_ids: list[str] = Field(default_factory=list)
evidence_refs: list[str] = Field(default_factory=list)
observation_refs: list[str] = Field(default_factory=list)
@@ -92,9 +97,9 @@ class CodeArtifactClaimedOutcome(BaseModel):
class CodeArtifactPageDependency(BaseModel):
- id: str
- scope: str
- status: ArtifactEvidenceStatus
+ id: str = ""
+ scope: str = ""
+ status: ArtifactEvidenceStatus = "observed_not_verified"
url_hint: str | None = None
page_state_hint: str | None = None
required_affordances: list[str] = Field(default_factory=list)
@@ -104,9 +109,9 @@ class CodeArtifactPageDependency(BaseModel):
class CodeArtifactCompletionCriterion(BaseModel):
- id: str
- text: str
- level: Literal["terminal", "outcome", "prefix", "method"]
+ id: str = ""
+ text: str = ""
+ level: Literal["terminal", "outcome", "prefix", "method"] = "terminal"
outcome: str | None = None
terminal: bool | None = None
@@ -117,7 +122,9 @@ class CodeArtifactScopedRef(BaseModel):
criterion_id: str | None = None
evidence_ref: str | None = None
observation_ref: str | None = None
- status: ArtifactEvidenceStatus = Field(validation_alias=AliasChoices("status", "evidence_status"))
+ status: ArtifactEvidenceStatus = Field(
+ default="observed_not_verified", validation_alias=AliasChoices("status", "evidence_status")
+ )
source_tool: str | None = None
observation_step: Annotated[int, Field(ge=0)] | None = None
run_sample_id: str | None = None
@@ -127,15 +134,15 @@ class CodeArtifactScopedRef(BaseModel):
class CodeArtifactTerminalVerifierExpectation(BaseModel):
- id: str
- text: str
+ id: str = ""
+ text: str = ""
criteria_ids: list[str] = Field(default_factory=list)
claimed_outcome_ids: list[str] = Field(default_factory=list)
class CodeArtifactExplorationObservation(BaseModel):
- id: str
- text: str
+ id: str = ""
+ text: str = ""
status: Literal["observed_not_verified"] = Field(
default="observed_not_verified",
validation_alias=AliasChoices("status", "evidence_status"),
@@ -145,21 +152,59 @@ class CodeArtifactExplorationObservation(BaseModel):
observation_step: Annotated[int, Field(ge=0)] | None = None
current_url: str | None = None
source_label: str | None = None
- checkpoint_next_mode: Literal["stop"] | None = None
+ checkpoint_next_mode: Literal["advance", "stop"] | None = None
class CodeArtifactMetadata(BaseModel):
- artifact_id: str | None = None
- block_label: str | None = None
+ artifact_id: str | None = Field(
+ default=None, description="Server-owned id; defaults to `code_artifact:<block_label>` when omitted."
+ )
+ block_label: str | None = Field(
+ default=None, description="Label of the authored `code` block this artifact describes."
+ )
block_id: str | None = None
- declared_goal: str
- claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(default_factory=list)
- page_dependencies: list[CodeArtifactPageDependency] = Field(default_factory=list)
- completion_criteria: list[CodeArtifactCompletionCriterion] = Field(default_factory=list)
- evidence_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
- observation_refs: list[CodeArtifactScopedRef] = Field(default_factory=list)
- terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(default_factory=list)
- exploration_observations: list[CodeArtifactExplorationObservation] = Field(default_factory=list)
+ declared_goal: str = Field(default="", description="The durable goal this block accomplishes; model-owned.")
+ claimed_outcomes: list[CodeArtifactClaimedOutcome] = Field(
+ default_factory=list,
+ description=(
+ "Outcomes this block claims. Each claim links `depends_on` page-dependency ids, covered "
+ "criterion ids, and claim-scoped observation/evidence refs; a `satisfied` claim additionally "
+ "requires claim-scoped `evidence_refs`. Mechanical links are server-defaulted at the "
+ "persist seam in code-block authoring mode."
+ ),
+ )
+ page_dependencies: list[CodeArtifactPageDependency] = Field(
+ default_factory=list,
+ description=(
+ "Pages or states the code depends on; non-`missing` rows carry scoped evidence or observation "
+ "refs. Server-defaulted when omitted."
+ ),
+ )
+ completion_criteria: list[CodeArtifactCompletionCriterion] = Field(
+ default_factory=list,
+ description="Completion criteria; include at least one `terminal` criterion.",
+ )
+ evidence_refs: list[CodeArtifactScopedRef] = Field(
+ default_factory=list,
+ description=(
+ "Artifact-level refs: each entry carries its ref id, a scoped id (claim/dependency/criterion), "
+ "and `source_tool` unless status is `missing`."
+ ),
+ )
+ observation_refs: list[CodeArtifactScopedRef] = Field(
+ default_factory=list,
+ description=(
+ "Artifact-level observation refs; same shape rules as `evidence_refs`. Server-defaulted when omitted."
+ ),
+ )
+ terminal_verifier_expectations: list[CodeArtifactTerminalVerifierExpectation] = Field(
+ default_factory=list,
+ description="What terminal verification must observe; link `criteria_ids` or `claimed_outcome_ids`.",
+ )
+ exploration_observations: list[CodeArtifactExplorationObservation] = Field(
+ default_factory=list,
+ description="Scout-time observations; status stays `observed_not_verified` until verification passes.",
+ )
_CODE_ARTIFACT_REQUIRED_LIST_FIELDS = (
@@ -190,13 +235,24 @@ def _format_code_artifact_violations(violations: list[str]) -> str:
def _normalize_code_artifact_metadata(
raw_metadata: Any,
workflow_yaml: str,
+ *,
+ impose_defaults: bool = False,
+ scout_trajectory: list[ScoutedInteraction] | None = None,
) -> tuple[dict[str, dict[str, Any]], str | None]:
+ """Normalize submitted artifact metadata at the persist seam.
+
+ Entries keyed to a missing code-block label are re-keyed to the single
+ uncovered block or dropped, never rejected. When imposing, mechanical graph
+ fields are server-defaulted (and uncovered labels get a deterministic
+ skeleton) so only contradictions reject."""
if raw_metadata in (None, [], {}):
return {}, None
items = _code_artifact_metadata_items(raw_metadata)
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
- normalized: dict[str, dict[str, Any]] = {}
+ trajectory: list[ScoutedInteraction] = scout_trajectory or []
violations: list[str] = []
+ anchored: list[dict[str, Any]] = []
+ unanchored: list[dict[str, Any]] = []
seen_labels: set[str] = set()
for raw_item in items:
try:
@@ -210,32 +266,55 @@ def _normalize_code_artifact_metadata(
continue
dumped = metadata.model_dump(mode="json", exclude_none=True)
label = str(dumped.get("block_label") or "").strip()
- if not label:
- violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")
- continue
- if label not in code_blocks:
- violations.append(
- f"Artifact metadata for `{label}` does not reference an existing code block label. "
- "Attach metadata only to authored code blocks."
- )
+ if not label or label not in code_blocks:
+ unanchored.append(dumped)
continue
if label in seen_labels:
- violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")
+ LOG.info("copilot_code_artifact_metadata_duplicate_dropped", block_label=label)
continue
seen_labels.add(label)
dumped["block_label"] = label
- item_violations: list[str] = []
+ anchored.append(dumped)
+
+ uncovered = [label for label in code_blocks if label not in seen_labels]
+ if len(unanchored) == 1 and len(uncovered) == 1:
+ # The only metadata-less code block is the only plausible owner of the
+ # only unanchored entry; re-key instead of dropping.
+ rekeyed = unanchored.pop()
+ LOG.info(
+ "copilot_code_artifact_metadata_rekeyed",
+ stale_label=str(rekeyed.get("block_label") or "") or None,
+ block_label=uncovered[0],
+ )
+ rekeyed["block_label"] = uncovered[0]
+ rekeyed.pop("artifact_id", None)
+ rekeyed.pop("block_id", None)
+ seen_labels.add(uncovered[0])
+ anchored.append(rekeyed)
+ uncovered = []
+ if unanchored:
+ LOG.info(
+ "copilot_code_artifact_metadata_stale_entries_dropped",
+ stale_labels=[str(item.get("block_label") or "") or None for item in unanchored],
+ )
+
+ normalized: dict[str, dict[str, Any]] = {}
+ for dumped in anchored:
+ label = str(dumped["block_label"])
artifact_id = str(dumped.get("artifact_id") or "").strip()
- if artifact_id and not artifact_id.startswith("code_artifact:"):
- item_violations.append(
- f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."
- )
- dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)
+ if not artifact_id.startswith("code_artifact:"):
+ artifact_id = _artifact_id_for_block_label(label)
+ dumped["artifact_id"] = artifact_id
block_id = str(
dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or ""
).strip()
if block_id:
dumped["block_id"] = block_id
+ if impose_defaults:
+ if not str(dumped.get("declared_goal") or "").strip():
+ dumped["declared_goal"] = _block_goal_fallback(label, code_blocks[label])
+ _impose_code_artifact_metadata_defaults(label, dumped, trajectory)
+ item_violations: list[str] = []
declared_goal = str(dumped.get("declared_goal") or "").strip()
if not declared_goal:
item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")
@@ -250,11 +329,221 @@ def _normalize_code_artifact_metadata(
violations.extend(item_violations)
continue
normalized[label] = dumped
+ if impose_defaults:
+ for label in uncovered:
+ normalized[label] = _imposed_artifact_skeleton(label, code_blocks[label], trajectory)
if violations:
- return {}, _format_code_artifact_violations(violations)
+ return normalized, _format_code_artifact_violations(violations)
return normalized, None
+def _artifact_label_fragment(label: str) -> str:
+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"
+
+
+def _artifact_mutable_rows(value: Any) -> list[dict[str, Any]]:
+ return [row for row in value if isinstance(row, dict)] if isinstance(value, list) else []
+
+
+def _drop_contradictory_checkpoint_mode(ref: dict[str, Any]) -> None:
+ status = str(ref.get("status") or "").strip()
+ mode = ref.get("checkpoint_next_mode")
+ if mode == "advance" and status != "diagnostic_only":
+ ref.pop("checkpoint_next_mode", None)
+ elif mode == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:
+ ref.pop("checkpoint_next_mode", None)
+
+
+def _impose_code_artifact_metadata_defaults(
+ label: str,
+ artifact: dict[str, Any],
+ scout_trajectory: list[ScoutedInteraction],
+) -> None:
+ """Fill the mechanical evidence-graph fields the persist seam owns; semantic
+ content (declared_goal, claim/criterion text) stays model-authored, and
+ contradictions (e.g. `satisfied` without evidence) are left for validation."""
+ fragment = _artifact_label_fragment(label)
+ dependency_id = artifact_dependency_id(label)
+ observation_ref_id = artifact_observation_ref_id(label)
+ declared_goal = str(artifact.get("declared_goal") or "").strip()
+
+ dependencies = _artifact_mutable_rows(artifact.get("page_dependencies"))
+ for index, dependency in enumerate(dependencies):
+ if not str(dependency.get("id") or "").strip():
+ dependency["id"] = f"dependency:{fragment}_{index}"
+ if not str(dependency.get("scope") or "").strip():
+ dependency["scope"] = "page"
+ if not dependencies:
+ default_dependency: dict[str, Any] = {
+ "id": dependency_id,
+ "scope": "page",
+ "status": "observed_not_verified",
+ "observation_refs": [observation_ref_id],
+ }
+ entry_url_hint = next(
+ (url for url in (str(item.get("source_url") or "").strip() for item in scout_trajectory) if url),
+ None,
+ )
+ if entry_url_hint:
+ default_dependency["url_hint"] = entry_url_hint
+ artifact["page_dependencies"] = [default_dependency]
+ dependencies = _artifact_mutable_rows(artifact.get("page_dependencies"))
+ primary_dependency_id = str(dependencies[0].get("id") or "").strip() or dependency_id
+
+ claims = _artifact_mutable_rows(artifact.get("claimed_outcomes"))
+ for index, claim in enumerate(claims):
+ if not str(claim.get("id") or "").strip():
+ claim["id"] = f"claim:{fragment}_{index}"
+ if not str(claim.get("scope") or "").strip():
+ claim["scope"] = "outcome"
+ if not str(claim.get("text") or "").strip():
+ claim["text"] = declared_goal or f"outcome of `{label}`"
+ if not claims and declared_goal:
+ artifact["claimed_outcomes"] = [
+ {
+ "id": f"claim:{fragment}_goal",
+ "scope": "outcome",
+ "text": declared_goal,
+ "status": "observed_not_verified",
+ "depends_on": [primary_dependency_id],
+ "observation_refs": [observation_ref_id],
+ }
+ ]
+ claims = _artifact_mutable_rows(artifact.get("claimed_outcomes"))
+
+ criteria = _artifact_mutable_rows(artifact.get("completion_criteria"))
+ for index, criterion in enumerate(criteria):
+ if not str(criterion.get("id") or "").strip():
+ criterion["id"] = f"criterion:{fragment}_goal_{index}"
+ if not str(criterion.get("text") or "").strip():
+ criterion["text"] = declared_goal or f"criterion for `{label}`"
+ if not criteria and claims:
+ artifact["completion_criteria"] = [
+ {
+ "id": f"criterion:{fragment}_goal_{index}",
+ "text": str(claim.get("text") or declared_goal or "").strip() or f"criterion for `{label}`",
+ "level": "terminal",
+ "terminal": True,
+ }
+ for index, claim in enumerate(claims)
+ ]
+ criteria = _artifact_mutable_rows(artifact.get("completion_criteria"))
+ criterion_ids = [str(criterion.get("id") or "").strip() for criterion in criteria]
+ criterion_ids = [criterion_id for criterion_id in criterion_ids if criterion_id]
+
+ for claim in claims:
+ if not isinstance(claim, dict):
+ continue
+ if not _artifact_string_list(claim.get("depends_on")):
+ claim["depends_on"] = [primary_dependency_id]
+ if (
+ criterion_ids
+ and not _artifact_string_list(claim.get("covered_criteria"))
+ and not _artifact_string_list(claim.get("criteria_ids"))
+ ):
+ claim["covered_criteria"] = list(criterion_ids)
+ status = str(claim.get("status") or "").strip()
+ if (
+ status not in {"missing", "satisfied"}
+ and not _artifact_string_list(claim.get("evidence_refs"))
+ and not _artifact_string_list(claim.get("observation_refs"))
+ ):
+ claim["observation_refs"] = [observation_ref_id]
+
+ for dependency in dependencies:
+ if not isinstance(dependency, dict):
+ continue
+ status = str(dependency.get("status") or "").strip()
+ if (
+ status not in {"missing", "satisfied"}
+ and not _artifact_string_list(dependency.get("evidence_refs"))
+ and not _artifact_string_list(dependency.get("observation_refs"))
+ ):
+ dependency["observation_refs"] = [observation_ref_id]
+
+ expectations = _artifact_mutable_rows(artifact.get("terminal_verifier_expectations"))
+ if not expectations and (criterion_ids or declared_goal):
+ artifact["terminal_verifier_expectations"] = [
+ {
+ "id": f"expectation:{fragment}_terminal",
+ "text": f"Terminal verification observes: {declared_goal or label}",
+ "criteria_ids": list(criterion_ids),
+ }
+ ]
+ expectations = _artifact_mutable_rows(artifact.get("terminal_verifier_expectations"))
+ for index, expectation in enumerate(expectations):
+ if not isinstance(expectation, dict):
+ continue
+ if not str(expectation.get("id") or "").strip():
+ expectation["id"] = f"expectation:{fragment}_{index}"
+ if not str(expectation.get("text") or "").strip():
+ expectation["text"] = f"Terminal verification observes: {declared_goal or label}"
+ if (
+ criterion_ids
+ and not _artifact_string_list(expectation.get("criteria_ids"))
+ and not _artifact_string_list(expectation.get("claimed_outcome_ids"))
+ ):
+ expectation["criteria_ids"] = list(criterion_ids)
+
+ if not _artifact_mutable_rows(artifact.get("evidence_refs")) and not _artifact_mutable_rows(
+ artifact.get("observation_refs")
+ ):
+ artifact["observation_refs"] = [
+ {
+ "observation_ref": observation_ref_id,
+ "dependency_id": primary_dependency_id,
+ "status": "observed_not_verified",
+ "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,
+ }
+ ]
+
+ for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):
+ for index, ref in enumerate(_artifact_mutable_rows(artifact.get(field_name))):
+ if not str(ref.get(ref_key) or "").strip():
+ prefix = "evidence" if ref_key == "evidence_ref" else "observation"
+ ref[ref_key] = f"{prefix}:{fragment}_{index}"
+ if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):
+ ref["dependency_id"] = primary_dependency_id
+ status = str(ref.get("status") or "").strip()
+ if status != "missing" and not str(ref.get("source_tool") or "").strip():
+ ref["source_tool"] = SCOUT_INTERACTION_EVIDENCE_TOOL
+ _drop_contradictory_checkpoint_mode(ref)
+
+ if isinstance(artifact.get("exploration_observations"), list):
+ observations: list[dict[str, Any]] = []
+ for index, observation in enumerate(_artifact_mutable_rows(artifact["exploration_observations"])):
+ if not str(observation.get("text") or "").strip():
+ continue
+ if not str(observation.get("id") or "").strip():
+ observation["id"] = f"exploration:{fragment}_{index}"
+ if observation.get("checkpoint_next_mode") == "advance":
+ observation.pop("checkpoint_next_mode", None)
+ observations.append(observation)
+ artifact["exploration_observations"] = observations
+
+
+def _block_goal_fallback(label: str, block: Mapping[str, Any]) -> str:
+ title = str(block.get("title") or "").strip()
+ return title or label.replace("_", " ").strip() or label
+
+
+def _imposed_artifact_skeleton(
+ label: str,
+ block: Mapping[str, Any],
+ scout_trajectory: list[ScoutedInteraction],
+) -> dict[str, Any]:
+ artifact: dict[str, Any] = {
+ "block_label": label,
+ "artifact_id": _artifact_id_for_block_label(label),
+ "declared_goal": _block_goal_fallback(label, block),
+ }
+ block_id = str(block.get("block_id") or block.get("id") or "").strip()
+ if block_id:
+ artifact["block_id"] = block_id
+ _impose_code_artifact_metadata_defaults(label, artifact, scout_trajectory)
+ return artifact
+
+
def _code_artifact_metadata_items(raw_metadata: Any) -> list[Any]:
if isinstance(raw_metadata, Mapping):
items: list[Any] = []
@@ -278,8 +567,42 @@ def _workflow_yaml_code_blocks_by_label(workflow_yaml: str | None) -> dict[str,
def _artifact_id_for_block_label(label: str) -> str:
- fragment = re.sub(r"[^A-Za-z0-9_.-]+", "_", label).strip("_") or "artifact"
- return f"code_artifact:{fragment}"
+ return f"code_artifact:{_artifact_label_fragment(label)}"
+
+
+def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None) -> list[str]:
+ """Run the sandbox's static safety rule on new/changed code blocks before any run.
+
+ Label-scoped diff so legacy code blocks the model did not touch cannot wedge
+ every subsequent update."""
+ prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)
+ errors: list[str] = []
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ prior_block = prior_blocks.get(label)
+ if prior_block is not None and str(prior_block.get("code") or "") == code:
+ continue
+ try:
+ CodeBlock.is_safe_code(code)
+ except SyntaxError as exc:
+ errors.append(f"Code block `{label}` is not valid Python: {exc}")
+ except InsecureCodeDetected as exc:
+ errors.append(
+ f"Code block `{label}` failed the sandbox safety check: {exc}. Rewrite without import "
+ "statements, dunder access, or private attributes; the sandbox already provides `page`, "
+ "`json`, `re`, `html`, `asyncio.sleep`, and common builtins."
+ )
+ return errors
+
+
+def _code_seam_rejection_user_summary(*, metadata_rejected: bool, code_rejected: bool) -> str:
+ if metadata_rejected and code_rejected:
+ return "I need to adjust the workflow's code and its verification details before testing."
+ if code_rejected:
+ return "I need to adjust the workflow's code so it can run safely before testing."
+ return "I need to adjust how the workflow verifies its results before testing."
def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:
@@ -392,16 +715,42 @@ async def _update_workflow(
ctx.raw_block_observation_refs = params.get("raw_block_observation_refs", params.get("block_observation_refs"))
ctx.block_observation_refs = normalize_block_observation_refs(params.get("block_observation_refs"))
ctx.raw_code_artifact_metadata = params.get("raw_code_artifact_metadata", params.get("code_artifact_metadata"))
+ scout_trajectory = getattr(ctx, "scout_trajectory", None)
code_artifact_metadata, code_artifact_metadata_error = _normalize_code_artifact_metadata(
params.get("code_artifact_metadata"),
workflow_yaml,
+ impose_defaults=_copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER,
+ scout_trajectory=scout_trajectory if isinstance(scout_trajectory, list) else None,
)
- if code_artifact_metadata_error is not None:
- return {"ok": False, "error": code_artifact_metadata_error}
- if code_artifact_metadata:
- ctx.code_artifact_metadata = code_artifact_metadata
- ctx.workflow_verification_evidence.code_artifact_metadata = code_artifact_metadata
- params["code_artifact_metadata"] = code_artifact_metadata
+ prior_workflow_yaml = getattr(ctx, "workflow_yaml", None)
+ code_safety_errors = _code_block_safety_errors(workflow_yaml, prior_workflow_yaml)
+ # Per-label salvage keeps conforming metadata across a rejection; a
+ # rejected code block keeps nothing, since its yaml never becomes the
+ # draft. Prior-draft labels survive every rejection gate below — the
+ # accept path prunes to the submitted blocks once the draft switches.
+ if code_artifact_metadata and not code_safety_errors:
+ existing_metadata = getattr(ctx, "code_artifact_metadata", None)
+ merged_metadata = {
+ **(existing_metadata if isinstance(existing_metadata, dict) else {}),
+ **code_artifact_metadata,
+ }
+ retained_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml)) | set(
+ _workflow_yaml_code_blocks_by_label(prior_workflow_yaml)
+ )
+ merged_metadata = {block: row for block, row in merged_metadata.items() if block in retained_labels}
+ ctx.code_artifact_metadata = merged_metadata
+ ctx.workflow_verification_evidence.code_artifact_metadata = merged_metadata
+ params["code_artifact_metadata"] = merged_metadata
+ seam_errors = [error for error in (code_artifact_metadata_error, *code_safety_errors) if error]
+ if seam_errors:
+ return {
+ "ok": False,
+ "error": "\n".join(seam_errors),
+ "user_facing_summary": _code_seam_rejection_user_summary(
+ metadata_rejected=code_artifact_metadata_error is not None,
+ code_rejected=bool(code_safety_errors),
+ ),
+ }
if allow_missing_credentials is None:
allow_missing_credentials = getattr(ctx, "allow_untested_workflow_draft", False) is True
if not allow_missing_credentials:
@@ -521,6 +870,13 @@ async def _update_workflow(
ctx.staged_workflow = workflow
ctx.has_staged_proposal = True
ctx.workflow_yaml = workflow_yaml
+ accepted_metadata = getattr(ctx, "code_artifact_metadata", None)
+ if isinstance(accepted_metadata, dict) and accepted_metadata:
+ accepted_labels = set(_workflow_yaml_code_blocks_by_label(workflow_yaml))
+ pruned_metadata = {block: row for block, row in accepted_metadata.items() if block in accepted_labels}
+ if pruned_metadata != accepted_metadata:
+ ctx.code_artifact_metadata = pruned_metadata
+ ctx.workflow_verification_evidence.code_artifact_metadata = pruned_metadata
# Best-effort — narrative emit failures must never abort an
# otherwise-successful update_workflow tool call. ``isinstance``
# narrows the parameter's declared ``AgentContext`` to the
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index 76a3e1875..bce793bac 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -1182,7 +1182,10 @@ workflow_definition:
)
process_mock.assert_not_called()
- assert "inspect_page_for_composition" in agent_result.user_response
+ # The reject note must be product language, never the gate's
+ # agent-directed tool instruction.
+ assert "(Note:" in agent_result.user_response
+ assert "inspect_page_for_composition" not in agent_result.user_response
assert agent_result.updated_workflow is None
assert agent_result.workflow_yaml is None
diff --git a/tests/unit/test_copilot_blocker_signal.py b/tests/unit/test_copilot_blocker_signal.py
index fcf278f41..f6e159259 100644
--- a/tests/unit/test_copilot_blocker_signal.py
+++ b/tests/unit/test_copilot_blocker_signal.py
@@ -91,6 +91,20 @@ def test_assert_clean_passes_normal_product_language() -> None:
assert_clean_user_facing_text("I couldn't complete that on this turn.")
+def test_assert_clean_raises_on_internal_budget_vocabulary() -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("The run exceeded the 6s per-tool-call budget while still making progress.")
+
+
+def test_assert_clean_raises_on_raw_run_id() -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("Run ID: wr_538438176486379954. Outcome is uncertain.")
+
+
+def test_assert_clean_allows_prose_mentioning_runs_without_ids() -> None:
+ assert_clean_user_facing_text("The last run didn't finish; I stopped without claiming results.")
+
+
class _Ctx:
blocker_signal: CopilotToolBlockerSignal | None = None
latest_tool_blocker_signal: CopilotToolBlockerSignal | None = None
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
index 10892f6f8..a10d9de81 100644
--- a/tests/unit/test_copilot_code_artifact_metadata_violations.py
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -28,6 +28,38 @@ def _violation_count(error: str) -> int:
return len(re.findall(r"^\d+\.", error, flags=re.M))
+def _valid_metadata(label: str) -> dict:
+ return {
+ "block_label": label,
+ "artifact_id": f"code_artifact:{label}",
+ "declared_goal": "g",
+ "claimed_outcomes": [
+ {
+ "id": "claim:x",
+ "scope": "outcome",
+ "text": "x",
+ "status": "observed_not_verified",
+ "depends_on": ["dependency:p"],
+ "covered_criteria": ["criterion:c"],
+ "observation_refs": ["obs1"],
+ }
+ ],
+ "page_dependencies": [
+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
+ ],
+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
+ "observation_refs": [
+ {
+ "observation_ref": "obs1",
+ "dependency_id": "dependency:p",
+ "status": "observed_not_verified",
+ "source_tool": "scout_interaction",
+ }
+ ],
+ }
+
+
def _broken_metadata(label: str) -> dict:
return {
"block_label": label,
@@ -52,46 +84,25 @@ class TestAccumulateAllViolations:
assert _violation_count(error) >= 5
assert error.startswith("Artifact metadata has ")
assert "fix all of them in one update" in error
- assert "`artifact_id` to start with `code_artifact:`" in error
assert "requires `source_tool`" in error
assert "requires `depends_on`" in error
assert "is `satisfied` but has no" in error
+ def test_non_conforming_artifact_id_is_imposed_not_rejected(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata["artifact_id"] = "not-prefixed"
+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
+ assert error is None
+ assert normalized["my_block"]["artifact_id"] == "code_artifact:my_block"
+
def test_single_violation_is_not_numbered(self) -> None:
- metadata = {
- "block_label": "my_block",
- "artifact_id": "not-prefixed",
- "declared_goal": "g",
- "claimed_outcomes": [
- {
- "id": "claim:x",
- "scope": "outcome",
- "text": "x",
- "status": "observed_not_verified",
- "depends_on": ["dependency:p"],
- "covered_criteria": ["criterion:c"],
- "observation_refs": ["obs1"],
- }
- ],
- "page_dependencies": [
- {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
- ],
- "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
- "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
- "observation_refs": [
- {
- "observation_ref": "obs1",
- "dependency_id": "dependency:p",
- "status": "observed_not_verified",
- "source_tool": "scout_interaction",
- }
- ],
- }
+ metadata = _valid_metadata("my_block")
+ metadata["claimed_outcomes"][0].pop("depends_on")
normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
assert normalized == {}
assert error is not None
assert _violation_count(error) == 0
- assert error == "Artifact metadata for `my_block` requires `artifact_id` to start with `code_artifact:`."
+ assert error == "Artifact metadata claim `claim:x` for `my_block` requires `depends_on`."
def test_violations_aggregate_across_multiple_artifacts(self) -> None:
yaml = textwrap.dedent(
@@ -116,50 +127,302 @@ class TestAccumulateAllViolations:
assert "block_one" in error
assert "block_two" in error
- def test_unknown_label_does_not_short_circuit_other_artifacts(self) -> None:
+ def test_unknown_label_is_dropped_and_other_artifacts_still_validated(self) -> None:
normalized, error = _normalize_code_artifact_metadata(
[_broken_metadata("ghost_label"), _broken_metadata("my_block")], _code_block_yaml("my_block")
)
assert normalized == {}
assert error is not None
- assert "does not reference an existing code block label" in error
- # The second artifact's shape violations are still surfaced.
+ # The stale entry is pruned, never rejected; the anchored artifact's
+ # shape violations are still surfaced.
+ assert "ghost_label" not in error
assert "requires `source_tool`" in error
def test_valid_metadata_passes(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("my_block")], _code_block_yaml("my_block")
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_empty_metadata_is_noop(self) -> None:
+ assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)
+ assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)
+
+
+_SCOUT_TRAJECTORY = [
+ {
+ "tool_name": "click",
+ "selector": "#search-submit",
+ "source_url": "https://registry.example.com/search",
+ "role": "button",
+ "accessible_name": "Search",
+ "trajectory_index": 0,
+ },
+]
+
+
+def _assert_passes_full_validator(row: dict) -> None:
+ """The imposed row must conform without any trajectory-driven defaulting."""
+ renormalized, error = _normalize_code_artifact_metadata([row], _code_block_yaml(row["block_label"]))
+ assert error is None, error
+ assert list(renormalized.keys()) == [row["block_label"]]
+
+
+class TestSeamImposition:
+ def test_minimal_metadata_is_fully_defaulted(self) -> None:
+ metadata = {"block_label": "my_block", "declared_goal": "Search the registry and expand result rows"}
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ row = normalized["my_block"]
+ assert row["artifact_id"] == "code_artifact:my_block"
+ assert row["page_dependencies"]
+ assert row["claimed_outcomes"]
+ assert row["completion_criteria"]
+ assert row["terminal_verifier_expectations"]
+ assert row["observation_refs"]
+ _assert_passes_full_validator(row)
+
+ def test_omitted_page_dependencies_filled_from_trajectory(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("page_dependencies")
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ dependency = normalized["my_block"]["page_dependencies"][0]
+ assert dependency["status"] == "observed_not_verified"
+ assert dependency["url_hint"] == "https://registry.example.com/search"
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_omitted_claims_derived_from_declared_goal(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("claimed_outcomes")
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ claim = normalized["my_block"]["claimed_outcomes"][0]
+ assert claim["text"] == "g"
+ assert claim["status"] == "observed_not_verified"
+ assert claim["depends_on"] == ["dependency:p"]
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_omitted_criteria_derived_from_claims(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("completion_criteria")
+ metadata["claimed_outcomes"][0].pop("covered_criteria")
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ row = normalized["my_block"]
+ criterion = row["completion_criteria"][0]
+ assert criterion["level"] == "terminal"
+ assert criterion["text"] == "x"
+ assert row["claimed_outcomes"][0]["covered_criteria"] == [criterion["id"]]
+ _assert_passes_full_validator(row)
+
+ def test_omitted_expectations_linked_to_criteria(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("terminal_verifier_expectations")
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ expectation = normalized["my_block"]["terminal_verifier_expectations"][0]
+ assert expectation["criteria_ids"] == ["criterion:c"]
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_omitted_artifact_refs_default_to_scout_observation(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("observation_refs")
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ ref = normalized["my_block"]["observation_refs"][0]
+ assert ref["source_tool"] == "scout_interaction"
+ assert ref["status"] == "observed_not_verified"
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_missing_ref_source_tool_and_scope_filled(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata["observation_refs"] = [{"observation_ref": "obs1", "status": "observed_not_verified"}]
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ ref = normalized["my_block"]["observation_refs"][0]
+ assert ref["source_tool"] == "scout_interaction"
+ assert ref["dependency_id"] == "dependency:p"
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_contradictory_checkpoint_advance_dropped(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata["observation_refs"][0]["checkpoint_next_mode"] = "advance"
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ assert "checkpoint_next_mode" not in normalized["my_block"]["observation_refs"][0]
+
+ def test_satisfied_claim_without_evidence_is_undefaultable(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata["claimed_outcomes"][0]["status"] = "satisfied"
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "is `satisfied` but has no" in error
+
+ def test_no_imposition_when_not_imposing(self) -> None:
+ metadata = {"block_label": "my_block", "declared_goal": "g"}
+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
+ assert normalized == {}
+ assert error is not None
+ assert "requires non-empty" in error
+
+ def test_imposition_works_without_scout_trajectory(self) -> None:
+ # The run-3 class: model-authored per-entry refs missing the scoped id
+ # and source_tool must be normalized, not rejected, even when no scout
+ # interaction was recorded before authoring.
+ metadata = {
+ "block_label": "my_block",
+ "declared_goal": "g",
+ "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],
+ }
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True
+ )
+ assert error is None
+ ref = normalized["my_block"]["observation_refs"][0]
+ assert ref["dependency_id"]
+ assert ref["source_tool"] == "scout_interaction"
+ assert "url_hint" not in normalized["my_block"]["page_dependencies"][0]
+ _assert_passes_full_validator(normalized["my_block"])
+
+ def test_imposition_fills_missing_mechanical_ids(self) -> None:
metadata = {
"block_label": "my_block",
- "artifact_id": "code_artifact:my_block",
"declared_goal": "g",
- "claimed_outcomes": [
- {
- "id": "claim:x",
- "scope": "outcome",
- "text": "x",
- "status": "observed_not_verified",
- "depends_on": ["dependency:p"],
- "covered_criteria": ["criterion:c"],
- "observation_refs": ["obs1"],
- }
- ],
- "page_dependencies": [
- {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
- ],
- "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
- "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
- "observation_refs": [
- {
- "observation_ref": "obs1",
- "dependency_id": "dependency:p",
- "status": "observed_not_verified",
- "source_tool": "scout_interaction",
- }
- ],
+ "claimed_outcomes": [{"text": "rows visible"}],
+ "completion_criteria": [{"text": "rows shown"}],
}
+ normalized, error = _normalize_code_artifact_metadata(
+ [metadata], _code_block_yaml("my_block"), impose_defaults=True, scout_trajectory=_SCOUT_TRAJECTORY
+ )
+ assert error is None
+ row = normalized["my_block"]
+ assert row["claimed_outcomes"][0]["id"]
+ assert row["claimed_outcomes"][0]["scope"] == "outcome"
+ assert row["completion_criteria"][0]["id"]
+ _assert_passes_full_validator(row)
+
+
+class TestPerLabelSalvage:
+ def test_conforming_label_survives_offending_label(self) -> None:
+ yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_one
+ code: |
+ await page.goto("https://example.com/")
+ - block_type: code
+ label: block_two
+ code: |
+ await page.goto("https://example.com/")
+ """
+ ).strip()
+ bad = _valid_metadata("block_two")
+ bad["claimed_outcomes"][0]["status"] = "satisfied"
+ normalized, error = _normalize_code_artifact_metadata([_valid_metadata("block_one"), bad], yaml)
+ assert list(normalized.keys()) == ["block_one"]
+ assert error is not None
+ assert "block_two" in error
+ assert "block_one" not in error
+
+ def test_unknown_label_dropped_alone(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("ghost_label"), _valid_metadata("my_block")], _code_block_yaml("my_block")
+ )
+ assert list(normalized.keys()) == ["my_block"]
+ assert error is None
+
+
+class TestStaleLabelRekey:
+ def test_single_stale_entry_rekeys_to_single_uncovered_label(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("stale_label")], _code_block_yaml("my_block")
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+ assert normalized["my_block"]["block_label"] == "my_block"
+ assert normalized["my_block"]["artifact_id"] == "code_artifact:my_block"
+
+ def test_label_less_entry_rekeys_to_single_uncovered_label(self) -> None:
+ metadata = _valid_metadata("my_block")
+ metadata.pop("block_label")
normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
assert error is None
assert list(normalized.keys()) == ["my_block"]
- def test_empty_metadata_is_noop(self) -> None:
- assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)
- assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)
+ def test_multiple_stale_entries_are_dropped_not_rekeyed(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("ghost_a"), _valid_metadata("ghost_b")], _code_block_yaml("my_block")
+ )
+ assert normalized == {}
+ assert error is None
+
+ def test_dropped_stale_entry_leaves_skeleton_for_uncovered_label_when_imposing(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("ghost_a"), _valid_metadata("ghost_b")],
+ _code_block_yaml("my_block"),
+ impose_defaults=True,
+ scout_trajectory=_SCOUT_TRAJECTORY,
+ )
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+ row = normalized["my_block"]
+ assert row["declared_goal"]
+ assert row["page_dependencies"]
+ assert row["observation_refs"]
+ _assert_passes_full_validator(row)
+
+ def test_partial_coverage_gets_skeleton_for_uncovered_label_when_imposing(self) -> None:
+ yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_one
+ code: |
+ await page.goto("https://example.com/")
+ - block_type: code
+ label: block_two
+ code: |
+ await page.goto("https://example.com/")
+ """
+ ).strip()
+ normalized, error = _normalize_code_artifact_metadata(
+ [_valid_metadata("block_one")],
+ yaml,
+ impose_defaults=True,
+ scout_trajectory=_SCOUT_TRAJECTORY,
+ )
+ assert error is None
+ assert sorted(normalized.keys()) == ["block_one", "block_two"]
+ _assert_passes_full_validator(normalized["block_two"])
+
+ def test_duplicate_label_keeps_first_entry(self) -> None:
+ first = _valid_metadata("my_block")
+ second = _valid_metadata("my_block")
+ second["declared_goal"] = "different"
+ normalized, error = _normalize_code_artifact_metadata([first, second], _code_block_yaml("my_block"))
+ assert error is None
+ assert normalized["my_block"]["declared_goal"] == "g"
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
new file mode 100644
index 000000000..7d0995437
--- /dev/null
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -0,0 +1,268 @@
+"""Tests for the code-block persist seam in `_update_workflow`.
+
+OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.
+"""
+
+from __future__ import annotations
+
+import textwrap
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot.blocker_signal import assert_clean_user_facing_text
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.tools import (
+ _code_block_safety_errors,
+ _detect_stale_block_metadata,
+ _update_workflow,
+)
+
+
+def _yaml(body: str) -> str:
+ return textwrap.dedent(body).strip() + "\n"
+
+
+_IMPORTING_CODE_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ import asyncio
+ await page.goto("https://example.com/search")
+ """
+)
+
+_SAFE_CODE_YAML = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: search_registry
+ code: |
+ await page.goto("https://example.com/search")
+ """
+)
+
+
+def _code_only_ctx() -> CopilotContext:
+ ctx = CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+ ctx.block_authoring_policy = BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ctx.scout_trajectory = [
+ {
+ "tool_name": "click",
+ "selector": "#search-submit",
+ "source_url": "https://example.com/search",
+ "trajectory_index": 0,
+ }
+ ]
+ return ctx
+
+
+class TestCodeSafetySeam:
+ def test_import_in_new_code_block_is_a_seam_error(self) -> None:
+ errors = _code_block_safety_errors(_IMPORTING_CODE_YAML, None)
+ assert len(errors) == 1
+ assert "search_registry" in errors[0]
+ assert "Not allowed to import modules" in errors[0]
+
+ def test_unchanged_legacy_code_block_is_not_rechecked(self) -> None:
+ assert _code_block_safety_errors(_IMPORTING_CODE_YAML, _IMPORTING_CODE_YAML) == []
+
+ def test_changed_code_block_is_rechecked(self) -> None:
+ assert _code_block_safety_errors(_IMPORTING_CODE_YAML, _SAFE_CODE_YAML)
+
+ def test_safe_code_passes(self) -> None:
+ assert _code_block_safety_errors(_SAFE_CODE_YAML, None) == []
+
+ def test_syntax_error_is_a_seam_error(self) -> None:
+ broken = _SAFE_CODE_YAML.replace('await page.goto("https://example.com/search")', "await page.goto(")
+ errors = _code_block_safety_errors(broken, None)
+ assert len(errors) == 1
+ assert "not valid Python" in errors[0]
+
+ @pytest.mark.asyncio
+ async def test_update_workflow_rejects_import_before_any_run(self) -> None:
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": _IMPORTING_CODE_YAML}, ctx)
+ assert result["ok"] is False
+ assert "Not allowed to import modules" in result["error"]
+ assert "import" not in result["user_facing_summary"]
+ assert result["user_facing_summary"]
+
+ @pytest.mark.asyncio
+ async def test_code_rejection_does_not_salvage_metadata_into_ctx(self) -> None:
+ ctx = _code_only_ctx()
+ metadata = [{"block_label": "search_registry", "declared_goal": "search the registry"}]
+ result = await _update_workflow(
+ {"workflow_yaml": _IMPORTING_CODE_YAML, "code_artifact_metadata": metadata}, ctx
+ )
+ assert result["ok"] is False
+ assert ctx.code_artifact_metadata == {}
+
+
+class TestSeamSalvageIntoContext:
+ @pytest.mark.asyncio
+ async def test_conforming_label_persists_when_sibling_label_rejects(self) -> None:
+ workflow_yaml = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_one
+ code: |
+ await page.goto("https://example.com/search")
+ - block_type: code
+ label: block_two
+ code: |
+ await page.goto("https://example.com/results")
+ """
+ )
+ metadata = [
+ {"block_label": "block_one", "declared_goal": "search the registry"},
+ {
+ "block_label": "block_two",
+ "declared_goal": "expand the result rows",
+ "claimed_outcomes": [
+ {"id": "claim:rows", "scope": "outcome", "text": "rows expanded", "status": "satisfied"}
+ ],
+ },
+ ]
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": workflow_yaml, "code_artifact_metadata": metadata}, ctx)
+
+ assert result["ok"] is False
+ assert "block_two" in result["error"]
+ assert "block_one" not in result["error"]
+ assert list(ctx.code_artifact_metadata.keys()) == ["block_one"]
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+ assert "contract violation" not in result["user_facing_summary"]
+ assert "`" not in result["user_facing_summary"]
+
+ @pytest.mark.asyncio
+ async def test_stale_entry_dropped_and_draft_metadata_survives_unaccepted_submission(self) -> None:
+ draft_yaml = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_a
+ code: |
+ await page.goto("https://example.com/search")
+ - block_type: code
+ label: block_b
+ code: |
+ await page.goto("https://example.com/results")
+ """
+ )
+ submitted_yaml = _yaml(
+ """
+ title: Registry lookup
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_a
+ code: |
+ await page.goto("https://example.com/search")
+ """
+ )
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = draft_yaml
+ stored_block_b = {"block_label": "block_b", "declared_goal": "expand the result rows"}
+ ctx.code_artifact_metadata = {
+ "block_a": {"block_label": "block_a", "declared_goal": "search the registry"},
+ "block_b": stored_block_b,
+ }
+ ctx.workflow_verification_evidence.code_artifact_metadata = dict(ctx.code_artifact_metadata)
+ metadata = [
+ {"block_label": "block_a", "declared_goal": "search the registry"},
+ {"block_label": "ghost", "declared_goal": "does not exist"},
+ ]
+
+ result = await _update_workflow({"workflow_yaml": submitted_yaml, "code_artifact_metadata": metadata}, ctx)
+
+ # The stale entry is pruned at the seam; the submission proceeds and
+ # only non-metadata gates may reject it, so prior-draft metadata stays.
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "ghost" not in error_text
+ assert sorted(ctx.code_artifact_metadata) == ["block_a", "block_b"]
+ assert ctx.code_artifact_metadata["block_b"] == stored_block_b
+ assert ctx.workflow_verification_evidence.code_artifact_metadata == ctx.code_artifact_metadata
+
+ @pytest.mark.asyncio
+ async def test_minimal_metadata_with_trajectory_produces_no_violation_error(self) -> None:
+ ctx = _code_only_ctx()
+ metadata = [{"block_label": "search_registry", "declared_goal": "search the registry"}]
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)
+ # The seam may reject later (credential checks need the app); the metadata
+ # contract itself must not be the rejection.
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "contract violation" not in error_text
+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
+
+
+class TestStaleLabelSeamFlow:
+ @pytest.mark.asyncio
+ async def test_stale_metadata_label_rekeys_without_any_stale_rejection(self) -> None:
+ # Run-2 shape: metadata keyed to a label absent from the submitted
+ # YAML. The seam re-keys it to the only uncovered code block, so
+ # neither the metadata gate nor the stale-block-metadata validation
+ # path can bounce the submission back to the model.
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _SAFE_CODE_YAML
+ metadata = [{"block_label": "search_certificant_stale", "declared_goal": "search the registry"}]
+
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)
+
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "still appears stale" not in error_text
+ assert list(ctx.code_artifact_metadata.keys()) == ["search_registry"]
+ assert ctx.code_artifact_metadata["search_registry"]["artifact_id"] == "code_artifact:search_registry"
+ # The seam never rewrites YAML labels, so its output cannot trip the
+ # stale-block-metadata validation that fires on label/title renames.
+ assert _detect_stale_block_metadata(_SAFE_CODE_YAML, ctx.workflow_yaml) == []
+
+ @pytest.mark.asyncio
+ async def test_malformed_per_entry_refs_normalize_without_scout_interactions(self) -> None:
+ # Run-3 shape: model-authored observation_refs rows missing the scoped
+ # id, authored before any scout interaction was recorded.
+ ctx = _code_only_ctx()
+ ctx.scout_trajectory = []
+ metadata = [
+ {
+ "block_label": "search_registry",
+ "declared_goal": "search the registry",
+ "observation_refs": [{"observation_ref": "obs1", "status": "observed_not_verified"}],
+ }
+ ]
+
+ result = await _update_workflow({"workflow_yaml": _SAFE_CODE_YAML, "code_artifact_metadata": metadata}, ctx)
+
+ error_text = str(result.get("error") or "")
+ assert "Artifact metadata" not in error_text
+ assert "contract violation" not in error_text
+ ref = ctx.code_artifact_metadata["search_registry"]["observation_refs"][0]
+ assert ref["dependency_id"]
+ assert ref["source_tool"]
+
+
+def test_run_id_leak_check_covers_non_numeric_ids() -> None:
+ with pytest.raises(ValueError):
+ assert_clean_user_facing_text("Outcome uncertain for wr_sample_123abc.")
diff --git a/tests/unit/test_copilot_composition_evidence.py b/tests/unit/test_copilot_composition_evidence.py
index dd2dd66fd..3aa143a93 100644
--- a/tests/unit/test_copilot_composition_evidence.py
+++ b/tests/unit/test_copilot_composition_evidence.py
@@ -2443,3 +2443,109 @@ async def test_capture_prefers_html_parse_over_hollow_structured_on_fallback(mon
assert "Welcome notice text" in evidence["visible_text_excerpt"]
assert evidence["schema_empty_page"] is True
assert server.calls.count("skyvern_get_html") == 1
+
+
+class TestSemanticChallengeSplit:
+ def test_passive_vendor_markup_does_not_assert_human_verification(self) -> None:
+ html = """
+ <html><head><title>Registry search results</title>
+ <script src="https://cdn.example/challenge-platform/api.js"></script>
+ </head><body>
+ <form><input name="q" /><button type="submit" disabled>Search</button></form>
+ <table id="results"><tbody><tr><td>SAMPLE, PERSON</td></tr></tbody></table>
+ </body></html>
+ """
+ parsed = parse_composition_html(
+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"
+ )
+ state = parsed["challenge_state"]
+ assert state["detected"] is True
+ assert state["requires_human_verification"] is False
+ assert state["gates_submit_controls"] is False
+ # Substring detection still arms the visual fallback for confirmation.
+ assert page_evidence_needs_visual_fallback(parsed) is True
+
+ def test_rendered_challenge_widget_asserts_human_verification(self) -> None:
+ html = """
+ <html><head><title>Verify you are human</title></head><body>
+ <form>
+ <input name="q" />
+ <div id="cf-turnstile-widget" data-sitekey="abc"></div>
+ <button type="submit" disabled>Search</button>
+ </form>
+ </body></html>
+ """
+ parsed = parse_composition_html(
+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"
+ )
+ state = parsed["challenge_state"]
+ assert state["detected"] is True
+ assert state["requires_human_verification"] is True
+ assert state["gates_submit_controls"] is True
+
+ def test_consent_typed_visual_obstruction_does_not_promote_challenge(self) -> None:
+ html = """
+ <html><head><title>Search</title>
+ <script src="https://cdn.example/challenge-platform/api.js"></script>
+ </head><body>
+ <form><input name="q" /><button type="submit" disabled>Search</button></form>
+ </body></html>
+ """
+ parsed = parse_composition_html(
+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"
+ )
+ merged = merge_visual_composition_evidence(
+ parsed,
+ visual_summary={
+ "summary": "A privacy settings dialog covers the page.",
+ "challenge_detected": True,
+ "challenge_kind": "",
+ "challenge_location": "",
+ "submit_blocked": True,
+ "blocked_submit_controls": ["Search"],
+ "page_obstruction_detected": True,
+ "obstruction_kind": "cookie_consent",
+ "obstruction_location": "center",
+ "underlying_page_blocked": True,
+ "visible_dismiss_controls": ["Accept all"],
+ "omissions": [],
+ },
+ )
+ state = merged["challenge_state"]
+ assert state["requires_human_verification"] is False
+ assert state["gates_submit_controls"] is False
+ kinds = [obstruction.get("kind") for obstruction in merged["page_obstructions"]]
+ assert "cookie_consent" in kinds
+
+ def test_vision_confirmation_still_promotes_challenge(self) -> None:
+ html = """
+ <html><head><title>Search</title>
+ <script src="https://cdn.example/challenge-platform/api.js"></script>
+ </head><body>
+ <form><input name="q" /><button type="submit" disabled>Search</button></form>
+ </body></html>
+ """
+ parsed = parse_composition_html(
+ html, inspected_url="https://example.com/search", current_url="https://example.com/search"
+ )
+ merged = merge_visual_composition_evidence(
+ parsed,
+ visual_summary={
+ "summary": "A human-verification widget sits above the Search button.",
+ "challenge_detected": True,
+ "challenge_kind": "human_verification",
+ "challenge_location": "above the Search button",
+ "submit_blocked": True,
+ "blocked_submit_controls": ["Search"],
+ "page_obstruction_detected": False,
+ "obstruction_kind": "",
+ "obstruction_location": "",
+ "underlying_page_blocked": None,
+ "visible_dismiss_controls": [],
+ "omissions": [],
+ },
+ )
+ state = merged["challenge_state"]
+ assert state["requires_human_verification"] is True
+ assert state["gates_submit_controls"] is True
+ assert {"text": "Search", "disabled": True} in state["gated_submit_controls"]
diff --git a/tests/unit/test_copilot_per_tool_budget.py b/tests/unit/test_copilot_per_tool_budget.py
index 8e596a7fe..6ad1a68e5 100644
--- a/tests/unit/test_copilot_per_tool_budget.py
+++ b/tests/unit/test_copilot_per_tool_budget.py
@@ -43,6 +43,8 @@ from skyvern.forge.sdk.copilot.tools import (
_mark_page_inspected,
_mark_pending_reconciliation_run,
_maybe_clear_reconciliation_flag,
+ _post_budget_terminal_challenge_signal,
+ _post_run_terminal_challenge_reason,
_record_composition_page_observation,
_record_per_tool_budget_problem_blocks_from_results,
_record_run_blocks_result,
@@ -1069,3 +1071,103 @@ def test_nudge_counter_preserved_across_consecutive_budget_trips() -> None:
assert ctx.per_tool_budget_nudge_count == 1
assert ctx.last_failure_category_top == PER_TOOL_BUDGET_FAILURE_CATEGORY
+
+
+def _challenge_gated_post_run_evidence(**extra: object) -> dict:
+ evidence = {
+ "current_url": "https://example.com/registry/search",
+ "workflow_run_id": "wr_budget",
+ "observed_after_workflow_run": True,
+ "anti_bot_indicators": ["human-verification"],
+ "forms": [{"submit_controls": [{"text": "Search", "selector": "#search", "disabled": True}]}],
+ "challenge_state": {
+ "detected": True,
+ "kind": "human_verification",
+ "requires_human_verification": True,
+ "gates_submit_controls": True,
+ "gated_submit_controls": [{"text": "Search", "disabled": True}],
+ },
+ }
+ evidence.update(extra)
+ return evidence
+
+
+def _armed_post_run_context() -> CopilotContext:
+ ctx = _fresh_context()
+ ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY
+ ctx.last_test_ok = False
+ ctx.per_tool_budget_problem_block_labels = ["submit_registry_search"]
+ return ctx
+
+
+def test_terminal_challenge_routes_to_answer_path_when_result_evidence_in_same_packet() -> None:
+ ctx = _armed_post_run_context()
+ ctx.composition_page_evidence = _challenge_gated_post_run_evidence(
+ result_containers=[{"tag": "table", "selector": "#results", "row_selector": "#results tbody tr"}],
+ visible_text_excerpt="SAMPLE, PERSON Anytown, XY Exampleland Certified Active",
+ )
+
+ signal = _post_budget_terminal_challenge_signal(
+ ctx, {"block_labels": ["submit_registry_search"]}, "update_and_run_blocks"
+ )
+
+ assert signal is not None
+ assert signal.internal_reason_code == "tool_error_post_budget_challenge_result_evidence"
+ assert signal.renders_final_reply is False
+ assert signal.preserves_workflow_draft is True
+ assert "Answer from the observed page" in signal.agent_steering_text
+ assert "result container" in signal.agent_steering_text
+
+
+def test_terminal_challenge_fires_without_result_evidence_and_names_observed_facts() -> None:
+ ctx = _armed_post_run_context()
+ ctx.composition_page_evidence = _challenge_gated_post_run_evidence()
+
+ signal = _post_budget_terminal_challenge_signal(ctx, None, "update_and_run_blocks")
+
+ assert signal is not None
+ assert signal.internal_reason_code == "tool_error_post_budget_challenge_blocker"
+ assert signal.renders_final_reply is True
+ assert "human verification" in signal.user_facing_reason
+ assert "Search" in signal.user_facing_reason
+ assert "per-tool-call budget" not in signal.user_facing_reason
+ assert "wr_" not in signal.user_facing_reason
+
+
+def test_bare_substring_indicators_do_not_produce_terminal_reason() -> None:
+ # A passive challenge-vendor token over raw HTML plus an unrelated disabled
+ # submit control must not assert a terminal blocked claim.
+ evidence = _challenge_gated_post_run_evidence(
+ challenge_state={
+ "detected": True,
+ "kind": "human_verification",
+ "requires_human_verification": False,
+ "gates_submit_controls": False,
+ "gated_submit_controls": [],
+ },
+ )
+ evidence["anti_bot_indicators"] = ["challenge"]
+ evidence["challenge_controls"] = [{"tag": "script", "src": "https://cdn.example/challenge-platform/api.js"}]
+
+ assert _post_run_terminal_challenge_reason(evidence) is None
+
+ ctx = _armed_post_run_context()
+ ctx.composition_page_evidence = evidence
+ assert _post_budget_terminal_challenge_signal(ctx, None, "update_and_run_blocks") is None
+
+
+def test_rendered_challenge_widget_still_produces_terminal_reason() -> None:
+ evidence = _challenge_gated_post_run_evidence(
+ challenge_state={
+ "detected": True,
+ "kind": "captcha",
+ "requires_human_verification": False,
+ "gates_submit_controls": False,
+ "gated_submit_controls": [],
+ },
+ )
+ evidence["challenge_controls"] = [{"tag": "div", "class": "cf-turnstile", "data_sitekey": "abc"}]
+
+ reason = _post_run_terminal_challenge_reason(evidence)
+ assert reason is not None
+ assert "disabled" in reason
diff --git a/tests/unit/test_copilot_terminal_halt_narrative.py b/tests/unit/test_copilot_terminal_halt_narrative.py
new file mode 100644
index 000000000..1fd6843b4
--- /dev/null
+++ b/tests/unit/test_copilot_terminal_halt_narrative.py
@@ -0,0 +1,155 @@
+"""Terminal-output invariant and observed-facts halt narrative tests.
+
+OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.forge.sdk.copilot.agent import (
+ _build_wip_exit_result,
+ _clean_recorded_failure_text,
+ _observed_facts_halt_reply,
+ _recorded_failure_reply,
+)
+from skyvern.forge.sdk.copilot.blocker_signal import contains_internal_machinery_leak
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY
+from skyvern.forge.sdk.copilot.output_policy import (
+ CopilotOutputKind,
+ OutputPolicyReason,
+ evaluate_output_policy,
+)
+
+_ANTI_RERUN_GUARD_TEXT = (
+ "The prior PER_TOOL_BUDGET run for run wr_538488327000571062 advanced the live browser at "
+ "https://www.example.com/. Before another block-running tool, inspect the current browser page with "
+ 'inspect_page_for_composition(target_url="current_page").'
+)
+
+_PACING_GUARD_TEXT = "The run exceeded the 3s per-tool-call budget while still making progress."
+
+
+def _ctx() -> CopilotContext:
+ return CopilotContext(
+ organization_id="o",
+ workflow_id="w",
+ workflow_permanent_id="wp",
+ workflow_yaml="",
+ browser_session_id=None,
+ stream=SimpleNamespace(), # type: ignore[arg-type]
+ )
+
+
+class TestInternalMachineryLeakPredicate:
+ @pytest.mark.parametrize(
+ "text",
+ [
+ _ANTI_RERUN_GUARD_TEXT,
+ _PACING_GUARD_TEXT,
+ "Outcome is uncertain for run wr_538438176486379954.",
+ "Browser session pbs_538476723643067648 is live.",
+ "Before another block-running tool, inspect the current browser page.",
+ "Call update_and_run_blocks with the smaller chain.",
+ "Use inspect_page_for_composition to confirm the page state.",
+ ],
+ )
+ def test_internal_machinery_is_detected(self, text: str) -> None:
+ assert contains_internal_machinery_leak(text) is True
+
+ @pytest.mark.parametrize(
+ "text",
+ [
+ "I created and tested the workflow successfully.",
+ "I built a 2-block draft and was still testing it when the turn ran out of time.",
+ "The site's verification page was still blocking the search control.",
+ "Send me the registry URL you want to search.",
+ "",
+ ],
+ )
+ def test_product_language_is_clean(self, text: str) -> None:
+ assert contains_internal_machinery_leak(text) is False
+
+ def test_output_policy_hard_blocks_guard_text(self) -> None:
+ verdict = evaluate_output_policy(
+ request_policy=None,
+ response_type="REPLY",
+ user_response=_ANTI_RERUN_GUARD_TEXT,
+ output_kind=CopilotOutputKind.INFORMATIONAL_ANSWER,
+ )
+ assert OutputPolicyReason.INTERNAL_TOOL_INSTRUCTION_LEAK in verdict.reason_codes
+
+
+class TestRecordedFailureReply:
+ def test_guard_text_renders_observed_facts_not_test_failed(self) -> None:
+ ctx = _ctx()
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))
+ ctx.last_update_block_count = 1
+ ctx.last_test_ok = False
+ ctx.last_test_failure_reason = _ANTI_RERUN_GUARD_TEXT
+ ctx.workflow_verification_evidence.current_url = "https://www.example.com/registry"
+
+ reply = _recorded_failure_reply(ctx)
+
+ assert reply is not None
+ assert "wr_" not in reply
+ assert "PER_TOOL_BUDGET" not in reply
+ assert "per-tool-call budget" not in reply
+ assert "block-running tool" not in reply
+ assert "the test failed" not in reply
+ assert "ran out of time" in reply
+ assert "1-block draft" in reply
+ assert "https://www.example.com/registry" in reply
+
+ def test_budget_paced_run_does_not_render_as_test_failed(self) -> None:
+ ctx = _ctx()
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))
+ ctx.last_update_block_count = 1
+ ctx.last_test_ok = False
+ ctx.last_failure_category_top = PER_TOOL_BUDGET_FAILURE_CATEGORY
+ ctx.last_test_failure_reason = "The run was canceled while still making progress."
+
+ reply = _recorded_failure_reply(ctx)
+
+ assert reply is not None
+ assert "the test failed" not in reply
+ assert "ran out of time" in reply
+
+ def test_fragment_cleaner_substitutes_guard_text(self) -> None:
+ cleaned = _clean_recorded_failure_text(_ANTI_RERUN_GUARD_TEXT)
+ assert "wr_" not in cleaned
+ assert "PER_TOOL_BUDGET" not in cleaned
+
+ def test_observed_facts_reply_without_draft_states_timeout(self) -> None:
+ ctx = _ctx()
+ reply = _observed_facts_halt_reply(ctx)
+ assert "ran out of time" in reply
+ assert not contains_internal_machinery_leak(reply)
+
+
+class TestWipExitTerminalInvariant:
+ def test_timeout_exit_with_guard_failure_reason_renders_clean_terminal(self) -> None:
+ ctx = _ctx()
+ ctx.last_workflow = SimpleNamespace(workflow_definition=SimpleNamespace(blocks=[object()]))
+ ctx.last_workflow_yaml = "title: wf\nworkflow_definition:\n blocks: []\n"
+ ctx.last_update_block_count = 1
+ ctx.last_test_ok = False
+ ctx.last_test_failure_reason = _ANTI_RERUN_GUARD_TEXT
+ ctx.workflow_verification_evidence.current_url = "https://www.example.com/registry"
+
+ result = _build_wip_exit_result(
+ ctx,
+ None,
+ default_reply="I ran out of time processing your request.",
+ unvalidated_reply="I ran out of time; the draft is untested.",
+ tested_reply="I ran out of time, but I have a tested draft for you.",
+ terminal_reason="timeout",
+ )
+
+ assert not contains_internal_machinery_leak(result.user_response)
+ assert "the test failed" not in result.user_response
+ # The halt surfaces the draft for review instead of discarding it.
+ assert result.updated_workflow is ctx.last_workflow
diff --git a/tests/unit/test_copilot_timeout_wip.py b/tests/unit/test_copilot_timeout_wip.py
index 484365139..a19100e72 100644
--- a/tests/unit/test_copilot_timeout_wip.py
+++ b/tests/unit/test_copilot_timeout_wip.py
@@ -311,10 +311,9 @@ class TestBuildUnexpectedErrorExitResult:
result = _build_unexpected_error_exit_result(ctx, global_llm_context=None)
assert result.user_response.startswith(
- "I built a 3-block draft and tested it, but the test couldn't finish: "
- "The previous workflow run did not finish before the turn budget expired. "
- "Last run status: canceled."
+ "I built a 3-block draft and was still testing it when the turn ran out of time."
)
+ assert "the test failed" not in result.user_response
assert "draft workflow proposal" in result.user_response
assert "Do NOT" not in result.user_response
assert "block-running tools" not in result.user_response