Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
SKY-10795: batch artifact-metadata violations + deterministic metadata for synthesized code blocks (#6464)
Andrew Neilsonf8b52147ce67
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex a2b2f776f..d7ab2f595 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -38,7 +38,8 @@ from skyvern.forge.sdk.copilot.blocker_signal import ()from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_datafrom skyvern.forge.sdk.copilot.build_phase import initial_build_phase-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig+from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig, normalize_block_authoring_policyfrom skyvern.forge.sdk.copilot.context import (COPILOT_RESPONSE_TYPES,AgentResult,@@ -160,6 +161,12 @@ Code block runtime facts:- For multi-line code in workflow YAML, always use a YAML block scalar(`code: |`). Never include placeholder text such as `[... truncated ...]`;pass complete workflow YAML to update tools.++When a SYNTHESIZED CODE BLOCK is offered to you, it already encodes the page+interactions you scouted as deterministic Playwright. Persist that block VERBATIM+via update_workflow / update_and_run_blocks — do not rewrite, reorder, or+re-derive its locators. Only hand-author the steps it does not cover, such as the+extraction or report block that returns the structured result."""@@ -475,6 +482,43 @@ def _runtime_verification_evidence_prompt(ctx: CopilotContext | None) -> str:)+def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:+ """Pre-authoring offer of the synthesized code block.++ Trips the ``synthesized_block_offered`` latch (shared with the post-turn enforcement offer)+ only when a non-None offer is actually rendered, so an empty trajectory leaves it open.+ """+ if ctx is None:+ return ""+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="policy_not_code_only_browser")+ return ""+ if ctx.update_workflow_called:+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="already_authored")+ return ""+ if ctx.synthesized_block_offered:+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="already_offered")+ return ""+ if not ctx.scout_trajectory:+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="empty_trajectory")+ return ""+ synthesized = synthesize_code_block(ctx.scout_trajectory)+ if synthesized is None:+ LOG.debug(+ "copilot_synthesized_block_offer_skipped",+ reason="synthesis_returned_none",+ trajectory_len=len(ctx.scout_trajectory),+ )+ return ""+ ctx.synthesized_block_offered = True+ LOG.info(+ "copilot_synthesized_block_offer_rendered",+ trajectory_len=len(ctx.scout_trajectory),+ code_len=len(synthesized.code),+ )+ return "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory)++def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -> Callable[[object, object], str]:base_system_prompt = _build_system_prompt(tool_usage_guide=tool_usage_guide, config=config)@@ -501,7 +545,12 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -+ "and tell the user to store the redacted secret as a saved credential before testing. "+ "If `resolved_credentials` are present, use those `credential_id` values.")- return prompt + _runtime_verification_evidence_prompt(ctx) + _docs_answer_turn_directive(ctx.turn_intent)+ return (+ prompt+ + _runtime_verification_evidence_prompt(ctx)+ + _synthesized_block_offer_prompt(ctx)+ + _docs_answer_turn_directive(ctx.turn_intent)+ )return instructionsdiff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.pyindex 3c54a0baf..edf0661b9 100644--- a/skyvern/forge/sdk/copilot/code_block_preflight.py+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py@@ -68,8 +68,16 @@ def preflight_code_block(def _static_ast_diagnostics(code: str) -> list[CodeBlockPreflightDiagnostic]:try:tree = ast.parse(_build_typed_module(code, parameter_keys=()))- except SyntaxError:- return []+ except SyntaxError as exc:+ # The wrapper scaffolding is static and valid, so any SyntaxError is in the supplied code —+ # e.g. an attacker-page string with a raw line-boundary codepoint that splits a literal. Surface+ # it at authoring time instead of letting the block fail silently at run time.+ return [+ CodeBlockPreflightDiagnostic(+ code="SYNTAX_ERROR",+ message=f"Code block does not parse as Python: {exc.msg}. Fix the snippet before persisting it.",+ )+ ]diagnostics: list[CodeBlockPreflightDiagnostic] = []for node in ast.walk(tree):diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.pynew file mode 100644index 000000000..c90a9e0e6--- /dev/null+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py@@ -0,0 +1,379 @@+"""Deterministic synthesis of a copilot `code` block from a scout trajectory.++Pure module: ``synthesize_code_block`` is a function of its input trajectory+only — no LLM, no I/O, byte-identical output per trajectory. It turns the+scout's captured interaction sequence into a bounded, linear Playwright snippet+that runs on the raw ``page`` object the copilot code block executes against.+"""++from __future__ import annotations++import json+import keyword+import re+from collections.abc import Mapping, Sequence+from dataclasses import dataclass, field+from typing import Any++from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL++_MAX_STEPS = 20+_INDENT = " "++_SYNTHESIZED_BLOCK_LABEL = "scout_synthesized_browser_steps"++# Names the code-block executor reserves in its exec() namespace (block.py build_safe_vars+# plus the injected `page`). A parameter key colliding with one of these is silently dropped+# at bind time, so the synthesized fill would stringify the builtin instead of the user value.+_RESERVED_PARAM_NAMES = frozenset(+ {+ "page",+ "print",+ "len",+ "range",+ "str",+ "int",+ "float",+ "dict",+ "list",+ "tuple",+ "set",+ "bool",+ "isinstance",+ "enumerate",+ "any",+ "all",+ "max",+ "min",+ "sum",+ "sorted",+ "sleep",+ "asyncio",+ "re",+ "json",+ "html",+ "Exception",+ }+)++# role=<role>[name="<name>"] optionally followed by `>> nth=<n>` or other engines.+_ROLE_NAME_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')++# Positional/index engines whose match depends on document order, not element identity. A captured+# selector containing one of these is fragile, so an ARIA role/name anchor (when available) is preferred.+_POSITIONAL_RE = re.compile(+ r":nth-of-type\(|:nth-child\(|:nth-last-of-type\(|:nth-last-child\(|>>\s*nth=|:first-child|:last-child"+)+++@dataclass+class SynthesizedCodeBlock:+ code: str+ parameters: list[dict[str, str]] = field(default_factory=list)+ notes: list[str] = field(default_factory=list)+++# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a+# captured accessible name or option value would split the emitted one-line literal across lines and+# corrupt the block. repr() does not escape U+2028/U+2029, so they (and the C0/C1 controls below) are+# escaped explicitly to keep every emitted literal single-line.+_EXTRA_LINE_SEPARATORS = ("\u2028", "\u2029")+_CONTROL_CODEPOINTS = frozenset(+ chr(cp) for cp in (*range(0x00, 0x20), 0x7F, *range(0x80, 0xA0)) if chr(cp) not in ("\n", "\r")+)+++def _py_str(value: str) -> str:+ """A deterministic double-quoted Python string literal."""+ escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r")+ escaped = "".join(f"\\x{ord(ch):02x}" if ch in _CONTROL_CODEPOINTS else ch for ch in escaped)+ for separator in _EXTRA_LINE_SEPARATORS:+ escaped = escaped.replace(separator, f"\\u{ord(separator):04x}")+ return f'"{escaped}"'+++def _unescape_role_name(value: str) -> str:+ # ref_to_selector escapes only `"` as `\"`; reverse it for the get_by_role name.+ return value.replace('\\"', '"')+++def _parse_role_name(selector: str) -> tuple[str, str | None, str] | None:+ """Parse a `role=...[name="..."]` selector. Returns (role, name, suffix) or None.++ ``suffix`` is the residual engine chain (e.g. ` >> nth=2`); a non-empty suffix+ means the selector cannot be expressed as a plain get_by_role and the caller+ falls back to page.locator.+ """+ match = _ROLE_NAME_RE.match(selector)+ if not match:+ return None+ role, raw_name, suffix = match.group(1), match.group(2), match.group(3)+ name = _unescape_role_name(raw_name) if raw_name is not None else None+ return role, name, suffix.strip()+++def _is_positional_selector(selector: str) -> bool:+ """True when the captured selector's match depends on document position, not element identity.++ Stable anchors (id, [name=...], [data-testid=...], [aria-label=...], a non-indexed CSS path) are+ preferred verbatim; only a positional/index selector is worth trading for an ARIA role/name anchor.+ """+ return bool(_POSITIONAL_RE.search(selector))+++def _slug(value: str) -> str:+ cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", value.strip().lower()).strip("_")+ if cleaned and cleaned[0].isdigit():+ cleaned = f"field_{cleaned}"+ return cleaned or "field"+++def _safe_param_base(value: str) -> str:+ base = _slug(value)+ if keyword.iskeyword(base) or base in _RESERVED_PARAM_NAMES:+ return f"{base}_field"+ return base+++def _get_by_role_expr(role: str, name: str) -> str:+ if name:+ return f"page.get_by_role({_py_str(role)}, name={_py_str(name)})"+ return f"page.get_by_role({_py_str(role)})"+++def _locator_expr(+ interaction: Mapping[str, Any],+ notes: list[str],+) -> str:+ """Selector-first: emit the scout's captured working selector verbatim, and only fall back to an+ ARIA get_by_role anchor when that selector is positional/index-based (and a role/name is available).++ The accessible name is read on the scout/MCP surface; a `code:` block runs on a raw Playwright page+ where get_by_role's native name computation may not reproduce it. The captured selector is the proven,+ timing-independent locator the scout actually drove, so it wins for any stable selector.+ """+ selector = str(interaction.get("selector") or "").strip()+ role = str(interaction.get("role") or "").strip()+ name = str(interaction.get("accessible_name") or "").strip()++ parsed = _parse_role_name(selector) if selector else None+ if parsed is not None:+ parsed_role, parsed_name, suffix = parsed+ # A `role=...` selector is itself an ARIA anchor (ref_to_selector form), not a native CSS path —+ # there is no underlying stable selector to prefer, so express it as get_by_role.+ if not suffix:+ return _get_by_role_expr(role or parsed_role, name or (parsed_name or ""))+ # Residual engine chain (e.g. `>> nth=`) makes the parsed form positional; anchor on role/name.+ anchor_role = role or parsed_role+ anchor_name = name or (parsed_name or "")+ if anchor_role and anchor_name:+ return _get_by_role_expr(anchor_role, anchor_name)+ return f"page.locator({_py_str(selector)})"++ if selector:+ if _is_positional_selector(selector) and role and name:+ return _get_by_role_expr(role, name)+ if _is_positional_selector(selector):+ notes.append(f"low-confidence locator: positional selector {selector!r} with no role/name to anchor on")+ return f"page.locator({_py_str(selector)})"++ if role and name:+ return _get_by_role_expr(role, name)++ notes.append("dropped an interaction with no selector and no role/name")+ return ""+++def _param_key(interaction: Mapping[str, Any], used: set[str]) -> str:+ name = str(interaction.get("accessible_name") or "").strip()+ role = str(interaction.get("role") or "").strip()+ base = _safe_param_base(name or role or "value")+ candidate = base+ suffix = 2+ while candidate in used:+ candidate = f"{base}_{suffix}"+ suffix += 1+ used.add(candidate)+ return candidate+++def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> SynthesizedCodeBlock | None:+ """Deterministically synthesize a code block from a scout trajectory, or None if empty."""+ if not trajectory:+ return None++ lines: list[str] = []+ notes: list[str] = []+ parameters: list[dict[str, str]] = []+ used_param_keys: set[str] = set()++ entry_url = ""+ entry_index = -1+ for index, interaction in enumerate(trajectory):+ candidate = str(interaction.get("source_url") or "").strip()+ if candidate:+ entry_url = candidate+ entry_index = index+ break+ if entry_url:+ if entry_index > 0:+ notes.append("entry URL taken from a later interaction; earlier steps had no source_url")+ lines.append(f"{_INDENT}await page.goto({_py_str(entry_url)})")+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')++ emitted = 0+ for interaction in trajectory:+ if emitted >= _MAX_STEPS:+ notes.append(f"trajectory truncated at {_MAX_STEPS} steps")+ break+ tool_name = str(interaction.get("tool_name") or "")++ if tool_name == "press_key":+ key = str(interaction.get("key") or "").strip()+ if not key:+ continue+ locator = _locator_expr(interaction, notes) if interaction.get("selector") else ""+ if locator:+ lines.append(f"{_INDENT}await {locator}.press({_py_str(key)})")+ else:+ lines.append(f"{_INDENT}await page.keyboard.press({_py_str(key)})")+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')+ emitted += 1+ continue++ locator = _locator_expr(interaction, notes)+ if not locator:+ continue++ if tool_name == "click":+ lines.append(f"{_INDENT}await {locator}.click()")+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')+ elif tool_name == "type_text":+ param_key = _param_key(interaction, used_param_keys)+ parameters.append({"key": param_key})+ lines.append(f"{_INDENT}await {locator}.fill(str({param_key}))")+ elif tool_name == "select_option":+ value = str(interaction.get("value") or "").strip()+ if not value:+ notes.append("dropped a select_option interaction with no recorded value")+ continue+ lines.append(f"{_INDENT}await {locator}.select_option({_py_str(value)})")+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')+ else:+ notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")+ continue+ emitted += 1++ if not lines:+ return None++ code = "\n".join(lines) + "\n"+ return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes)+++_DEPENDENCY_ID = f"dependency:{_SYNTHESIZED_BLOCK_LABEL}_reached"+_OBSERVATION_REF_ID = f"observation:{_SYNTHESIZED_BLOCK_LABEL}_scout"++# Model-owned slots the synthesizer cannot prove; the model fills these.+_FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"+_FILL_CLAIM_ID = "claim:<fill>"+_FILL_CLAIM_TEXT = "<fill: the user-facing outcome this block claims>"+_FILL_CRITERION_ID = "criterion:<fill>"+_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"+++def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:+ """Fill only the scout-proven evidence shape (`observed_not_verified`); the terminal goal and+ outcomes stay as `<fill>` placeholders the model owns."""+ # First recorded source_url is the page the synthesized block's leading `page.goto` lands on.+ # The trajectory carries only pre-action source pages, not post-action reached URLs, so this is+ # an advisory entry-page hint, never a reached-page identity (which SPAs would mis-key anyway).+ entry_url_hint = next(+ (url for url in (str(interaction.get("source_url") or "").strip() for interaction in trajectory) if url),+ None,+ )++ page_dependency: dict[str, Any] = {+ "id": _DEPENDENCY_ID,+ "scope": "page",+ "status": "observed_not_verified",+ "observation_refs": [_OBSERVATION_REF_ID],+ }+ if entry_url_hint:+ page_dependency["url_hint"] = entry_url_hint++ observation_ref: dict[str, Any] = {+ "observation_ref": _OBSERVATION_REF_ID,+ "dependency_id": _DEPENDENCY_ID,+ "status": "observed_not_verified",+ "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,+ }++ return {+ "block_label": _SYNTHESIZED_BLOCK_LABEL,+ "declared_goal": _FILL_DECLARED_GOAL,+ "page_dependencies": [page_dependency],+ "observation_refs": [observation_ref],+ "claimed_outcomes": [+ {+ "id": _FILL_CLAIM_ID,+ "scope": "outcome",+ "text": _FILL_CLAIM_TEXT,+ "status": "observed_not_verified",+ "depends_on": [_DEPENDENCY_ID],+ "covered_criteria": [_FILL_CRITERION_ID],+ "observation_refs": [_OBSERVATION_REF_ID],+ }+ ],+ "completion_criteria": [+ {+ "id": _FILL_CRITERION_ID,+ "text": _FILL_CRITERION_TEXT,+ "level": "terminal",+ "terminal": True,+ }+ ],+ "terminal_verifier_expectations": [+ {+ "id": "expectation:<fill>",+ "text": "<fill: what terminal verification must observe>",+ "criteria_ids": [_FILL_CRITERION_ID],+ }+ ],+ }+++def _render_artifact_metadata_block(metadata: Mapping[str, Any]) -> str:+ return json.dumps(metadata, indent=2, sort_keys=True)+++def render_synthesized_offer_text(+ synthesized: SynthesizedCodeBlock,+ trajectory: Sequence[Mapping[str, Any]] | None = None,+) -> str:+ """Render the offer body the copilot sees for a synthesized block (pure)."""+ param_keys = [p.get("key", "") for p in synthesized.parameters if p.get("key")]+ parts = [+ "SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a "+ "deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; only hand-author the "+ "extract/report steps it does not cover.",+ "```python",+ synthesized.code.rstrip("\n"),+ "```",+ ]+ if param_keys:+ parts.append("Workflow parameters referenced (bind these): " + ", ".join(param_keys) + ".")+ if synthesized.notes:+ parts.append("Synthesis notes: " + "; ".join(synthesized.notes) + ".")+ if trajectory:+ metadata = build_synthesized_artifact_metadata(trajectory)+ parts.append(+ "Pass this `code_artifact_metadata` for the block (the scouted page evidence is filled in; "+ "replace the `<fill: ...>` slots with the terminal goal and outcome this block delivers, then "+ "submit it whole — the validator returns every remaining violation at once):"+ )+ parts.append("```json")+ parts.append(_render_artifact_metadata_block(metadata))+ parts.append("```")+ return "\n".join(parts)diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.pyindex b1f90a4d7..a91843324 100644--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py@@ -40,6 +40,66 @@ COMPOSITION_STRIPPED_HTML_EXPRESSION = ("})()")++# Given a CSS selector, return the element's ARIA role and accessible name so the code-block+# synthesizer has a get_by_role fallback anchor for a positional/unstable captured selector. The+# name is read only from true label sources, never the element's own textContent/value.+def scout_accessible_role_name_expression(css_selector: str) -> str:+ sel = json.dumps(css_selector)+ return (+ "(() => {"+ f" const el = document.querySelector({sel});"+ " if (!el) return null;"+ " const text = (v) => String(v == null ? '' : v).replace(/\\s+/g, ' ').trim();"+ " const implicitRole = (node) => {"+ " const tag = (node.tagName || '').toLowerCase();"+ " const type = (node.getAttribute('type') || '').toLowerCase();"+ " if (tag === 'a' && node.hasAttribute('href')) return 'link';"+ " if (tag === 'button') return 'button';"+ " if (tag === 'select') return 'combobox';"+ " if (tag === 'textarea') return 'textbox';"+ " if (tag === 'input') {"+ " if (['button', 'submit', 'reset'].includes(type)) return 'button';"+ " if (type === 'checkbox') return 'checkbox';"+ " if (type === 'radio') return 'radio';"+ " if (['text', 'search', 'email', 'tel', 'url', 'password', ''].includes(type)) return 'textbox';"+ " }"+ " if (/^h[1-6]$/.test(tag)) return 'heading';"+ " return '';"+ " };"+ " const accessibleName = (node) => {"+ " const aria = text(node.getAttribute('aria-label'));"+ " if (aria) return aria;"+ " const labelledby = node.getAttribute('aria-labelledby');"+ " if (labelledby) {"+ " const parts = labelledby.split(/\\s+/).map((id) => {"+ " const ref = document.getElementById(id);"+ " return ref ? text(ref.textContent) : '';"+ " }).filter(Boolean);"+ " if (parts.length) return text(parts.join(' '));"+ " }"+ " const id = node.getAttribute('id');"+ " if (id) {"+ " let lab = null;"+ " try { lab = document.querySelector('label[for=\"' + (window.CSS && CSS.escape ? CSS.escape(id) : id) + '\"]'); } catch (e) { lab = null; }"+ " if (lab) { const t = text(lab.textContent); if (t) return t; }"+ " }"+ " const parentLabel = node.closest ? node.closest('label') : null;"+ " if (parentLabel) { const t = text(parentLabel.textContent); if (t) return t; }"+ # textContent/value are never name sources: for a typed-into textbox/contenteditable+ # they would leak the raw typed value as accessible_name.+ " const title = text(node.getAttribute('title'));"+ " if (title) return title;"+ " const placeholder = text(node.getAttribute('placeholder'));"+ " if (placeholder) return placeholder;"+ " return '';"+ " };"+ " const role = text(el.getAttribute('role')) || implicitRole(el);"+ " return { role: role, accessible_name: accessibleName(el) };"+ "})()"+ )++COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION = ("(() => {"" const body = document.body; if (!body) return [];"diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.pyindex 244c35fbf..fd2557a6f 100644--- a/skyvern/forge/sdk/copilot/context.py+++ b/skyvern/forge/sdk/copilot/context.py@@ -152,9 +152,26 @@ class StructuredContext(BaseModel):self.urls_visited.append(UrlVisit(url=url, summary=""))elif tool == "list_credentials":- match = re.search(r"Found (\d+)", summary)- found = int(match.group(1)) > 0 if match else False- self.credentials_checked.append(CredentialCheck(credential_name=summary, found=found))+ resolved = entry.get("credentials")+ if isinstance(resolved, list) and resolved:+ for credential in resolved:+ if not isinstance(credential, dict):+ continue+ credential_id = credential.get("credential_id")+ if not isinstance(credential_id, str):+ continue+ name = credential.get("name")+ self.credentials_checked.append(+ CredentialCheck(+ credential_name=name if isinstance(name, str) else "",+ credential_id=credential_id,+ found=True,+ )+ )+ else:+ match = re.search(r"Found (\d+)", summary)+ found = int(match.group(1)) > 0 if match else False+ self.credentials_checked.append(CredentialCheck(credential_name=summary, found=found))elif tool == "type_text":parts = summary.split("into ")diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 17ca00a84..d5741b44a 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -15,6 +15,7 @@ from agents.run import Runnerfrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot import config as copilot_config_defaults+from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_blockfrom skyvern.forge.sdk.copilot.config import (DEFAULT_ENFORCEMENT_NUDGES,DEFAULT_TOKEN_BUDGET,@@ -37,7 +38,9 @@ from skyvern.forge.sdk.copilot.config import (POST_SUSPICIOUS_SUCCESS_NUDGE,POST_UPDATE_NUDGE,SCREENSHOT_DROPPED_NUDGE,+ BlockAuthoringPolicy,CopilotConfig,+ normalize_block_authoring_policy,)from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextActionfrom skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason@@ -1446,6 +1449,52 @@ async def _run_streamed_with_deadline(return result+def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:+ """Post-turn fallback offer of a deterministically synthesized code block, in code-only mode.++ Returns a single user message wrapping the synthesized Playwright block, or+ None when the policy/latch/empty-trajectory guards do not hold. Shares the+ one-shot latch with the pre-authoring prompt-side offer, so whichever fires+ first wins and the model sees the offer at most once.+ """+ if getattr(ctx, "synthesized_block_offered", False):+ return None+ if getattr(ctx, "update_workflow_called", False):+ return None+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (+ BlockAuthoringPolicy.CODE_ONLY_BROWSER+ ):+ return None+ trajectory = getattr(ctx, "scout_trajectory", None) or []+ if not trajectory:+ return None+ synthesized = synthesize_code_block(trajectory)+ if synthesized is None:+ return None++ ctx.synthesized_block_offered = True+ return {"role": "user", "content": render_synthesized_offer_text(synthesized, trajectory)}+++def _assemble_enforcement_messages(+ screenshot_msg: dict[str, Any] | None,+ nudge_content: str | None,+ synthesized_msg: dict[str, Any] | None,+) -> list[Any]:+ """Build the extra messages for an enforcement retry, ordered so a nudge, when present, stays last.++ The screenshot rides as its own user-role message because OpenAI rejects image parts inside a tool message.+ """+ extra_msgs: list[Any] = []+ if screenshot_msg is not None:+ extra_msgs.append(screenshot_msg)+ if nudge_content is not None:+ extra_msgs.append({"role": "user", "content": NUDGE_SENTINEL + nudge_content})+ if synthesized_msg is not None:+ extra_msgs.insert(0, synthesized_msg)+ return extra_msgs++async def run_with_enforcement(agent: Agent,initial_input: str | list,@@ -1573,12 +1622,19 @@ async def run_with_enforcement(pending_recovery_nudge = Noneelse:nudge = _check_enforcement(ctx, result, copilot_config)- if nudge is None:++ # The offer is independent of the nudge: a clean scout-then-author turn+ # finalizes with nudge=None, so injecting it only inside the nudge branch+ # would never reach the model. Compute it once here so it rides both the+ # nudge path and the finalize path.+ synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)++ if nudge is None and synthesized_msg is None:_consume_pending_screenshots(ctx)_maybe_raise_non_retriable_nav(ctx)return result- if nudge == _nudge(copilot_config, "post_update"):+ if nudge is not None and nudge == _nudge(copilot_config, "post_update"):if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:LOG.warning("Enforcement exhausted post-update nudges, allowing response",@@ -1589,7 +1645,7 @@ async def run_with_enforcement(return resultctx.post_update_nudge_count += 1- nudge_type = _nudge_type_for_log(nudge, copilot_config)+ nudge_type = _nudge_type_for_log(nudge, copilot_config) if nudge is not None else "synthesized_block_offer"LOG.info("Enforcement nudge", nudge_type=nudge_type, iteration=iteration)# OpenAI rejects images in tool messages, so a queued post-run@@ -1599,8 +1655,7 @@ async def run_with_enforcement(LOG.info("Injecting screenshot user message", count=len(screenshot_msg["content"]) - 1)with copilot_span("enforcement_nudge", data={"nudge_type": nudge_type, "iteration": iteration}):- nudge_msg = {"role": "user", "content": NUDGE_SENTINEL + nudge}- extra_msgs = [nudge_msg] if screenshot_msg is None else [screenshot_msg, nudge_msg]+ extra_msgs = _assemble_enforcement_messages(screenshot_msg, nudge, synthesized_msg)current_input = (extra_msgs if session is not None else _prune_input_list(result.to_input_list()) + extra_msgs)diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.pyindex fc01f0f71..fcc6d90f1 100644--- a/skyvern/forge/sdk/copilot/hooks.py+++ b/skyvern/forge/sdk/copilot/hooks.py@@ -88,6 +88,17 @@ class CopilotRunHooks(RunHooksBase):total_tokens=getattr(self._ctx, "total_tokens_used", None),)+ if tool_name == "list_credentials" and parsed.get("ok"):+ data = parsed.get("data") or {}+ listed = data.get("credentials", []) if isinstance(data, dict) else []+ resolved = [+ {"credential_id": c.get("credential_id"), "name": c.get("name")}+ for c in listed+ if isinstance(c, dict) and isinstance(c.get("credential_id"), str)+ ]+ if resolved:+ activity_entry["credentials"] = resolved+if tool_name in _BLOCK_OUTPUT_TOOLS and parsed.get("ok"):data = parsed.get("data") or {}blocks = data.get("blocks", []) if isinstance(data, dict) else []diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.pyindex 46f134469..707b2b091 100644--- a/skyvern/forge/sdk/copilot/output_policy.py+++ b/skyvern/forge/sdk/copilot/output_policy.py@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (parse_workflow_yaml,url_origin,workflow_blocks,+ workflow_credential_ids_from_parsed,workflow_credential_origins_from_parsed,)@@ -481,7 +482,10 @@ def _apply_credential_policy(approved_ids = _approved_credential_ids(request_policy)allowed_unresolved_ids = _allowed_unresolved_credential_ids(request_policy)- allowed_ids = approved_ids | allowed_unresolved_ids | _existing_workflow_credential_ids(request_policy)+ bound_approved_ids = _bound_approved_credential_ids(request_policy, workflow_yaml)+ allowed_ids = (+ approved_ids | allowed_unresolved_ids | bound_approved_ids | _existing_workflow_credential_ids(request_policy)+ )if any(credential_id not in allowed_ids for credential_id in found_ids):verdict.add(OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE)@@ -503,6 +507,19 @@ def _approved_credential_ids(request_policy: RequestPolicy) -> set[str]:}+def _bound_approved_credential_ids(request_policy: RequestPolicy, workflow_yaml: str | None) -> set[str]:+ # Requiring binding keeps a discovered ID leaked into a non-credential field+ # caught by misbinding rather than approved here.+ discovered = {credential.credential_id for credential in request_policy.discovered_credentials}+ discovered = {cid for cid in discovered if cid.startswith("cred_")}+ if not discovered or not workflow_yaml:+ return set()+ parsed = parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, dict):+ return set()+ return discovered & workflow_credential_ids_from_parsed(parsed)++def _allowed_unresolved_credential_ids(request_policy: RequestPolicy) -> set[str]:if not request_policy.allow_missing_credentials_in_draft:return set()@@ -583,7 +600,7 @@ def _workflow_broadens_credential_scope(parsed_workflow: dict[str, Any], requestdef _approved_origins_by_id(request_policy: RequestPolicy) -> dict[str, set[str]]:origins: dict[str, set[str]] = {}- for credential in request_policy.resolved_credentials:+ for credential in [*request_policy.resolved_credentials, *request_policy.discovered_credentials]:credential_id = getattr(credential, "credential_id", None)tested_url = getattr(credential, "tested_url", None)if isinstance(credential_id, str) and isinstance(tested_url, str):diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.pyindex 86dac24a3..aa77d5e13 100644--- a/skyvern/forge/sdk/copilot/request_policy.py+++ b/skyvern/forge/sdk/copilot/request_policy.py@@ -144,6 +144,9 @@ class RequestPolicy:completion_contract: str | None = Nonecompletion_criteria: list[CompletionCriterion] = field(default_factory=list)resolved_credentials: list[Credential] = field(default_factory=list)+ # Approves persisting a bound credential, not running it: run authority stays+ # scoped to resolved_credentials (ADR 0002).+ discovered_credentials: list[Credential] = field(default_factory=list)invalid_credential_ids: list[str] = field(default_factory=list)clarification_question: str | None = Noneraw_secret_detected: bool = False@@ -777,6 +780,30 @@ def _previous_credential_clarification_was_asked(global_llm_context: str) -> boo)+def _discovered_credential_ids_from_context(global_llm_context: str) -> set[str]:+ structured = StructuredContext.from_json_str(global_llm_context)+ return {+ check.credential_id+ for check in structured.credentials_checked+ if check.found and isinstance(check.credential_id, str) and check.credential_id.startswith("cred_")+ }+++async def _seed_discovered_credentials(+ policy: RequestPolicy,+ *,+ organization_id: str,+ global_llm_context: str,+) -> None:+ discovered_ids = _discovered_credential_ids_from_context(global_llm_context)+ if not discovered_ids:+ return+ policy.discovered_credentials = await app.DATABASE.credentials.get_credentials_by_ids(+ sorted(discovered_ids),+ organization_id=organization_id,+ )++def _last_assistant_message_was_saved_credential_question(chat_history: list[WorkflowCopilotChatHistoryMessage],) -> bool:@@ -1035,6 +1062,18 @@ async def build_request_policy(policy.existing_workflow_credential_origins = {credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()}+ try:+ await _seed_discovered_credentials(+ policy,+ organization_id=organization_id,+ global_llm_context=global_llm_context,+ )+ except Exception:+ LOG.warning(+ "request-policy discovered-credential seeding failed",+ organization_id=organization_id,+ exc_info=True,+ )_prioritize_credential_clarification(policy)if policy.raw_secret_detected and policy.raw_secret_handling == "redacted_draft":diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.pyindex 517904895..5be680246 100644--- a/skyvern/forge/sdk/copilot/runtime.py+++ b/skyvern/forge/sdk/copilot/runtime.py@@ -113,6 +113,9 @@ class ScoutedInteraction(TypedDict):value: NotRequired[str]key: NotRequired[str]typed_length: NotRequired[int]+ role: NotRequired[str]+ accessible_name: NotRequired[str]+ trajectory_index: NotRequired[int]@dataclass@@ -228,6 +231,12 @@ class AgentContext:observed_browser_urls: list[str] = field(default_factory=list)# Ephemeral within-turn scout captures; not persisted across turns.scouted_interactions: list[ScoutedInteraction] = field(default_factory=list)+ # Append-only, non-deduped record of the scout's interaction sequence in+ # acted order. Unlike scouted_interactions (deduped for auto-credit), this+ # preserves repeats and ordering so code_block_synthesis can emit a faithful+ # linear Playwright trajectory.+ scout_trajectory: list[ScoutedInteraction] = field(default_factory=list)+ synthesized_block_offered: bool = False# Source page of an in-flight scout action, captured before it may navigate away.pending_scout_source_url: str | None = Nonediff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.pyindex e3a05b605..14aef65fb 100644--- a/skyvern/forge/sdk/copilot/tools.py+++ b/skyvern/forge/sdk/copilot/tools.py@@ -11,7 +11,7 @@ from collections import Counter, defaultdictfrom collections.abc import Mappingfrom dataclasses import dataclassfrom datetime import datetime-from typing import Annotated, Any, Literal+from typing import Annotated, Any, Literal, castfrom urllib.parse import urljoin, urlparseimport structlog@@ -70,6 +70,9 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (from skyvern.forge.sdk.copilot.composition_browser_expressions import (COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,)+from skyvern.forge.sdk.copilot.composition_browser_expressions import (+ scout_accessible_role_name_expression as _scout_accessible_role_name_expression,+)from skyvern.forge.sdk.copilot.composition_evidence import (SCOUT_INTERACTION_EVIDENCE_TOOL,composition_page_evidence_error,@@ -144,6 +147,7 @@ from skyvern.forge.sdk.copilot.turn_intent import ()from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidencefrom skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml+from skyvern.forge.sdk.schemas.credentials import Credentialfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPExceptionfrom skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocks@@ -1530,6 +1534,83 @@ def _consume_scout_source_url(ctx: AgentContext) -> str | None:return source_url+_ROLE_NAME_SELECTOR_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')+++def _role_name_from_selector(selector: str) -> tuple[str, str] | None:+ """Parse the ``role=<role>[name="<name>"]`` form (ref_to_selector) — TIER 1, no browser read.++ Returns (role, accessible_name) when the selector is a plain role/name locator;+ None for bare CSS/xpath or when an engine chain (`>> nth=`) trails the role/name.+ """+ selector = selector.strip()+ match = _ROLE_NAME_SELECTOR_RE.match(selector)+ if not match:+ return None+ role, raw_name, suffix = match.group(1), match.group(2), match.group(3)+ if suffix.strip():+ return None+ name = raw_name.replace('\\"', '"') if raw_name is not None else ""+ return role, name+++async def _capture_accessible_role_name(ctx: AgentContext, selector: str) -> tuple[str, str] | None:+ """TIER 2: read the element's role/accessible name for a bare CSS/xpath selector.++ A failed read degrades gracefully to None so the selector-only auto-credit+ path (SKY-10712) stays intact.+ """+ selector = selector.strip()+ if not selector:+ return None+ server = getattr(ctx, "discovery_mcp_server", None)+ if server is None:+ return None+ try:+ result = await asyncio.wait_for(+ server.call_internal_tool(+ "skyvern_evaluate",+ {"expression": _scout_accessible_role_name_expression(selector)},+ ),+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,+ )+ except Exception:+ return None+ if not isinstance(result, dict) or not result.get("ok"):+ return None+ value = (result.get("data") or {}).get("result")+ if not isinstance(value, dict):+ return None+ role = str(value.get("role") or "").strip()+ name = str(value.get("accessible_name") or "").strip()+ if not role and not name:+ return None+ return role, name+++async def _resolve_scout_role_name(+ ctx: AgentContext, selector: str, *, allow_browser_read: bool = True+) -> tuple[str, str]:+ """Resolve (role, accessible_name) for a scouted selector. TIER 1 parse first;+ TIER 2 browser read only for bare CSS/xpath. Always degrades to ("", "").++ ``allow_browser_read=False`` skips TIER 2 when the action navigated: a post-action+ read against the landing page would capture the wrong element's name, so the bare+ selector is kept verbatim (the synthesizer prefers it anyway)."""+ selector = selector.strip()+ if not selector:+ return "", ""+ parsed = _role_name_from_selector(selector)+ if parsed is not None:+ return parsed+ if not allow_browser_read:+ return "", ""+ captured = await _capture_accessible_role_name(ctx, selector)+ if captured is not None:+ return captured+ return "", ""++def _record_scouted_interaction(ctx: AgentContext,*,@@ -1539,6 +1620,8 @@ def _record_scouted_interaction(value: str = "",key: str = "",typed_length: int = 0,+ role: str = "",+ accessible_name: str = "",) -> None:selector = selector.strip()# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.@@ -1555,6 +1638,10 @@ def _record_scouted_interaction(artifact["key"] = keyif typed_length:artifact["typed_length"] = typed_length+ if role:+ artifact["role"] = role+ if accessible_name:+ artifact["accessible_name"] = accessible_nameinteractions = [itemfor item in ctx.scouted_interactions@@ -1566,12 +1653,21 @@ def _record_scouted_interaction(]interactions.append(artifact)ctx.scouted_interactions = interactions[-_MAX_SCOUTED_INTERACTIONS:]++ trajectory = list(ctx.scout_trajectory)+ trajectory_artifact = cast(ScoutedInteraction, artifact.copy())+ trajectory_artifact["trajectory_index"] = len(trajectory)+ trajectory.append(trajectory_artifact)+ ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]+LOG.info("copilot_scout_interaction_captured",tool_name=tool_name,selector=selector or None,source_url=artifact.get("source_url"),+ role=role or None,total_scouted_interactions=len(ctx.scouted_interactions),+ total_scout_trajectory=len(ctx.scout_trajectory),)@@ -2420,6 +2516,15 @@ def _code_artifact_metadata_as_tool_argument(return [item.model_dump(mode="json", exclude_none=True) for item in metadata]+def _format_code_artifact_violations(violations: list[str]) -> str:+ # Surface every contract violation at once so the agent fixes them in a single+ # update instead of round-tripping one error per `update_and_run_blocks` call.+ if len(violations) == 1:+ return violations[0]+ numbered = "\n".join(f"{index}. {message}" for index, message in enumerate(violations, start=1))+ return f"Artifact metadata has {len(violations)} contract violations; fix all of them in one update:\n{numbered}"++def _normalize_code_artifact_metadata(raw_metadata: Any,workflow_yaml: str,@@ -2429,6 +2534,8 @@ def _normalize_code_artifact_metadata(items = _code_artifact_metadata_items(raw_metadata)code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)normalized: dict[str, dict[str, Any]] = {}+ violations: list[str] = []+ seen_labels: set[str] = set()for raw_item in items:try:metadata = (@@ -2437,22 +2544,30 @@ def _normalize_code_artifact_metadata(else CodeArtifactMetadata.model_validate(raw_item))except ValidationError as exc:- return {}, f"Artifact metadata is malformed: {exc}"+ violations.append(f"Artifact metadata is malformed: {exc}")+ continuedumped = metadata.model_dump(mode="json", exclude_none=True)label = str(dumped.get("block_label") or "").strip()if not label:- return {}, "Artifact metadata requires a non-empty `block_label` for each artifact."+ violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")+ continueif label not in code_blocks:- return {}, (+ violations.append(f"Artifact metadata for `{label}` does not reference an existing code block label. ""Attach metadata only to authored code blocks.")- if label in normalized:- return {}, f"Artifact metadata contains duplicate entries for `{label}`."+ continue+ if label in seen_labels:+ violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")+ continue+ seen_labels.add(label)dumped["block_label"] = label+ item_violations: list[str] = []artifact_id = str(dumped.get("artifact_id") or "").strip()if artifact_id and not artifact_id.startswith("code_artifact:"):- return {}, f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."+ item_violations.append(+ f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."+ )dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)block_id = str(dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or ""@@ -2461,17 +2576,20 @@ def _normalize_code_artifact_metadata(dumped["block_id"] = block_iddeclared_goal = str(dumped.get("declared_goal") or "").strip()if not declared_goal:- return {}, f"Artifact metadata for `{label}` requires a non-empty `declared_goal`."+ item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:value = dumped.get(field_name)if not isinstance(value, list) or not value:- return {}, f"Artifact metadata for `{label}` requires non-empty `{field_name}`."+ item_violations.append(f"Artifact metadata for `{label}` requires non-empty `{field_name}`.")if not dumped.get("evidence_refs") and not dumped.get("observation_refs"):- return {}, f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`."- validation_error = _code_artifact_metadata_shape_error(label, dumped)- if validation_error is not None:- return {}, validation_error+ item_violations.append(f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`.")+ item_violations.extend(_code_artifact_metadata_shape_errors(label, dumped))+ if item_violations:+ violations.extend(item_violations)+ continuenormalized[label] = dumped+ if violations:+ return {}, _format_code_artifact_violations(violations)return normalized, None@@ -2502,45 +2620,47 @@ def _artifact_id_for_block_label(label: str) -> str:return f"code_artifact:{fragment}"-def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any]) -> str | None:+def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:+ """Return every shape violation for one artifact; the caller aggregates them."""+ errors: list[str] = []for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):for index, ref in enumerate(_artifact_rows(artifact.get(field_name))):if not str(ref.get(ref_key) or "").strip():- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`."+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`.")if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id."+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id.")status = str(ref.get("status") or "").strip()if ref.get("checkpoint_next_mode") == "advance" and status != "diagnostic_only":- return (+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} has ""`checkpoint_next_mode=advance`; it must stay `diagnostic_only`.")if ref.get("checkpoint_next_mode") == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:- return (+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} has ""`checkpoint_next_mode=stop`; it must remain `observed_not_verified` or `diagnostic_only`.")if status != "missing" and not str(ref.get("source_tool") or "").strip():- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`."+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`.")for index, claim in enumerate(_artifact_rows(artifact.get("claimed_outcomes"))):claim_id = str(claim.get("id") or "").strip()if not _artifact_string_list(claim.get("depends_on")):- return f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`."+ errors.append(f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`.")claim_criteria = _artifact_string_list(claim.get("covered_criteria")) or _artifact_string_list(claim.get("criteria_ids"))if not claim_criteria:- return f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids."+ errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids.")claim_evidence_refs = _artifact_string_list(claim.get("evidence_refs"))claim_observation_refs = _artifact_string_list(claim.get("observation_refs"))if claim.get("status") == "satisfied" and not claim_evidence_refs:- return (+ errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` is `satisfied` but has no ""claim-scoped `evidence_refs`.")if claim.get("status") != "missing" and not claim_evidence_refs and not claim_observation_refs:- return (+ errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` requires claim-scoped ""`evidence_refs` or `observation_refs` unless status is `missing`.")@@ -2550,12 +2670,12 @@ def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any])dependency_evidence_refs = _artifact_string_list(dependency.get("evidence_refs"))dependency_observation_refs = _artifact_string_list(dependency.get("observation_refs"))if dependency.get("status") == "satisfied" and not dependency_evidence_refs:- return (+ errors.append(f"Artifact metadata dependency `{dependency_id}` for `{label}` is `satisfied` but has no ""dependency-scoped `evidence_refs`.")if dependency.get("status") != "missing" and not dependency_evidence_refs and not dependency_observation_refs:- return (+ errors.append(f"Artifact metadata dependency `{dependency_id}` for `{label}` requires scoped ""`evidence_refs` or `observation_refs` unless status is `missing`.")@@ -2565,23 +2685,23 @@ def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any])if not _artifact_string_list(expectation.get("criteria_ids")) and not _artifact_string_list(expectation.get("claimed_outcome_ids")):- return (+ errors.append(f"Artifact metadata terminal verifier expectation `{expectation_id or index}` for `{label}` ""requires `criteria_ids` or `claimed_outcome_ids`.")for index, observation in enumerate(_artifact_rows(artifact.get("exploration_observations"))):if observation.get("status") != "observed_not_verified":- return (+ errors.append(f"Artifact metadata for `{label}` exploration observation {index} must be marked ""`observed_not_verified` until authored execution and terminal verification pass.")if observation.get("checkpoint_next_mode") == "advance":- return (+ errors.append(f"Artifact metadata for `{label}` exploration observation {index} cannot carry ""`checkpoint_next_mode=advance`; record that as `diagnostic_only` evidence instead.")- return None+ return errorsdef _artifact_rows(value: Any) -> list[Mapping[str, Any]]:@@ -2788,6 +2908,7 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[selif cred.secret_label:entry["secret_label"] = cred.secret_labelserialized.append(entry)+ _record_discovered_credentials_on_policy(ctx, credentials)return {"ok": True,"data": {@@ -2800,6 +2921,16 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[s}+def _record_discovered_credentials_on_policy(ctx: AgentContext, credentials: list[Credential]) -> None:+ request_policy = ctx.request_policy+ if not isinstance(request_policy, RequestPolicy):+ return+ known_ids = {credential.credential_id for credential in request_policy.discovered_credentials}+ for credential in credentials:+ if credential.credential_id not in known_ids:+ request_policy.discovered_credentials.append(credential)++# Block types that establish browser state (loaded page / authenticated# session / navigation target). These are valid upstream anchors to walk back# to when a downstream edit invalidates part of the chain.@@ -5930,7 +6061,18 @@ async def _click_post_hook("url": url,"title": title,}- _record_scouted_interaction(ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url)+ navigated = bool(source_url) and bool(url) and source_url != url+ role, accessible_name = await _resolve_scout_role_name(+ ctx, data.get("selector", ""), allow_browser_read=not navigated+ )+ _record_scouted_interaction(+ ctx,+ tool_name="click",+ selector=data.get("selector", ""),+ source_url=source_url,+ role=role,+ accessible_name=accessible_name,+ )observation_step = _register_scout_interaction_observation(ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url, url=url)@@ -6020,8 +6162,15 @@ async def _type_text_post_hook(if landing_failure is not None:return landing_failure_mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)+ role, accessible_name = await _resolve_scout_role_name(ctx, selector)_record_scouted_interaction(- ctx, tool_name="type_text", selector=selector, source_url=source_url, typed_length=typed_length+ ctx,+ tool_name="type_text",+ selector=selector,+ source_url=source_url,+ typed_length=typed_length,+ role=role,+ accessible_name=accessible_name,)observation_step = _register_scout_interaction_observation(ctx, tool_name="type_text", selector=selector, source_url=source_url, url=url@@ -6105,12 +6254,15 @@ async def _select_option_post_hook("value": data.get("value", ""),"url": url,}+ role, accessible_name = await _resolve_scout_role_name(ctx, data.get("selector", ""))_record_scouted_interaction(ctx,tool_name="select_option",selector=data.get("selector", ""),source_url=source_url,value=data.get("value", ""),+ role=role,+ accessible_name=accessible_name,)observation_step = _register_scout_interaction_observation(ctx, tool_name="select_option", selector=data.get("selector", ""), source_url=source_url, url=urldiff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.pynew file mode 100644index 000000000..10892f6f8--- /dev/null+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py@@ -0,0 +1,165 @@+"""Tests for the code-artifact-metadata validator returning every violation at once.++OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.+"""++from __future__ import annotations++import re+import textwrap++from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata+++def _code_block_yaml(label: str) -> str:+ return textwrap.dedent(+ f"""+ workflow_definition:+ blocks:+ - block_type: code+ label: {label}+ code: |+ await page.goto("https://example.com/")+ """+ ).strip()+++def _violation_count(error: str) -> int:+ return len(re.findall(r"^\d+\.", error, flags=re.M))+++def _broken_metadata(label: str) -> dict:+ return {+ "block_label": label,+ "artifact_id": "not-prefixed",+ "declared_goal": "do the thing",+ "claimed_outcomes": [{"id": "claim:x", "scope": "outcome", "text": "x", "status": "satisfied"}],+ "page_dependencies": [{"id": "dependency:p", "scope": "page", "status": "satisfied"}],+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],+ "terminal_verifier_expectations": [{"id": "exp", "text": "e"}],+ "observation_refs": [{"observation_ref": "obs1", "status": "satisfied", "checkpoint_next_mode": "advance"}],+ }+++class TestAccumulateAllViolations:+ def test_every_violation_returned_at_once(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_broken_metadata("my_block")], _code_block_yaml("my_block")+ )+ assert normalized == {}+ assert error is not None+ # The 5+ sequential failures from the repair loop now surface together.+ assert _violation_count(error) >= 5+ assert error.startswith("Artifact metadata has ")+ assert "fix all of them in one update" in error+ assert "`artifact_id` to start with `code_artifact:`" in error+ assert "requires `source_tool`" in error+ assert "requires `depends_on`" in error+ assert "is `satisfied` but has no" in error++ def test_single_violation_is_not_numbered(self) -> None:+ metadata = {+ "block_label": "my_block",+ "artifact_id": "not-prefixed",+ "declared_goal": "g",+ "claimed_outcomes": [+ {+ "id": "claim:x",+ "scope": "outcome",+ "text": "x",+ "status": "observed_not_verified",+ "depends_on": ["dependency:p"],+ "covered_criteria": ["criterion:c"],+ "observation_refs": ["obs1"],+ }+ ],+ "page_dependencies": [+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}+ ],+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],+ "observation_refs": [+ {+ "observation_ref": "obs1",+ "dependency_id": "dependency:p",+ "status": "observed_not_verified",+ "source_tool": "scout_interaction",+ }+ ],+ }+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))+ assert normalized == {}+ assert error is not None+ assert _violation_count(error) == 0+ assert error == "Artifact metadata for `my_block` requires `artifact_id` to start with `code_artifact:`."++ def test_violations_aggregate_across_multiple_artifacts(self) -> None:+ yaml = textwrap.dedent(+ """+ workflow_definition:+ blocks:+ - block_type: code+ label: block_one+ code: |+ await page.goto("https://example.com/")+ - block_type: code+ label: block_two+ code: |+ await page.goto("https://example.com/")+ """+ ).strip()+ normalized, error = _normalize_code_artifact_metadata(+ [_broken_metadata("block_one"), _broken_metadata("block_two")], yaml+ )+ assert normalized == {}+ assert error is not None+ assert "block_one" in error+ assert "block_two" in error++ def test_unknown_label_does_not_short_circuit_other_artifacts(self) -> None:+ normalized, error = _normalize_code_artifact_metadata(+ [_broken_metadata("ghost_label"), _broken_metadata("my_block")], _code_block_yaml("my_block")+ )+ assert normalized == {}+ assert error is not None+ assert "does not reference an existing code block label" in error+ # The second artifact's shape violations are still surfaced.+ assert "requires `source_tool`" in error++ def test_valid_metadata_passes(self) -> None:+ metadata = {+ "block_label": "my_block",+ "artifact_id": "code_artifact:my_block",+ "declared_goal": "g",+ "claimed_outcomes": [+ {+ "id": "claim:x",+ "scope": "outcome",+ "text": "x",+ "status": "observed_not_verified",+ "depends_on": ["dependency:p"],+ "covered_criteria": ["criterion:c"],+ "observation_refs": ["obs1"],+ }+ ],+ "page_dependencies": [+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}+ ],+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],+ "observation_refs": [+ {+ "observation_ref": "obs1",+ "dependency_id": "dependency:p",+ "status": "observed_not_verified",+ "source_tool": "scout_interaction",+ }+ ],+ }+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))+ assert error is None+ assert list(normalized.keys()) == ["my_block"]++ def test_empty_metadata_is_noop(self) -> None:+ assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)+ assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pynew file mode 100644index 000000000..bfecc8e63--- /dev/null+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -0,0 +1,647 @@+"""Tests for the pure copilot code-block synthesizer.++OSS-synced: only example.* / RFC-2606 placeholder targets.+"""++from __future__ import annotations++import ast+import keyword+from typing import Any++from skyvern.forge.sdk.copilot.code_block_preflight import preflight_code_block+from skyvern.forge.sdk.copilot.code_block_synthesis import (+ _SYNTHESIZED_BLOCK_LABEL,+ build_synthesized_artifact_metadata,+ render_synthesized_offer_text,+ synthesize_code_block,+)+from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata+++def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:+ return {"tool_name": tool_name, **fields}+++class TestLocatorSynthesis:+ def test_role_selector_emits_get_by_role(self) -> None:+ # A `role=...` selector is an ARIA anchor (ref_to_selector form), not a native CSS path.+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='role=button[name="Add to cart"]',+ source_url="https://example.com/product",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("button", name="Add to cart").click()' in result.code+ assert "get_by_role" in result.code++ def test_stable_id_selector_is_emitted_verbatim_not_get_by_role(self) -> None:+ # Selector-first: a captured stable selector (id) wins over a get_by_role anchor, because the+ # scout's a11y-name read may not reproduce on the raw page the code block runs against.+ result = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector="#search-box",+ source_url="https://example.com/",+ typed_length=11,+ role="textbox",+ accessible_name="Search",+ )+ ]+ )+ assert result is not None+ assert 'await page.locator("#search-box").fill(str(search))' in result.code+ assert "get_by_role" not in result.code++ def test_stable_attribute_selector_is_emitted_verbatim(self) -> None:+ # [name=...] / [data-testid=...] etc. are stable identity anchors — kept verbatim.+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='[data-testid="add-to-cart"]',+ source_url="https://example.com/product",+ role="button",+ accessible_name="Add to cart",+ )+ ]+ )+ assert result is not None+ assert 'await page.locator("[data-testid=\\"add-to-cart\\"]").click()' in result.code+ assert "get_by_role" not in result.code++ def test_positional_nth_of_type_selector_uses_get_by_role_fallback(self) -> None:+ # A positional CSS path is fragile; when a role/name anchor exists, prefer it.+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector="div.list > button:nth-of-type(3)",+ source_url="https://example.com/list",+ role="button",+ accessible_name="More",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("button", name="More").click()' in result.code++ def test_nth_engine_chain_uses_get_by_role_anchor(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='role=button[name="More"] >> nth=2',+ source_url="https://example.com/list",+ role="button",+ accessible_name="More",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("button", name="More").click()' in result.code++ def test_positional_selector_without_role_name_keeps_selector_with_note(self) -> None:+ result = synthesize_code_block(+ [_interaction("click", selector="ul > li:nth-child(2)", source_url="https://example.com/results")]+ )+ assert result is not None+ assert 'await page.locator("ul > li:nth-child(2)").click()' in result.code+ assert any("low-confidence" in note for note in result.notes)++ def test_stable_bare_css_without_role_name_is_emitted_verbatim_no_note(self) -> None:+ result = synthesize_code_block(+ [_interaction("click", selector=".results .item", source_url="https://example.com/results")]+ )+ assert result is not None+ assert 'await page.locator(".results .item").click()' in result.code+ assert not any("low-confidence" in note for note in result.notes)++ def test_never_emits_first_or_last_callables(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='role=link[name="Next"] >> nth=0',+ source_url="https://example.com/",+ )+ ]+ )+ assert result is not None+ assert ".first" not in result.code+ assert ".last" not in result.code+++class TestActionSynthesis:+ def test_type_text_becomes_param_slot_fill(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector='role=textbox[name="Search"]',+ source_url="https://example.com/",+ typed_length=11,+ role="textbox",+ accessible_name="Search",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("textbox", name="Search").fill(str(search))' in result.code+ assert result.parameters == [{"key": "search"}]+ # Raw typed value is never captured.+ assert "value" not in result.code++ def test_goto_entry_url_and_load_state(self) -> None:+ result = synthesize_code_block([_interaction("click", selector="#go", source_url="https://example.com/start")])+ assert result is not None+ lines = result.code.splitlines()+ assert lines[0] == ' await page.goto("https://example.com/start")'+ assert lines[1] == ' await page.wait_for_load_state("load")'++ def test_press_enter_uses_keyboard_when_no_selector(self) -> None:+ result = synthesize_code_block([_interaction("press_key", key="Enter")])+ assert result is not None+ assert 'await page.keyboard.press("Enter")' in result.code++ def test_press_key_on_located_element(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "press_key",+ selector='role=textbox[name="Search"]',+ key="Enter",+ role="textbox",+ accessible_name="Search",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("textbox", name="Search").press("Enter")' in result.code++ def test_select_option_emits_value(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "select_option",+ selector='role=combobox[name="Size"]',+ source_url="https://example.com/",+ value="large",+ role="combobox",+ accessible_name="Size",+ )+ ]+ )+ assert result is not None+ assert 'await page.get_by_role("combobox", name="Size").select_option("large")' in result.code++ def test_select_option_without_value_is_dropped_with_note(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "select_option",+ selector='role=combobox[name="Size"]',+ source_url="https://example.com/",+ role="combobox",+ accessible_name="Size",+ )+ ]+ )+ assert result is None or "select_option" not in result.code+ if result is not None:+ assert any("select_option" in note for note in result.notes)+++class TestParamKeySafety:+ @staticmethod+ def _emitted_wrapper(code: str, param_keys: list[str]) -> str:+ # Mirror block.py generate_async_user_function: the param keys become the wrapper signature.+ signature = ", ".join(f"{key}=None" for key in param_keys)+ body = "\n".join(f" {line.strip()}" for line in code.splitlines())+ return f"async def wrapper({signature}):\n{body or ' pass'}"++ def test_keyword_accessible_name_yields_bindable_identifier(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector='role=textbox[name="Class"]',+ source_url="https://example.com/",+ typed_length=4,+ role="textbox",+ accessible_name="Class",+ )+ ]+ )+ assert result is not None+ keys = [p["key"] for p in result.parameters]+ assert keys == ["class_field"]+ assert all(key.isidentifier() and not keyword.iskeyword(key) for key in keys)+ ast.parse(self._emitted_wrapper(result.code, keys))++ def test_reserved_safe_var_name_is_suffixed(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector='role=textbox[name="Page"]',+ source_url="https://example.com/",+ typed_length=4,+ role="textbox",+ accessible_name="Page",+ )+ ]+ )+ assert result is not None+ assert result.parameters == [{"key": "page_field"}]+ assert "fill(str(page))" not in result.code+ assert "fill(str(page_field))" in result.code++ def test_leading_digit_name_is_valid_identifier(self) -> None:+ result = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector='role=textbox[name="2nd line"]',+ source_url="https://example.com/",+ typed_length=4,+ role="textbox",+ accessible_name="2nd line",+ )+ ]+ )+ assert result is not None+ keys = [p["key"] for p in result.parameters]+ assert keys and all(key.isidentifier() for key in keys)+ ast.parse(self._emitted_wrapper(result.code, keys))+++class TestTrajectoryFidelity:+ def test_two_same_selector_clicks_both_emitted(self) -> None:+ # Regression: scout_trajectory is append-only/non-deduped, so a repeated+ # click on the same selector must produce two clicks (the deduped list would drop one).+ trajectory = [+ _interaction(+ "click",+ selector='role=button[name="Add to cart"]',+ source_url="https://example.com/p",+ role="button",+ accessible_name="Add to cart",+ ),+ _interaction(+ "click",+ selector='role=button[name="Add to cart"]',+ source_url="https://example.com/p",+ role="button",+ accessible_name="Add to cart",+ ),+ ]+ result = synthesize_code_block(trajectory)+ assert result is not None+ assert result.code.count('await page.get_by_role("button", name="Add to cart").click()') == 2++ def test_same_name_disambiguation_keeps_distinct_param_keys(self) -> None:+ trajectory = [+ _interaction(+ "type_text",+ selector='role=textbox[name="Name"]',+ source_url="https://example.com/",+ typed_length=3,+ role="textbox",+ accessible_name="Name",+ ),+ _interaction(+ "type_text",+ selector='role=textbox[name="Name"]',+ source_url="https://example.com/",+ typed_length=3,+ role="textbox",+ accessible_name="Name",+ ),+ ]+ result = synthesize_code_block(trajectory)+ assert result is not None+ assert result.parameters == [{"key": "name"}, {"key": "name_2"}]+ assert "fill(str(name))" in result.code+ assert "fill(str(name_2))" in result.code++ def test_param_keys_are_globally_unique_against_external_slug_collision(self) -> None:+ # An externally-derived slug "name 2" produces base "name_2", which must not collide with the+ # auto-suffix of two "name" fields. All keys are tracked in one global used-set.+ trajectory = [+ _interaction(+ "type_text",+ selector="#a",+ source_url="https://example.com/",+ typed_length=3,+ role="textbox",+ accessible_name="Name",+ ),+ _interaction(+ "type_text",+ selector="#b",+ source_url="https://example.com/",+ typed_length=3,+ role="textbox",+ accessible_name="name 2",+ ),+ _interaction(+ "type_text",+ selector="#c",+ source_url="https://example.com/",+ typed_length=3,+ role="textbox",+ accessible_name="Name",+ ),+ ]+ result = synthesize_code_block(trajectory)+ assert result is not None+ keys = [p["key"] for p in result.parameters]+ assert keys == ["name", "name_2", "name_3"]+ assert len(set(keys)) == len(keys)+ for key in keys:+ assert f"fill(str({key}))" in result.code++ def test_step_cap_truncates_at_twenty(self) -> None:+ trajectory = [+ _interaction("click", selector=f'role=button[name="b{i}"]', source_url="https://example.com/")+ for i in range(25)+ ]+ result = synthesize_code_block(trajectory)+ assert result is not None+ assert result.code.count(".click()") == 20+ assert any("truncated" in note for note in result.notes)+++class TestDeterminismAndEmpty:+ def test_empty_trajectory_returns_none(self) -> None:+ assert synthesize_code_block([]) is None++ def test_byte_identical_per_trajectory(self) -> None:+ trajectory = [+ _interaction(+ "type_text",+ selector='role=textbox[name="Search"]',+ source_url="https://example.com/",+ typed_length=4,+ role="textbox",+ accessible_name="Search",+ ),+ _interaction("press_key", key="Enter"),+ _interaction(+ "click",+ selector='role=button[name="Go"]',+ role="button",+ accessible_name="Go",+ ),+ ]+ first = synthesize_code_block(trajectory)+ second = synthesize_code_block(trajectory)+ assert first is not None and second is not None+ assert first.code == second.code+ assert first.parameters == second.parameters+ assert first.notes == second.notes+++class TestLineBoundaryEscaping:+ # str.splitlines() and several parsers treat each of these as a line boundary. An attacker-controlled+ # page can plant one in an accessible name or option value; left unescaped it splits the emitted+ # one-line literal across lines and corrupts the block (availability, not RCE — the leading quote+ # precedes the payload and every attacker quote is escaped).+ _BOUNDARY_CODEPOINTS = ("\x0b", "\x0c", "\x85", " ", " ")++ @staticmethod+ def _parses(code: str) -> ast.Module:+ wrapper = "async def __wrapper__(payload=None):\n" + "\n".join(+ f" {line.strip()}" for line in code.splitlines()+ )+ return ast.parse(wrapper)++ def test_accessible_name_boundary_codepoints_keep_block_parseable(self) -> None:+ for codepoint in self._BOUNDARY_CODEPOINTS:+ name = f"Search{codepoint}payload"+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector=f'role=button[name="{name}"]',+ source_url="https://example.com/",+ role="button",+ accessible_name=name,+ )+ ]+ )+ assert result is not None, f"codepoint U+{ord(codepoint):04X} produced no block"+ # The block parses with no SyntaxError despite the raw line boundary in the name.+ self._parses(result.code)+ # The raw codepoint never reaches the emitted source; it survives only as a backslash escape,+ # so the payload stays inert inside the single-line literal.+ assert codepoint not in result.code, f"raw U+{ord(codepoint):04X} leaked into emitted code"+ assert "payload" in result.code++ def test_select_option_value_boundary_codepoint_keeps_block_parseable(self) -> None:+ value = "small\x0bvalue"+ result = synthesize_code_block(+ [+ _interaction(+ "select_option",+ selector='role=combobox[name="Size"]',+ source_url="https://example.com/",+ value=value,+ role="combobox",+ accessible_name="Size",+ )+ ]+ )+ assert result is not None+ self._parses(result.code)+ assert "\x0b" not in result.code+ assert "select_option" in result.code++ def test_c0_control_codepoint_in_name_is_escaped(self) -> None:+ # Belt-and-suspenders: the defensive C0/C1 control pass also escapes non-line-boundary controls.+ name = "Search\x07bell"+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector=f'role=button[name="{name}"]',+ source_url="https://example.com/",+ role="button",+ accessible_name=name,+ )+ ]+ )+ assert result is not None+ self._parses(result.code)+ assert "\x07" not in result.code+++class TestPreflightSurfacesSyntaxError:+ def test_synthesized_block_round_trips_through_preflight(self) -> None:+ # A well-formed synthesized block (boundary codepoint escaped) yields no syntax diagnostic.+ result = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='role=button[name="Go\x0bx"]',+ source_url="https://example.com/",+ role="button",+ accessible_name="Go\x0bx",+ )+ ]+ )+ assert result is not None+ diagnostics = preflight_code_block(result.code, parameter_keys=())+ assert not any(d.code == "SYNTAX_ERROR" for d in diagnostics)++ def test_unparseable_block_surfaces_syntax_diagnostic(self) -> None:+ # A malformed block must be caught at authoring time, not swallowed into a silent run-time failure.+ diagnostics = preflight_code_block('await page.goto("unterminated)\n', parameter_keys=())+ assert any(d.code == "SYNTAX_ERROR" for d in diagnostics)+++class TestRenderSynthesizedOfferText:+ def test_renders_label_code_and_params(self) -> None:+ synthesized = synthesize_code_block(+ [+ _interaction(+ "type_text",+ selector="#searcher_s",+ source_url="https://example.com/",+ typed_length=5,+ role="textbox",+ accessible_name="Search",+ )+ ]+ )+ assert synthesized is not None+ text = render_synthesized_offer_text(synthesized)+ assert text.startswith("SYNTHESIZED CODE BLOCK (offered once).")+ assert _SYNTHESIZED_BLOCK_LABEL in text+ assert "```python" in text+ assert 'await page.locator("#searcher_s").fill(str(search))' in text+ assert "Workflow parameters referenced (bind these): search." in text++ def test_omits_param_line_when_no_parameters(self) -> None:+ synthesized = synthesize_code_block(+ [+ _interaction(+ "click",+ selector='role=button[name="Go"]',+ source_url="https://example.com/",+ role="button",+ accessible_name="Go",+ )+ ]+ )+ assert synthesized is not None+ text = render_synthesized_offer_text(synthesized)+ assert "Workflow parameters referenced" not in text++ def test_includes_synthesis_notes_when_present(self) -> None:+ synthesized = synthesize_code_block(+ [+ _interaction(+ "click",+ selector="li:nth-of-type(3) > a",+ source_url="https://example.com/",+ )+ ]+ )+ assert synthesized is not None+ assert synthesized.notes+ text = render_synthesized_offer_text(synthesized)+ assert "Synthesis notes: " in text+++def _code_block_yaml(label: str) -> str:+ return (+ "workflow_definition:\n"+ " blocks:\n"+ " - block_type: code\n"+ f" label: {label}\n"+ " code: |\n"+ ' await page.goto("https://example.com/")\n'+ )+++_SCOUT_TRAJECTORY = [+ {+ "tool_name": "type_text",+ "selector": "#search-box",+ "source_url": "https://example.com/",+ "typed_length": 5,+ "role": "textbox",+ "accessible_name": "Search",+ },+ {"tool_name": "press_key", "selector": "#search-box", "key": "Enter"},+ {+ "tool_name": "click",+ "selector": 'role=button[name="Add to cart"]',+ "role": "button",+ "accessible_name": "Add to cart",+ },+]+++class TestSynthesizedArtifactMetadata:+ def test_skeleton_passes_the_validator_with_placeholders(self) -> None:+ # The skeleton passes the validator with only <fill> placeholders for the model-owned slots.+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml(_SYNTHESIZED_BLOCK_LABEL))+ assert error is None+ assert list(normalized.keys()) == [_SYNTHESIZED_BLOCK_LABEL]++ def test_skeleton_never_asserts_satisfied_status(self) -> None:+ # The scout never ran+verified the authored block, so the only honest status is observed_not_verified.+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ statuses = [metadata["page_dependencies"][0]["status"], metadata["observation_refs"][0]["status"]]+ statuses += [claim["status"] for claim in metadata["claimed_outcomes"]]+ assert all(status == "observed_not_verified" for status in statuses)+ assert "satisfied" not in str(metadata)++ def test_skeleton_observation_ref_carries_scout_source_tool(self) -> None:+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ observation_ref = metadata["observation_refs"][0]+ assert observation_ref["source_tool"] == "scout_interaction"+ assert observation_ref["dependency_id"] == metadata["page_dependencies"][0]["id"]++ def test_skeleton_leaves_terminal_goal_for_the_model(self) -> None:+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ assert metadata["declared_goal"].startswith("<fill:")+ assert metadata["completion_criteria"][0]["text"].startswith("<fill:")+ assert metadata["claimed_outcomes"][0]["text"].startswith("<fill:")++ def test_skeleton_is_byte_identical_per_trajectory(self) -> None:+ import json++ first = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ second = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)+ assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)++ def test_skeleton_omits_url_hint_when_no_source_url(self) -> None:+ metadata = build_synthesized_artifact_metadata([_interaction("press_key", key="Enter")])+ assert "url_hint" not in metadata["page_dependencies"][0]+ assert "current_url" not in metadata["observation_refs"][0]++ def test_offer_text_embeds_metadata_when_trajectory_supplied(self) -> None:+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)+ assert synthesized is not None+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY)+ assert "code_artifact_metadata" in text+ assert "```json" in text+ assert "scout_interaction" in text+ assert "returns every remaining violation at once" in text++ def test_offer_text_omits_metadata_without_trajectory(self) -> None:+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)+ assert synthesized is not None+ text = render_synthesized_offer_text(synthesized)+ assert "code_artifact_metadata" not in text+ assert "```json" not in textdiff --git a/tests/unit/test_copilot_context.py b/tests/unit/test_copilot_context.pyindex 75a79d102..de91cd093 100644--- a/tests/unit/test_copilot_context.py+++ b/tests/unit/test_copilot_context.py@@ -34,6 +34,52 @@ def test_merge_turn_summary_caps_credentials_checked() -> None:assert len(ctx.credentials_checked) == 40+def test_merge_turn_summary_records_resolved_credential_ids() -> None:+ ctx = StructuredContext()+ activity = [+ {+ "tool": "list_credentials",+ "summary": "Found 2 credential(s)",+ "credentials": [+ {"credential_id": "cred_amazon", "name": "Amazon"},+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},+ ],+ }+ ]+ ctx.merge_turn_summary(activity)++ by_id = {check.credential_id: check for check in ctx.credentials_checked}+ assert set(by_id) == {"cred_amazon", "cred_quicken"}+ assert all(check.found for check in ctx.credentials_checked)+ assert by_id["cred_amazon"].credential_name == "Amazon"+++def test_resolved_credential_ids_survive_context_roundtrip() -> None:+ ctx = StructuredContext()+ ctx.merge_turn_summary(+ [+ {+ "tool": "list_credentials",+ "summary": "Found 1 credential(s)",+ "credentials": [{"credential_id": "cred_amazon", "name": "Amazon"}],+ }+ ]+ )++ rehydrated = StructuredContext.from_json_str(ctx.to_json_str())++ assert [check.credential_id for check in rehydrated.credentials_checked] == ["cred_amazon"]+++def test_merge_turn_summary_falls_back_to_summary_without_structured_credentials() -> None:+ ctx = StructuredContext()+ ctx.merge_turn_summary([{"tool": "list_credentials", "summary": "Found 0 credential(s)"}])++ assert len(ctx.credentials_checked) == 1+ assert ctx.credentials_checked[0].credential_id is None+ assert ctx.credentials_checked[0].found is False++def test_merge_observed_acted_pages_uses_nested_evidence_url() -> None:pages = _merge_observed_acted_pages([ObservedPage(url="https://example.com/old", had_bounded_schema=True, reached_via="navigate")],diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex a3fff1add..168f3ca73 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -117,6 +117,43 @@ async def test_on_tool_end_failed_whitelisted_tool_skips_preview() -> None:assert "output_preview" not in entry+async def test_on_tool_end_list_credentials_records_resolved_ids() -> None:+ ctx = _FakeContext()+ hooks = CopilotRunHooks(ctx)++ output = _mcp_text_output(+ {+ "ok": True,+ "data": {+ "credentials": [+ {"credential_id": "cred_amazon", "name": "Amazon", "username": "[email protected]"},+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},+ ],+ "count": 2,+ },+ }+ )+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("list_credentials"), output)++ entry = ctx.tool_activity[0]+ assert entry["credentials"] == [+ {"credential_id": "cred_amazon", "name": "Amazon"},+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},+ ]+++async def test_on_tool_end_list_credentials_empty_skips_field() -> None:+ ctx = _FakeContext()+ hooks = CopilotRunHooks(ctx)++ output = _mcp_text_output({"ok": True, "data": {"credentials": [], "count": 0}})+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("list_credentials"), output)++ assert "credentials" not in ctx.tool_activity[0]++@pytest.mark.asyncioasync def test_on_tool_end_swallows_unserializable_output() -> None:# json.dumps(default=str) can still raise if str() on the value raises --@@ -725,6 +762,7 @@ class TestBrowserInteractionObservationHooks:pending_browser_interaction_observation=None,discovery_mcp_server=None,scouted_interactions=[],+ scout_trajectory=[],pending_scout_source_url=None,)result = await _click_post_hook(@@ -754,6 +792,7 @@ class TestBrowserInteractionObservationHooks:),discovery_mcp_server=None,scouted_interactions=[],+ scout_trajectory=[],pending_scout_source_url=None,)@@ -781,6 +820,7 @@ class TestBrowserInteractionObservationHooks:pending_browser_interaction_observation=None,discovery_mcp_server=None,scouted_interactions=[],+ scout_trajectory=[],pending_scout_source_url=None,)@@ -803,6 +843,7 @@ class TestScoutedInteractionCapture:pending_browser_interaction_observation=None,discovery_mcp_server=None,scouted_interactions=[],+ scout_trajectory=[],observed_browser_urls=[],pending_scout_source_url=source_url,)@@ -1047,3 +1088,51 @@ class TestScoutedInteractionCapture:assert entry["reached_via"] == "interaction"assert entry["evidence"]["interaction_selector"] == "#sort"assert result["data"]["observation_step"] == entry["step"]+++class TestAssembleEnforcementMessages:+ @staticmethod+ def _screenshot_msg() -> dict[str, Any]:+ return {+ "role": "user",+ "content": [+ {"type": "text", "text": "screenshot"},+ {"type": "input_image", "image_url": "data:image/png;base64,AAAA"},+ ],+ }++ @staticmethod+ def _offer_msg() -> dict[str, Any]:+ return {"role": "user", "content": "Here is a code block you can add."}++ def test_screenshot_nudge_and_offer_ordering(self) -> None:+ from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, _assemble_enforcement_messages++ screenshot_msg = self._screenshot_msg()+ offer_msg = self._offer_msg()+ msgs = _assemble_enforcement_messages(screenshot_msg, "please finish the workflow", offer_msg)++ screenshot_indices = [i for i, m in enumerate(msgs) if m is screenshot_msg]+ assert screenshot_indices == [msgs.index(screenshot_msg)]+ assert len(screenshot_indices) == 1++ nudge_index = next(+ i+ for i, m in enumerate(msgs)+ if isinstance(m.get("content"), str) and m["content"].startswith(NUDGE_SENTINEL)+ )+ assert nudge_index == len(msgs) - 1++ offer_index = msgs.index(offer_msg)+ assert offer_index < nudge_index++ def test_offer_and_screenshot_without_nudge(self) -> None:+ from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, _assemble_enforcement_messages++ screenshot_msg = self._screenshot_msg()+ offer_msg = self._offer_msg()+ msgs = _assemble_enforcement_messages(screenshot_msg, None, offer_msg)++ assert offer_msg in msgs+ assert msgs.count(screenshot_msg) == 1+ assert not any(isinstance(m.get("content"), str) and m["content"].startswith(NUDGE_SENTINEL) for m in msgs)diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.pyindex 18dbf96cb..909f7b8b0 100644--- a/tests/unit/test_copilot_output_policy.py+++ b/tests/unit/test_copilot_output_policy.py@@ -1777,3 +1777,136 @@ def test_translate_to_agent_result_adds_unvalidated_affordance() -> None:assert agent_result.updated_workflow is workflowassert "Accept to save" in agent_result.user_responseassert "Reject to discard" in agent_result.user_response+++def _two_credential_workflow_yaml() -> str:+ return """+workflow_definition:+ parameters:+ - parameter_type: workflow+ workflow_parameter_type: credential_id+ key: site_a_credentials+ default_value: cred_amazon+ - parameter_type: workflow+ workflow_parameter_type: credential_id+ key: site_b_credentials+ default_value: cred_quicken+ blocks:+ - block_type: login+ label: login_a+ url: https://login-a.authenticationtest.test/login+ navigation_goal: Log in to site A.+ parameter_keys:+ - site_a_credentials+ - block_type: login+ label: login_b+ url: https://login-b.authenticationtest.test/login+ navigation_goal: Log in to site B.+ parameter_keys:+ - site_b_credentials+"""+++def _discovered(credential_id: str, tested_url: str | None) -> object:+ return SimpleNamespace(credential_id=credential_id, name=credential_id, tested_url=tested_url)+++def test_allows_discovered_bound_credential_with_no_resolved_credentials() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],+ credential_input_kind="none",+ ),+ workflow_yaml=_workflow_yaml(navigation_goal="Log in."),+ )++ assert verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes+++def test_allows_two_discovered_bound_credentials_in_one_save() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[+ _discovered("cred_amazon", "https://login-a.authenticationtest.test/login"),+ _discovered("cred_quicken", "https://login-b.authenticationtest.test/login"),+ ],+ credential_input_kind="none",+ ),+ workflow_yaml=_two_credential_workflow_yaml(),+ )++ assert verdict.allowed+++def test_allows_resolved_and_discovered_credentials_together_in_one_save() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[_credential("cred_amazon", "https://login-a.authenticationtest.test/login")],+ discovered_credentials=[_discovered("cred_quicken", "https://login-b.authenticationtest.test/login")],+ credential_input_kind="none",+ ),+ workflow_yaml=_two_credential_workflow_yaml(),+ )++ assert verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes+++def test_rejects_when_only_one_of_two_bound_credentials_is_discovered() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[_discovered("cred_amazon", "https://login-a.authenticationtest.test/login")],+ credential_input_kind="none",+ ),+ workflow_yaml=_two_credential_workflow_yaml(),+ )++ assert not verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes+++def test_rejects_discovered_credential_id_that_is_not_bound_in_workflow() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],+ credential_input_kind="none",+ ),+ user_response="I used cred_safe for the login.",+ )++ assert not verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes+++def test_rejects_fabricated_credential_id_not_in_discovered_set() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[_discovered("cred_amazon", "https://login.example.test/login")],+ credential_input_kind="none",+ ),+ workflow_yaml=_workflow_yaml(navigation_goal="Log in."),+ )++ assert not verdict.allowed+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes+++def test_rejects_discovered_credential_bound_to_new_origin() -> None:+ verdict = evaluate_output_policy(+ request_policy=_policy(+ resolved_credentials=[],+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],+ credential_input_kind="none",+ ),+ workflow_yaml=_workflow_yaml(url="https://evil.example.test/login"),+ )++ assert not verdict.allowed+ assert OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED in verdict.reason_codesdiff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.pyindex 39b7a7515..2c90badc1 100644--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py@@ -1,8 +1,13 @@from __future__ import annotationsimport textwrap+from types import SimpleNamespace+from unittest.mock import AsyncMock, patch-from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound+import pytest++from skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext+from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policydef _yaml(body: str) -> str:@@ -212,3 +217,70 @@ def test_malformed_or_empty_yaml_is_inert() -> None:assert _workflow_credential_inputs_unbound("") == []assert _workflow_credential_inputs_unbound("- not a workflow yaml\n") == []assert _workflow_credential_inputs_unbound(":: broken yaml ::") == []+++def _discovered_context(*credential_ids: str) -> str:+ structured = StructuredContext(+ credentials_checked=[+ CredentialCheck(credential_name=cid, credential_id=cid, found=True) for cid in credential_ids+ ]+ )+ return structured.to_json_str()+++async def test_discovered_credentials_seed_approved_set_on_none_turn() -> None:+ org_credentials = [+ SimpleNamespace(credential_id="cred_amazon"),+ SimpleNamespace(credential_id="cred_quicken"),+ ]+ with patch(+ "skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids",+ new=AsyncMock(return_value=org_credentials),+ ):+ policy = await build_request_policy(+ user_message="yes, use both of those",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_discovered_context("cred_amazon", "cred_quicken"),+ organization_id="o_test",+ handler=None,+ )++ assert policy.credential_input_kind == "none"+ assert [c.credential_id for c in policy.discovered_credentials] == ["cred_amazon", "cred_quicken"]+++async def test_discovered_credential_absent_from_org_is_not_approved() -> None:+ with patch(+ "skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids",+ new=AsyncMock(return_value=[SimpleNamespace(credential_id="cred_amazon")]),+ ):+ policy = await build_request_policy(+ user_message="yes",+ workflow_yaml="",+ chat_history=[],+ global_llm_context=_discovered_context("cred_amazon", "cred_ghost"),+ organization_id="o_test",+ handler=None,+ )++ assert [c.credential_id for c in policy.discovered_credentials] == ["cred_amazon"]+++async def test_no_discovered_credentials_leaves_approved_set_empty() -> None:+ get_by_ids = AsyncMock(return_value=[])+ with patch("skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids", new=get_by_ids):+ policy = await build_request_policy(+ user_message="add a step to download the report",+ workflow_yaml="",+ chat_history=[],+ global_llm_context="",+ organization_id="o_test",+ handler=None,+ )++ assert policy.discovered_credentials == []+ get_by_ids.assert_not_awaited()diff --git a/tests/unit/test_copilot_tools_credential_id_misbinding.py b/tests/unit/test_copilot_tools_credential_id_misbinding.pyindex 95a9f8524..23b3cc5e5 100644--- a/tests/unit/test_copilot_tools_credential_id_misbinding.py+++ b/tests/unit/test_copilot_tools_credential_id_misbinding.py@@ -5,10 +5,12 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from skyvern.forge.sdk.copilot.request_policy import RequestPolicyfrom skyvern.forge.sdk.copilot.runtime import AgentContextfrom skyvern.forge.sdk.copilot.tools import (_credential_id_misbinding_error_message,_credential_id_misbinding_findings,+ _list_credentials,_update_workflow,)@@ -456,3 +458,60 @@ def test_error_message_groups_multiple_findings() -> None:assert "navigation_goal" in messageassert "complete_criterion" in messageassert message.count("cred_111") >= 1+++class _ListedCredential:+ def __init__(self, credential_id: str, name: str, tested_url: str | None = None) -> None:+ self.credential_id = credential_id+ self.name = name+ self.tested_url = tested_url+ self.credential_type = "password"+ self.username = None+ self.totp_type = None+ self.card_last4 = None+ self.card_brand = None+ self.secret_label = None+++def _ctx_with_policy() -> MagicMock:+ ctx = _ctx()+ ctx.request_policy = RequestPolicy(credential_input_kind="none")+ return ctx+++async def test_list_credentials_records_discovered_credentials_on_policy() -> None:+ ctx = _ctx_with_policy()+ listed = [_ListedCredential("cred_site_a", "Site A"), _ListedCredential("cred_site_b", "Site B")]+ with patch(+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)+ ):+ await _list_credentials({}, ctx)++ assert [c.credential_id for c in ctx.request_policy.discovered_credentials] == ["cred_site_a", "cred_site_b"]+++async def test_list_credentials_unions_across_pages_without_duplicates() -> None:+ ctx = _ctx_with_policy()+ ctx.request_policy.discovered_credentials = [_ListedCredential("cred_site_a", "Site A")]+ listed = [_ListedCredential("cred_site_a", "Site A"), _ListedCredential("cred_site_b", "Site B")]+ with patch(+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)+ ):+ await _list_credentials({"page": 2}, ctx)++ assert [c.credential_id for c in ctx.request_policy.discovered_credentials] == ["cred_site_a", "cred_site_b"]+++async def test_list_credentials_without_request_policy_is_inert() -> None:+ ctx = _ctx()+ ctx.request_policy = None+ listed = [_ListedCredential("cred_site_a", "Site A")]+ with patch(+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)+ ):+ result = await _list_credentials({}, ctx)++ assert result["ok"] is Truediff --git a/tests/unit/test_summarize_output.py b/tests/unit/test_summarize_output.pyindex 8b1902836..4a7624cfc 100644--- a/tests/unit/test_summarize_output.py+++ b/tests/unit/test_summarize_output.py@@ -1,5 +1,6 @@"""Tests for the summarize-output endpoint and helpers."""+import sysfrom contextlib import contextmanagerfrom datetime import datetime, timezonefrom typing import Iterator@@ -60,10 +61,16 @@ class TestSummarizeOutputRequest:SummarizeOutputRequest(output_json="{}", workflow_title="x" * 501)def test_deeply_nested_json_rejected(self) -> None:- # 10k levels of nesting exceeds Python's recursion limit inside json.loads.+ # Rejection relies on json.loads overflowing the recursion limit; pin it so the test does not+ # depend on the suite-global limit another test may have raised (e.g. via networkx).deep = "[" * 10_000 + "]" * 10_000- with pytest.raises(ValidationError):- SummarizeOutputRequest(output_json=deep)+ old_limit = sys.getrecursionlimit()+ sys.setrecursionlimit(1000)+ try:+ with pytest.raises(ValidationError):+ SummarizeOutputRequest(output_json=deep)+ finally:+ sys.setrecursionlimit(old_limit)def _fake_org() -> Organization: