Skip to content

Automate browser based workflows with AI

SKY-10795: batch artifact-metadata violations + deterministic metadata for synthesized code blocks (#6464)

Andrew Neilsonf8b52147ce67
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index a2b2f776f..d7ab2f595 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -38,7 +38,8 @@ from skyvern.forge.sdk.copilot.blocker_signal import (
)
from skyvern.forge.sdk.copilot.blocker_signal import to_trace_data as blocker_signal_to_trace_data
from skyvern.forge.sdk.copilot.build_phase import initial_build_phase
-from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.context import (
COPILOT_RESPONSE_TYPES,
AgentResult,
@@ -160,6 +161,12 @@ Code block runtime facts:
- For multi-line code in workflow YAML, always use a YAML block scalar
(`code: |`). Never include placeholder text such as `[... truncated ...]`;
pass complete workflow YAML to update tools.
+
+When a SYNTHESIZED CODE BLOCK is offered to you, it already encodes the page
+interactions you scouted as deterministic Playwright. Persist that block VERBATIM
+via update_workflow / update_and_run_blocks — do not rewrite, reorder, or
+re-derive its locators. Only hand-author the steps it does not cover, such as the
+extraction or report block that returns the structured result.
"""
@@ -475,6 +482,43 @@ def _runtime_verification_evidence_prompt(ctx: CopilotContext | None) -> str:
)
+def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
+ """Pre-authoring offer of the synthesized code block.
+
+ Trips the ``synthesized_block_offered`` latch (shared with the post-turn enforcement offer)
+ only when a non-None offer is actually rendered, so an empty trajectory leaves it open.
+ """
+ if ctx is None:
+ return ""
+ if normalize_block_authoring_policy(ctx.block_authoring_policy) != BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="policy_not_code_only_browser")
+ return ""
+ if ctx.update_workflow_called:
+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="already_authored")
+ return ""
+ if ctx.synthesized_block_offered:
+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="already_offered")
+ return ""
+ if not ctx.scout_trajectory:
+ LOG.debug("copilot_synthesized_block_offer_skipped", reason="empty_trajectory")
+ return ""
+ synthesized = synthesize_code_block(ctx.scout_trajectory)
+ if synthesized is None:
+ LOG.debug(
+ "copilot_synthesized_block_offer_skipped",
+ reason="synthesis_returned_none",
+ trajectory_len=len(ctx.scout_trajectory),
+ )
+ return ""
+ ctx.synthesized_block_offered = True
+ LOG.info(
+ "copilot_synthesized_block_offer_rendered",
+ trajectory_len=len(ctx.scout_trajectory),
+ code_len=len(synthesized.code),
+ )
+ return "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory)
+
+
def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -> Callable[[object, object], str]:
base_system_prompt = _build_system_prompt(tool_usage_guide=tool_usage_guide, config=config)
@@ -501,7 +545,12 @@ def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -
+ "and tell the user to store the redacted secret as a saved credential before testing. "
+ "If `resolved_credentials` are present, use those `credential_id` values."
)
- return prompt + _runtime_verification_evidence_prompt(ctx) + _docs_answer_turn_directive(ctx.turn_intent)
+ return (
+ prompt
+ + _runtime_verification_evidence_prompt(ctx)
+ + _synthesized_block_offer_prompt(ctx)
+ + _docs_answer_turn_directive(ctx.turn_intent)
+ )
return instructions
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index 3c54a0baf..edf0661b9 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -68,8 +68,16 @@ def preflight_code_block(
def _static_ast_diagnostics(code: str) -> list[CodeBlockPreflightDiagnostic]:
try:
tree = ast.parse(_build_typed_module(code, parameter_keys=()))
- except SyntaxError:
- return []
+ except SyntaxError as exc:
+ # The wrapper scaffolding is static and valid, so any SyntaxError is in the supplied code —
+ # e.g. an attacker-page string with a raw line-boundary codepoint that splits a literal. Surface
+ # it at authoring time instead of letting the block fail silently at run time.
+ return [
+ CodeBlockPreflightDiagnostic(
+ code="SYNTAX_ERROR",
+ message=f"Code block does not parse as Python: {exc.msg}. Fix the snippet before persisting it.",
+ )
+ ]
diagnostics: list[CodeBlockPreflightDiagnostic] = []
for node in ast.walk(tree):
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
new file mode 100644
index 000000000..c90a9e0e6
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -0,0 +1,379 @@
+"""Deterministic synthesis of a copilot `code` block from a scout trajectory.
+
+Pure module: ``synthesize_code_block`` is a function of its input trajectory
+only — no LLM, no I/O, byte-identical output per trajectory. It turns the
+scout's captured interaction sequence into a bounded, linear Playwright snippet
+that runs on the raw ``page`` object the copilot code block executes against.
+"""
+
+from __future__ import annotations
+
+import json
+import keyword
+import re
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from typing import Any
+
+from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL
+
+_MAX_STEPS = 20
+_INDENT = " "
+
+_SYNTHESIZED_BLOCK_LABEL = "scout_synthesized_browser_steps"
+
+# Names the code-block executor reserves in its exec() namespace (block.py build_safe_vars
+# plus the injected `page`). A parameter key colliding with one of these is silently dropped
+# at bind time, so the synthesized fill would stringify the builtin instead of the user value.
+_RESERVED_PARAM_NAMES = frozenset(
+ {
+ "page",
+ "print",
+ "len",
+ "range",
+ "str",
+ "int",
+ "float",
+ "dict",
+ "list",
+ "tuple",
+ "set",
+ "bool",
+ "isinstance",
+ "enumerate",
+ "any",
+ "all",
+ "max",
+ "min",
+ "sum",
+ "sorted",
+ "sleep",
+ "asyncio",
+ "re",
+ "json",
+ "html",
+ "Exception",
+ }
+)
+
+# role=<role>[name="<name>"] optionally followed by `>> nth=<n>` or other engines.
+_ROLE_NAME_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')
+
+# Positional/index engines whose match depends on document order, not element identity. A captured
+# selector containing one of these is fragile, so an ARIA role/name anchor (when available) is preferred.
+_POSITIONAL_RE = re.compile(
+ r":nth-of-type\(|:nth-child\(|:nth-last-of-type\(|:nth-last-child\(|>>\s*nth=|:first-child|:last-child"
+)
+
+
+@dataclass
+class SynthesizedCodeBlock:
+ code: str
+ parameters: list[dict[str, str]] = field(default_factory=list)
+ notes: list[str] = field(default_factory=list)
+
+
+# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a
+# captured accessible name or option value would split the emitted one-line literal across lines and
+# corrupt the block. repr() does not escape U+2028/U+2029, so they (and the C0/C1 controls below) are
+# escaped explicitly to keep every emitted literal single-line.
+_EXTRA_LINE_SEPARATORS = ("\u2028", "\u2029")
+_CONTROL_CODEPOINTS = frozenset(
+ chr(cp) for cp in (*range(0x00, 0x20), 0x7F, *range(0x80, 0xA0)) if chr(cp) not in ("\n", "\r")
+)
+
+
+def _py_str(value: str) -> str:
+ """A deterministic double-quoted Python string literal."""
+ escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r")
+ escaped = "".join(f"\\x{ord(ch):02x}" if ch in _CONTROL_CODEPOINTS else ch for ch in escaped)
+ for separator in _EXTRA_LINE_SEPARATORS:
+ escaped = escaped.replace(separator, f"\\u{ord(separator):04x}")
+ return f'"{escaped}"'
+
+
+def _unescape_role_name(value: str) -> str:
+ # ref_to_selector escapes only `"` as `\"`; reverse it for the get_by_role name.
+ return value.replace('\\"', '"')
+
+
+def _parse_role_name(selector: str) -> tuple[str, str | None, str] | None:
+ """Parse a `role=...[name="..."]` selector. Returns (role, name, suffix) or None.
+
+ ``suffix`` is the residual engine chain (e.g. ` >> nth=2`); a non-empty suffix
+ means the selector cannot be expressed as a plain get_by_role and the caller
+ falls back to page.locator.
+ """
+ match = _ROLE_NAME_RE.match(selector)
+ if not match:
+ return None
+ role, raw_name, suffix = match.group(1), match.group(2), match.group(3)
+ name = _unescape_role_name(raw_name) if raw_name is not None else None
+ return role, name, suffix.strip()
+
+
+def _is_positional_selector(selector: str) -> bool:
+ """True when the captured selector's match depends on document position, not element identity.
+
+ Stable anchors (id, [name=...], [data-testid=...], [aria-label=...], a non-indexed CSS path) are
+ preferred verbatim; only a positional/index selector is worth trading for an ARIA role/name anchor.
+ """
+ return bool(_POSITIONAL_RE.search(selector))
+
+
+def _slug(value: str) -> str:
+ cleaned = re.sub(r"[^a-zA-Z0-9]+", "_", value.strip().lower()).strip("_")
+ if cleaned and cleaned[0].isdigit():
+ cleaned = f"field_{cleaned}"
+ return cleaned or "field"
+
+
+def _safe_param_base(value: str) -> str:
+ base = _slug(value)
+ if keyword.iskeyword(base) or base in _RESERVED_PARAM_NAMES:
+ return f"{base}_field"
+ return base
+
+
+def _get_by_role_expr(role: str, name: str) -> str:
+ if name:
+ return f"page.get_by_role({_py_str(role)}, name={_py_str(name)})"
+ return f"page.get_by_role({_py_str(role)})"
+
+
+def _locator_expr(
+ interaction: Mapping[str, Any],
+ notes: list[str],
+) -> str:
+ """Selector-first: emit the scout's captured working selector verbatim, and only fall back to an
+ ARIA get_by_role anchor when that selector is positional/index-based (and a role/name is available).
+
+ The accessible name is read on the scout/MCP surface; a `code:` block runs on a raw Playwright page
+ where get_by_role's native name computation may not reproduce it. The captured selector is the proven,
+ timing-independent locator the scout actually drove, so it wins for any stable selector.
+ """
+ selector = str(interaction.get("selector") or "").strip()
+ role = str(interaction.get("role") or "").strip()
+ name = str(interaction.get("accessible_name") or "").strip()
+
+ parsed = _parse_role_name(selector) if selector else None
+ if parsed is not None:
+ parsed_role, parsed_name, suffix = parsed
+ # A `role=...` selector is itself an ARIA anchor (ref_to_selector form), not a native CSS path —
+ # there is no underlying stable selector to prefer, so express it as get_by_role.
+ if not suffix:
+ return _get_by_role_expr(role or parsed_role, name or (parsed_name or ""))
+ # Residual engine chain (e.g. `>> nth=`) makes the parsed form positional; anchor on role/name.
+ anchor_role = role or parsed_role
+ anchor_name = name or (parsed_name or "")
+ if anchor_role and anchor_name:
+ return _get_by_role_expr(anchor_role, anchor_name)
+ return f"page.locator({_py_str(selector)})"
+
+ if selector:
+ if _is_positional_selector(selector) and role and name:
+ return _get_by_role_expr(role, name)
+ if _is_positional_selector(selector):
+ notes.append(f"low-confidence locator: positional selector {selector!r} with no role/name to anchor on")
+ return f"page.locator({_py_str(selector)})"
+
+ if role and name:
+ return _get_by_role_expr(role, name)
+
+ notes.append("dropped an interaction with no selector and no role/name")
+ return ""
+
+
+def _param_key(interaction: Mapping[str, Any], used: set[str]) -> str:
+ name = str(interaction.get("accessible_name") or "").strip()
+ role = str(interaction.get("role") or "").strip()
+ base = _safe_param_base(name or role or "value")
+ candidate = base
+ suffix = 2
+ while candidate in used:
+ candidate = f"{base}_{suffix}"
+ suffix += 1
+ used.add(candidate)
+ return candidate
+
+
+def synthesize_code_block(trajectory: Sequence[Mapping[str, Any]]) -> SynthesizedCodeBlock | None:
+ """Deterministically synthesize a code block from a scout trajectory, or None if empty."""
+ if not trajectory:
+ return None
+
+ lines: list[str] = []
+ notes: list[str] = []
+ parameters: list[dict[str, str]] = []
+ used_param_keys: set[str] = set()
+
+ entry_url = ""
+ entry_index = -1
+ for index, interaction in enumerate(trajectory):
+ candidate = str(interaction.get("source_url") or "").strip()
+ if candidate:
+ entry_url = candidate
+ entry_index = index
+ break
+ if entry_url:
+ if entry_index > 0:
+ notes.append("entry URL taken from a later interaction; earlier steps had no source_url")
+ lines.append(f"{_INDENT}await page.goto({_py_str(entry_url)})")
+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+
+ emitted = 0
+ for interaction in trajectory:
+ if emitted >= _MAX_STEPS:
+ notes.append(f"trajectory truncated at {_MAX_STEPS} steps")
+ break
+ tool_name = str(interaction.get("tool_name") or "")
+
+ if tool_name == "press_key":
+ key = str(interaction.get("key") or "").strip()
+ if not key:
+ continue
+ locator = _locator_expr(interaction, notes) if interaction.get("selector") else ""
+ if locator:
+ lines.append(f"{_INDENT}await {locator}.press({_py_str(key)})")
+ else:
+ lines.append(f"{_INDENT}await page.keyboard.press({_py_str(key)})")
+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ emitted += 1
+ continue
+
+ locator = _locator_expr(interaction, notes)
+ if not locator:
+ continue
+
+ if tool_name == "click":
+ lines.append(f"{_INDENT}await {locator}.click()")
+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ elif tool_name == "type_text":
+ param_key = _param_key(interaction, used_param_keys)
+ parameters.append({"key": param_key})
+ lines.append(f"{_INDENT}await {locator}.fill(str({param_key}))")
+ elif tool_name == "select_option":
+ value = str(interaction.get("value") or "").strip()
+ if not value:
+ notes.append("dropped a select_option interaction with no recorded value")
+ continue
+ lines.append(f"{_INDENT}await {locator}.select_option({_py_str(value)})")
+ lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ else:
+ notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")
+ continue
+ emitted += 1
+
+ if not lines:
+ return None
+
+ code = "\n".join(lines) + "\n"
+ return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes)
+
+
+_DEPENDENCY_ID = f"dependency:{_SYNTHESIZED_BLOCK_LABEL}_reached"
+_OBSERVATION_REF_ID = f"observation:{_SYNTHESIZED_BLOCK_LABEL}_scout"
+
+# Model-owned slots the synthesizer cannot prove; the model fills these.
+_FILL_DECLARED_GOAL = "<fill: the durable goal this block accomplishes>"
+_FILL_CLAIM_ID = "claim:<fill>"
+_FILL_CLAIM_TEXT = "<fill: the user-facing outcome this block claims>"
+_FILL_CRITERION_ID = "criterion:<fill>"
+_FILL_CRITERION_TEXT = "<fill: the terminal completion criterion>"
+
+
+def build_synthesized_artifact_metadata(trajectory: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
+ """Fill only the scout-proven evidence shape (`observed_not_verified`); the terminal goal and
+ outcomes stay as `<fill>` placeholders the model owns."""
+ # First recorded source_url is the page the synthesized block's leading `page.goto` lands on.
+ # The trajectory carries only pre-action source pages, not post-action reached URLs, so this is
+ # an advisory entry-page hint, never a reached-page identity (which SPAs would mis-key anyway).
+ entry_url_hint = next(
+ (url for url in (str(interaction.get("source_url") or "").strip() for interaction in trajectory) if url),
+ None,
+ )
+
+ page_dependency: dict[str, Any] = {
+ "id": _DEPENDENCY_ID,
+ "scope": "page",
+ "status": "observed_not_verified",
+ "observation_refs": [_OBSERVATION_REF_ID],
+ }
+ if entry_url_hint:
+ page_dependency["url_hint"] = entry_url_hint
+
+ observation_ref: dict[str, Any] = {
+ "observation_ref": _OBSERVATION_REF_ID,
+ "dependency_id": _DEPENDENCY_ID,
+ "status": "observed_not_verified",
+ "source_tool": SCOUT_INTERACTION_EVIDENCE_TOOL,
+ }
+
+ return {
+ "block_label": _SYNTHESIZED_BLOCK_LABEL,
+ "declared_goal": _FILL_DECLARED_GOAL,
+ "page_dependencies": [page_dependency],
+ "observation_refs": [observation_ref],
+ "claimed_outcomes": [
+ {
+ "id": _FILL_CLAIM_ID,
+ "scope": "outcome",
+ "text": _FILL_CLAIM_TEXT,
+ "status": "observed_not_verified",
+ "depends_on": [_DEPENDENCY_ID],
+ "covered_criteria": [_FILL_CRITERION_ID],
+ "observation_refs": [_OBSERVATION_REF_ID],
+ }
+ ],
+ "completion_criteria": [
+ {
+ "id": _FILL_CRITERION_ID,
+ "text": _FILL_CRITERION_TEXT,
+ "level": "terminal",
+ "terminal": True,
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "id": "expectation:<fill>",
+ "text": "<fill: what terminal verification must observe>",
+ "criteria_ids": [_FILL_CRITERION_ID],
+ }
+ ],
+ }
+
+
+def _render_artifact_metadata_block(metadata: Mapping[str, Any]) -> str:
+ return json.dumps(metadata, indent=2, sort_keys=True)
+
+
+def render_synthesized_offer_text(
+ synthesized: SynthesizedCodeBlock,
+ trajectory: Sequence[Mapping[str, Any]] | None = None,
+) -> str:
+ """Render the offer body the copilot sees for a synthesized block (pure)."""
+ param_keys = [p.get("key", "") for p in synthesized.parameters if p.get("key")]
+ parts = [
+ "SYNTHESIZED CODE BLOCK (offered once). The page interactions you scouted were compiled into a "
+ "deterministic Playwright snippet. Persist it VERBATIM as a `code` block labeled "
+ f"`{_SYNTHESIZED_BLOCK_LABEL}` via update_workflow / update_and_run_blocks; only hand-author the "
+ "extract/report steps it does not cover.",
+ "```python",
+ synthesized.code.rstrip("\n"),
+ "```",
+ ]
+ if param_keys:
+ parts.append("Workflow parameters referenced (bind these): " + ", ".join(param_keys) + ".")
+ if synthesized.notes:
+ parts.append("Synthesis notes: " + "; ".join(synthesized.notes) + ".")
+ if trajectory:
+ metadata = build_synthesized_artifact_metadata(trajectory)
+ parts.append(
+ "Pass this `code_artifact_metadata` for the block (the scouted page evidence is filled in; "
+ "replace the `<fill: ...>` slots with the terminal goal and outcome this block delivers, then "
+ "submit it whole — the validator returns every remaining violation at once):"
+ )
+ parts.append("```json")
+ parts.append(_render_artifact_metadata_block(metadata))
+ parts.append("```")
+ return "\n".join(parts)
diff --git a/skyvern/forge/sdk/copilot/composition_browser_expressions.py b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
index b1f90a4d7..a91843324 100644
--- a/skyvern/forge/sdk/copilot/composition_browser_expressions.py
+++ b/skyvern/forge/sdk/copilot/composition_browser_expressions.py
@@ -40,6 +40,66 @@ COMPOSITION_STRIPPED_HTML_EXPRESSION = (
"})()"
)
+
+# Given a CSS selector, return the element's ARIA role and accessible name so the code-block
+# synthesizer has a get_by_role fallback anchor for a positional/unstable captured selector. The
+# name is read only from true label sources, never the element's own textContent/value.
+def scout_accessible_role_name_expression(css_selector: str) -> str:
+ sel = json.dumps(css_selector)
+ return (
+ "(() => {"
+ f" const el = document.querySelector({sel});"
+ " if (!el) return null;"
+ " const text = (v) => String(v == null ? '' : v).replace(/\\s+/g, ' ').trim();"
+ " const implicitRole = (node) => {"
+ " const tag = (node.tagName || '').toLowerCase();"
+ " const type = (node.getAttribute('type') || '').toLowerCase();"
+ " if (tag === 'a' && node.hasAttribute('href')) return 'link';"
+ " if (tag === 'button') return 'button';"
+ " if (tag === 'select') return 'combobox';"
+ " if (tag === 'textarea') return 'textbox';"
+ " if (tag === 'input') {"
+ " if (['button', 'submit', 'reset'].includes(type)) return 'button';"
+ " if (type === 'checkbox') return 'checkbox';"
+ " if (type === 'radio') return 'radio';"
+ " if (['text', 'search', 'email', 'tel', 'url', 'password', ''].includes(type)) return 'textbox';"
+ " }"
+ " if (/^h[1-6]$/.test(tag)) return 'heading';"
+ " return '';"
+ " };"
+ " const accessibleName = (node) => {"
+ " const aria = text(node.getAttribute('aria-label'));"
+ " if (aria) return aria;"
+ " const labelledby = node.getAttribute('aria-labelledby');"
+ " if (labelledby) {"
+ " const parts = labelledby.split(/\\s+/).map((id) => {"
+ " const ref = document.getElementById(id);"
+ " return ref ? text(ref.textContent) : '';"
+ " }).filter(Boolean);"
+ " if (parts.length) return text(parts.join(' '));"
+ " }"
+ " const id = node.getAttribute('id');"
+ " if (id) {"
+ " let lab = null;"
+ " try { lab = document.querySelector('label[for=\"' + (window.CSS && CSS.escape ? CSS.escape(id) : id) + '\"]'); } catch (e) { lab = null; }"
+ " if (lab) { const t = text(lab.textContent); if (t) return t; }"
+ " }"
+ " const parentLabel = node.closest ? node.closest('label') : null;"
+ " if (parentLabel) { const t = text(parentLabel.textContent); if (t) return t; }"
+ # textContent/value are never name sources: for a typed-into textbox/contenteditable
+ # they would leak the raw typed value as accessible_name.
+ " const title = text(node.getAttribute('title'));"
+ " if (title) return title;"
+ " const placeholder = text(node.getAttribute('placeholder'));"
+ " if (placeholder) return placeholder;"
+ " return '';"
+ " };"
+ " const role = text(el.getAttribute('role')) || implicitRole(el);"
+ " return { role: role, accessible_name: accessibleName(el) };"
+ "})()"
+ )
+
+
COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION = (
"(() => {"
" const body = document.body; if (!body) return [];"
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 244c35fbf..fd2557a6f 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -152,9 +152,26 @@ class StructuredContext(BaseModel):
self.urls_visited.append(UrlVisit(url=url, summary=""))
elif tool == "list_credentials":
- match = re.search(r"Found (\d+)", summary)
- found = int(match.group(1)) > 0 if match else False
- self.credentials_checked.append(CredentialCheck(credential_name=summary, found=found))
+ resolved = entry.get("credentials")
+ if isinstance(resolved, list) and resolved:
+ for credential in resolved:
+ if not isinstance(credential, dict):
+ continue
+ credential_id = credential.get("credential_id")
+ if not isinstance(credential_id, str):
+ continue
+ name = credential.get("name")
+ self.credentials_checked.append(
+ CredentialCheck(
+ credential_name=name if isinstance(name, str) else "",
+ credential_id=credential_id,
+ found=True,
+ )
+ )
+ else:
+ match = re.search(r"Found (\d+)", summary)
+ found = int(match.group(1)) > 0 if match else False
+ self.credentials_checked.append(CredentialCheck(credential_name=summary, found=found))
elif tool == "type_text":
parts = summary.split("into ")
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 17ca00a84..d5741b44a 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -15,6 +15,7 @@ from agents.run import Runner
from skyvern.config import settings
from skyvern.forge.sdk.copilot import config as copilot_config_defaults
+from skyvern.forge.sdk.copilot.code_block_synthesis import render_synthesized_offer_text, synthesize_code_block
from skyvern.forge.sdk.copilot.config import (
DEFAULT_ENFORCEMENT_NUDGES,
DEFAULT_TOKEN_BUDGET,
@@ -37,7 +38,9 @@ from skyvern.forge.sdk.copilot.config import (
POST_SUSPICIOUS_SUCCESS_NUDGE,
POST_UPDATE_NUDGE,
SCREENSHOT_DROPPED_NUDGE,
+ BlockAuthoringPolicy,
CopilotConfig,
+ normalize_block_authoring_policy,
)
from skyvern.forge.sdk.copilot.diagnosis_repair_contract import RepairNextAction
from skyvern.forge.sdk.copilot.failure_tracking import PER_TOOL_BUDGET_FAILURE_CATEGORY, normalize_failure_reason
@@ -1446,6 +1449,52 @@ async def _run_streamed_with_deadline(
return result
+def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:
+ """Post-turn fallback offer of a deterministically synthesized code block, in code-only mode.
+
+ Returns a single user message wrapping the synthesized Playwright block, or
+ None when the policy/latch/empty-trajectory guards do not hold. Shares the
+ one-shot latch with the pre-authoring prompt-side offer, so whichever fires
+ first wins and the model sees the offer at most once.
+ """
+ if getattr(ctx, "synthesized_block_offered", False):
+ return None
+ if getattr(ctx, "update_workflow_called", False):
+ return None
+ if normalize_block_authoring_policy(getattr(ctx, "block_authoring_policy", None)) != (
+ BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ ):
+ return None
+ trajectory = getattr(ctx, "scout_trajectory", None) or []
+ if not trajectory:
+ return None
+ synthesized = synthesize_code_block(trajectory)
+ if synthesized is None:
+ return None
+
+ ctx.synthesized_block_offered = True
+ return {"role": "user", "content": render_synthesized_offer_text(synthesized, trajectory)}
+
+
+def _assemble_enforcement_messages(
+ screenshot_msg: dict[str, Any] | None,
+ nudge_content: str | None,
+ synthesized_msg: dict[str, Any] | None,
+) -> list[Any]:
+ """Build the extra messages for an enforcement retry, ordered so a nudge, when present, stays last.
+
+ The screenshot rides as its own user-role message because OpenAI rejects image parts inside a tool message.
+ """
+ extra_msgs: list[Any] = []
+ if screenshot_msg is not None:
+ extra_msgs.append(screenshot_msg)
+ if nudge_content is not None:
+ extra_msgs.append({"role": "user", "content": NUDGE_SENTINEL + nudge_content})
+ if synthesized_msg is not None:
+ extra_msgs.insert(0, synthesized_msg)
+ return extra_msgs
+
+
async def run_with_enforcement(
agent: Agent,
initial_input: str | list,
@@ -1573,12 +1622,19 @@ async def run_with_enforcement(
pending_recovery_nudge = None
else:
nudge = _check_enforcement(ctx, result, copilot_config)
- if nudge is None:
+
+ # The offer is independent of the nudge: a clean scout-then-author turn
+ # finalizes with nudge=None, so injecting it only inside the nudge branch
+ # would never reach the model. Compute it once here so it rides both the
+ # nudge path and the finalize path.
+ synthesized_msg = _maybe_synthesized_block_offer_msg(ctx)
+
+ if nudge is None and synthesized_msg is None:
_consume_pending_screenshots(ctx)
_maybe_raise_non_retriable_nav(ctx)
return result
- if nudge == _nudge(copilot_config, "post_update"):
+ if nudge is not None and nudge == _nudge(copilot_config, "post_update"):
if ctx.post_update_nudge_count >= MAX_POST_UPDATE_NUDGES:
LOG.warning(
"Enforcement exhausted post-update nudges, allowing response",
@@ -1589,7 +1645,7 @@ async def run_with_enforcement(
return result
ctx.post_update_nudge_count += 1
- nudge_type = _nudge_type_for_log(nudge, copilot_config)
+ nudge_type = _nudge_type_for_log(nudge, copilot_config) if nudge is not None else "synthesized_block_offer"
LOG.info("Enforcement nudge", nudge_type=nudge_type, iteration=iteration)
# OpenAI rejects images in tool messages, so a queued post-run
@@ -1599,8 +1655,7 @@ async def run_with_enforcement(
LOG.info("Injecting screenshot user message", count=len(screenshot_msg["content"]) - 1)
with copilot_span("enforcement_nudge", data={"nudge_type": nudge_type, "iteration": iteration}):
- nudge_msg = {"role": "user", "content": NUDGE_SENTINEL + nudge}
- extra_msgs = [nudge_msg] if screenshot_msg is None else [screenshot_msg, nudge_msg]
+ extra_msgs = _assemble_enforcement_messages(screenshot_msg, nudge, synthesized_msg)
current_input = (
extra_msgs if session is not None else _prune_input_list(result.to_input_list()) + extra_msgs
)
diff --git a/skyvern/forge/sdk/copilot/hooks.py b/skyvern/forge/sdk/copilot/hooks.py
index fc01f0f71..fcc6d90f1 100644
--- a/skyvern/forge/sdk/copilot/hooks.py
+++ b/skyvern/forge/sdk/copilot/hooks.py
@@ -88,6 +88,17 @@ class CopilotRunHooks(RunHooksBase):
total_tokens=getattr(self._ctx, "total_tokens_used", None),
)
+ if tool_name == "list_credentials" and parsed.get("ok"):
+ data = parsed.get("data") or {}
+ listed = data.get("credentials", []) if isinstance(data, dict) else []
+ resolved = [
+ {"credential_id": c.get("credential_id"), "name": c.get("name")}
+ for c in listed
+ if isinstance(c, dict) and isinstance(c.get("credential_id"), str)
+ ]
+ if resolved:
+ activity_entry["credentials"] = resolved
+
if tool_name in _BLOCK_OUTPUT_TOOLS and parsed.get("ok"):
data = parsed.get("data") or {}
blocks = data.get("blocks", []) if isinstance(data, dict) else []
diff --git a/skyvern/forge/sdk/copilot/output_policy.py b/skyvern/forge/sdk/copilot/output_policy.py
index 46f134469..707b2b091 100644
--- a/skyvern/forge/sdk/copilot/output_policy.py
+++ b/skyvern/forge/sdk/copilot/output_policy.py
@@ -17,6 +17,7 @@ from skyvern.forge.sdk.copilot.workflow_credential_utils import (
parse_workflow_yaml,
url_origin,
workflow_blocks,
+ workflow_credential_ids_from_parsed,
workflow_credential_origins_from_parsed,
)
@@ -481,7 +482,10 @@ def _apply_credential_policy(
approved_ids = _approved_credential_ids(request_policy)
allowed_unresolved_ids = _allowed_unresolved_credential_ids(request_policy)
- allowed_ids = approved_ids | allowed_unresolved_ids | _existing_workflow_credential_ids(request_policy)
+ bound_approved_ids = _bound_approved_credential_ids(request_policy, workflow_yaml)
+ allowed_ids = (
+ approved_ids | allowed_unresolved_ids | bound_approved_ids | _existing_workflow_credential_ids(request_policy)
+ )
if any(credential_id not in allowed_ids for credential_id in found_ids):
verdict.add(OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE)
@@ -503,6 +507,19 @@ def _approved_credential_ids(request_policy: RequestPolicy) -> set[str]:
}
+def _bound_approved_credential_ids(request_policy: RequestPolicy, workflow_yaml: str | None) -> set[str]:
+ # Requiring binding keeps a discovered ID leaked into a non-credential field
+ # caught by misbinding rather than approved here.
+ discovered = {credential.credential_id for credential in request_policy.discovered_credentials}
+ discovered = {cid for cid in discovered if cid.startswith("cred_")}
+ if not discovered or not workflow_yaml:
+ return set()
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return set()
+ return discovered & workflow_credential_ids_from_parsed(parsed)
+
+
def _allowed_unresolved_credential_ids(request_policy: RequestPolicy) -> set[str]:
if not request_policy.allow_missing_credentials_in_draft:
return set()
@@ -583,7 +600,7 @@ def _workflow_broadens_credential_scope(parsed_workflow: dict[str, Any], request
def _approved_origins_by_id(request_policy: RequestPolicy) -> dict[str, set[str]]:
origins: dict[str, set[str]] = {}
- for credential in request_policy.resolved_credentials:
+ for credential in [*request_policy.resolved_credentials, *request_policy.discovered_credentials]:
credential_id = getattr(credential, "credential_id", None)
tested_url = getattr(credential, "tested_url", None)
if isinstance(credential_id, str) and isinstance(tested_url, str):
diff --git a/skyvern/forge/sdk/copilot/request_policy.py b/skyvern/forge/sdk/copilot/request_policy.py
index 86dac24a3..aa77d5e13 100644
--- a/skyvern/forge/sdk/copilot/request_policy.py
+++ b/skyvern/forge/sdk/copilot/request_policy.py
@@ -144,6 +144,9 @@ class RequestPolicy:
completion_contract: str | None = None
completion_criteria: list[CompletionCriterion] = field(default_factory=list)
resolved_credentials: list[Credential] = field(default_factory=list)
+ # Approves persisting a bound credential, not running it: run authority stays
+ # scoped to resolved_credentials (ADR 0002).
+ discovered_credentials: list[Credential] = field(default_factory=list)
invalid_credential_ids: list[str] = field(default_factory=list)
clarification_question: str | None = None
raw_secret_detected: bool = False
@@ -777,6 +780,30 @@ def _previous_credential_clarification_was_asked(global_llm_context: str) -> boo
)
+def _discovered_credential_ids_from_context(global_llm_context: str) -> set[str]:
+ structured = StructuredContext.from_json_str(global_llm_context)
+ return {
+ check.credential_id
+ for check in structured.credentials_checked
+ if check.found and isinstance(check.credential_id, str) and check.credential_id.startswith("cred_")
+ }
+
+
+async def _seed_discovered_credentials(
+ policy: RequestPolicy,
+ *,
+ organization_id: str,
+ global_llm_context: str,
+) -> None:
+ discovered_ids = _discovered_credential_ids_from_context(global_llm_context)
+ if not discovered_ids:
+ return
+ policy.discovered_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
+ sorted(discovered_ids),
+ organization_id=organization_id,
+ )
+
+
def _last_assistant_message_was_saved_credential_question(
chat_history: list[WorkflowCopilotChatHistoryMessage],
) -> bool:
@@ -1035,6 +1062,18 @@ async def build_request_policy(
policy.existing_workflow_credential_origins = {
credential_id: sorted(origins) for credential_id, origins in workflow_credential_origins(workflow_yaml).items()
}
+ try:
+ await _seed_discovered_credentials(
+ policy,
+ organization_id=organization_id,
+ global_llm_context=global_llm_context,
+ )
+ except Exception:
+ LOG.warning(
+ "request-policy discovered-credential seeding failed",
+ organization_id=organization_id,
+ exc_info=True,
+ )
_prioritize_credential_clarification(policy)
if policy.raw_secret_detected and policy.raw_secret_handling == "redacted_draft":
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 517904895..5be680246 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -113,6 +113,9 @@ class ScoutedInteraction(TypedDict):
value: NotRequired[str]
key: NotRequired[str]
typed_length: NotRequired[int]
+ role: NotRequired[str]
+ accessible_name: NotRequired[str]
+ trajectory_index: NotRequired[int]
@dataclass
@@ -228,6 +231,12 @@ class AgentContext:
observed_browser_urls: list[str] = field(default_factory=list)
# Ephemeral within-turn scout captures; not persisted across turns.
scouted_interactions: list[ScoutedInteraction] = field(default_factory=list)
+ # Append-only, non-deduped record of the scout's interaction sequence in
+ # acted order. Unlike scouted_interactions (deduped for auto-credit), this
+ # preserves repeats and ordering so code_block_synthesis can emit a faithful
+ # linear Playwright trajectory.
+ scout_trajectory: list[ScoutedInteraction] = field(default_factory=list)
+ synthesized_block_offered: bool = False
# Source page of an in-flight scout action, captured before it may navigate away.
pending_scout_source_url: str | None = None
diff --git a/skyvern/forge/sdk/copilot/tools.py b/skyvern/forge/sdk/copilot/tools.py
index e3a05b605..14aef65fb 100644
--- a/skyvern/forge/sdk/copilot/tools.py
+++ b/skyvern/forge/sdk/copilot/tools.py
@@ -11,7 +11,7 @@ from collections import Counter, defaultdict
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
-from typing import Annotated, Any, Literal
+from typing import Annotated, Any, Literal, cast
from urllib.parse import urljoin, urlparse
import structlog
@@ -70,6 +70,9 @@ from skyvern.forge.sdk.copilot.composition_browser_expressions import (
from skyvern.forge.sdk.copilot.composition_browser_expressions import (
COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION as _COMPOSITION_VISUAL_OBSTRUCTION_CANDIDATES_EXPRESSION,
)
+from skyvern.forge.sdk.copilot.composition_browser_expressions import (
+ scout_accessible_role_name_expression as _scout_accessible_role_name_expression,
+)
from skyvern.forge.sdk.copilot.composition_evidence import (
SCOUT_INTERACTION_EVIDENCE_TOOL,
composition_page_evidence_error,
@@ -144,6 +147,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (
)
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+from skyvern.forge.sdk.schemas.credentials import Credential
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.workflow.exceptions import BaseWorkflowHTTPException
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar, get_all_blocks
@@ -1530,6 +1534,83 @@ def _consume_scout_source_url(ctx: AgentContext) -> str | None:
return source_url
+_ROLE_NAME_SELECTOR_RE = re.compile(r'^role=([a-zA-Z]+)(?:\[name="((?:[^"\\]|\\.)*)"\])?(.*)$')
+
+
+def _role_name_from_selector(selector: str) -> tuple[str, str] | None:
+ """Parse the ``role=<role>[name="<name>"]`` form (ref_to_selector) — TIER 1, no browser read.
+
+ Returns (role, accessible_name) when the selector is a plain role/name locator;
+ None for bare CSS/xpath or when an engine chain (`>> nth=`) trails the role/name.
+ """
+ selector = selector.strip()
+ match = _ROLE_NAME_SELECTOR_RE.match(selector)
+ if not match:
+ return None
+ role, raw_name, suffix = match.group(1), match.group(2), match.group(3)
+ if suffix.strip():
+ return None
+ name = raw_name.replace('\\"', '"') if raw_name is not None else ""
+ return role, name
+
+
+async def _capture_accessible_role_name(ctx: AgentContext, selector: str) -> tuple[str, str] | None:
+ """TIER 2: read the element's role/accessible name for a bare CSS/xpath selector.
+
+ A failed read degrades gracefully to None so the selector-only auto-credit
+ path (SKY-10712) stays intact.
+ """
+ selector = selector.strip()
+ if not selector:
+ return None
+ server = getattr(ctx, "discovery_mcp_server", None)
+ if server is None:
+ return None
+ try:
+ result = await asyncio.wait_for(
+ server.call_internal_tool(
+ "skyvern_evaluate",
+ {"expression": _scout_accessible_role_name_expression(selector)},
+ ),
+ timeout=_DISCOVERY_PER_CALL_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ return None
+ if not isinstance(result, dict) or not result.get("ok"):
+ return None
+ value = (result.get("data") or {}).get("result")
+ if not isinstance(value, dict):
+ return None
+ role = str(value.get("role") or "").strip()
+ name = str(value.get("accessible_name") or "").strip()
+ if not role and not name:
+ return None
+ return role, name
+
+
+async def _resolve_scout_role_name(
+ ctx: AgentContext, selector: str, *, allow_browser_read: bool = True
+) -> tuple[str, str]:
+ """Resolve (role, accessible_name) for a scouted selector. TIER 1 parse first;
+ TIER 2 browser read only for bare CSS/xpath. Always degrades to ("", "").
+
+ ``allow_browser_read=False`` skips TIER 2 when the action navigated: a post-action
+ read against the landing page would capture the wrong element's name, so the bare
+ selector is kept verbatim (the synthesizer prefers it anyway)."""
+ selector = selector.strip()
+ if not selector:
+ return "", ""
+ parsed = _role_name_from_selector(selector)
+ if parsed is not None:
+ return parsed
+ if not allow_browser_read:
+ return "", ""
+ captured = await _capture_accessible_role_name(ctx, selector)
+ if captured is not None:
+ return captured
+ return "", ""
+
+
def _record_scouted_interaction(
ctx: AgentContext,
*,
@@ -1539,6 +1620,8 @@ def _record_scouted_interaction(
value: str = "",
key: str = "",
typed_length: int = 0,
+ role: str = "",
+ accessible_name: str = "",
) -> None:
selector = selector.strip()
# press_key may be page-level, so it is recorded by key even with no selector; other tools require one.
@@ -1555,6 +1638,10 @@ def _record_scouted_interaction(
artifact["key"] = key
if typed_length:
artifact["typed_length"] = typed_length
+ if role:
+ artifact["role"] = role
+ if accessible_name:
+ artifact["accessible_name"] = accessible_name
interactions = [
item
for item in ctx.scouted_interactions
@@ -1566,12 +1653,21 @@ def _record_scouted_interaction(
]
interactions.append(artifact)
ctx.scouted_interactions = interactions[-_MAX_SCOUTED_INTERACTIONS:]
+
+ trajectory = list(ctx.scout_trajectory)
+ trajectory_artifact = cast(ScoutedInteraction, artifact.copy())
+ trajectory_artifact["trajectory_index"] = len(trajectory)
+ trajectory.append(trajectory_artifact)
+ ctx.scout_trajectory = trajectory[-_MAX_SCOUTED_INTERACTIONS:]
+
LOG.info(
"copilot_scout_interaction_captured",
tool_name=tool_name,
selector=selector or None,
source_url=artifact.get("source_url"),
+ role=role or None,
total_scouted_interactions=len(ctx.scouted_interactions),
+ total_scout_trajectory=len(ctx.scout_trajectory),
)
@@ -2420,6 +2516,15 @@ def _code_artifact_metadata_as_tool_argument(
return [item.model_dump(mode="json", exclude_none=True) for item in metadata]
+def _format_code_artifact_violations(violations: list[str]) -> str:
+ # Surface every contract violation at once so the agent fixes them in a single
+ # update instead of round-tripping one error per `update_and_run_blocks` call.
+ if len(violations) == 1:
+ return violations[0]
+ numbered = "\n".join(f"{index}. {message}" for index, message in enumerate(violations, start=1))
+ return f"Artifact metadata has {len(violations)} contract violations; fix all of them in one update:\n{numbered}"
+
+
def _normalize_code_artifact_metadata(
raw_metadata: Any,
workflow_yaml: str,
@@ -2429,6 +2534,8 @@ def _normalize_code_artifact_metadata(
items = _code_artifact_metadata_items(raw_metadata)
code_blocks = _workflow_yaml_code_blocks_by_label(workflow_yaml)
normalized: dict[str, dict[str, Any]] = {}
+ violations: list[str] = []
+ seen_labels: set[str] = set()
for raw_item in items:
try:
metadata = (
@@ -2437,22 +2544,30 @@ def _normalize_code_artifact_metadata(
else CodeArtifactMetadata.model_validate(raw_item)
)
except ValidationError as exc:
- return {}, f"Artifact metadata is malformed: {exc}"
+ violations.append(f"Artifact metadata is malformed: {exc}")
+ continue
dumped = metadata.model_dump(mode="json", exclude_none=True)
label = str(dumped.get("block_label") or "").strip()
if not label:
- return {}, "Artifact metadata requires a non-empty `block_label` for each artifact."
+ violations.append("Artifact metadata requires a non-empty `block_label` for each artifact.")
+ continue
if label not in code_blocks:
- return {}, (
+ violations.append(
f"Artifact metadata for `{label}` does not reference an existing code block label. "
"Attach metadata only to authored code blocks."
)
- if label in normalized:
- return {}, f"Artifact metadata contains duplicate entries for `{label}`."
+ continue
+ if label in seen_labels:
+ violations.append(f"Artifact metadata contains duplicate entries for `{label}`.")
+ continue
+ seen_labels.add(label)
dumped["block_label"] = label
+ item_violations: list[str] = []
artifact_id = str(dumped.get("artifact_id") or "").strip()
if artifact_id and not artifact_id.startswith("code_artifact:"):
- return {}, f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."
+ item_violations.append(
+ f"Artifact metadata for `{label}` requires `artifact_id` to start with `code_artifact:`."
+ )
dumped["artifact_id"] = artifact_id or _artifact_id_for_block_label(label)
block_id = str(
dumped.get("block_id") or code_blocks[label].get("block_id") or code_blocks[label].get("id") or ""
@@ -2461,17 +2576,20 @@ def _normalize_code_artifact_metadata(
dumped["block_id"] = block_id
declared_goal = str(dumped.get("declared_goal") or "").strip()
if not declared_goal:
- return {}, f"Artifact metadata for `{label}` requires a non-empty `declared_goal`."
+ item_violations.append(f"Artifact metadata for `{label}` requires a non-empty `declared_goal`.")
for field_name in _CODE_ARTIFACT_REQUIRED_LIST_FIELDS:
value = dumped.get(field_name)
if not isinstance(value, list) or not value:
- return {}, f"Artifact metadata for `{label}` requires non-empty `{field_name}`."
+ item_violations.append(f"Artifact metadata for `{label}` requires non-empty `{field_name}`.")
if not dumped.get("evidence_refs") and not dumped.get("observation_refs"):
- return {}, f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`."
- validation_error = _code_artifact_metadata_shape_error(label, dumped)
- if validation_error is not None:
- return {}, validation_error
+ item_violations.append(f"Artifact metadata for `{label}` requires `evidence_refs` or `observation_refs`.")
+ item_violations.extend(_code_artifact_metadata_shape_errors(label, dumped))
+ if item_violations:
+ violations.extend(item_violations)
+ continue
normalized[label] = dumped
+ if violations:
+ return {}, _format_code_artifact_violations(violations)
return normalized, None
@@ -2502,45 +2620,47 @@ def _artifact_id_for_block_label(label: str) -> str:
return f"code_artifact:{fragment}"
-def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any]) -> str | None:
+def _code_artifact_metadata_shape_errors(label: str, artifact: Mapping[str, Any]) -> list[str]:
+ """Return every shape violation for one artifact; the caller aggregates them."""
+ errors: list[str] = []
for field_name, ref_key in (("evidence_refs", "evidence_ref"), ("observation_refs", "observation_ref")):
for index, ref in enumerate(_artifact_rows(artifact.get(field_name))):
if not str(ref.get(ref_key) or "").strip():
- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`."
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `{ref_key}`.")
if not any(str(ref.get(key) or "").strip() for key in ("claim_id", "dependency_id", "criterion_id")):
- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id."
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires a scoped id.")
status = str(ref.get("status") or "").strip()
if ref.get("checkpoint_next_mode") == "advance" and status != "diagnostic_only":
- return (
+ errors.append(
f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
"`checkpoint_next_mode=advance`; it must stay `diagnostic_only`."
)
if ref.get("checkpoint_next_mode") == "stop" and status not in {"observed_not_verified", "diagnostic_only"}:
- return (
+ errors.append(
f"Artifact metadata for `{label}` `{field_name}` entry {index} has "
"`checkpoint_next_mode=stop`; it must remain `observed_not_verified` or `diagnostic_only`."
)
if status != "missing" and not str(ref.get("source_tool") or "").strip():
- return f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`."
+ errors.append(f"Artifact metadata for `{label}` `{field_name}` entry {index} requires `source_tool`.")
for index, claim in enumerate(_artifact_rows(artifact.get("claimed_outcomes"))):
claim_id = str(claim.get("id") or "").strip()
if not _artifact_string_list(claim.get("depends_on")):
- return f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`."
+ errors.append(f"Artifact metadata claim `{claim_id or index}` for `{label}` requires `depends_on`.")
claim_criteria = _artifact_string_list(claim.get("covered_criteria")) or _artifact_string_list(
claim.get("criteria_ids")
)
if not claim_criteria:
- return f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids."
+ errors.append(f"Artifact metadata claim `{claim_id}` for `{label}` requires covered criterion ids.")
claim_evidence_refs = _artifact_string_list(claim.get("evidence_refs"))
claim_observation_refs = _artifact_string_list(claim.get("observation_refs"))
if claim.get("status") == "satisfied" and not claim_evidence_refs:
- return (
+ errors.append(
f"Artifact metadata claim `{claim_id}` for `{label}` is `satisfied` but has no "
"claim-scoped `evidence_refs`."
)
if claim.get("status") != "missing" and not claim_evidence_refs and not claim_observation_refs:
- return (
+ errors.append(
f"Artifact metadata claim `{claim_id}` for `{label}` requires claim-scoped "
"`evidence_refs` or `observation_refs` unless status is `missing`."
)
@@ -2550,12 +2670,12 @@ def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any])
dependency_evidence_refs = _artifact_string_list(dependency.get("evidence_refs"))
dependency_observation_refs = _artifact_string_list(dependency.get("observation_refs"))
if dependency.get("status") == "satisfied" and not dependency_evidence_refs:
- return (
+ errors.append(
f"Artifact metadata dependency `{dependency_id}` for `{label}` is `satisfied` but has no "
"dependency-scoped `evidence_refs`."
)
if dependency.get("status") != "missing" and not dependency_evidence_refs and not dependency_observation_refs:
- return (
+ errors.append(
f"Artifact metadata dependency `{dependency_id}` for `{label}` requires scoped "
"`evidence_refs` or `observation_refs` unless status is `missing`."
)
@@ -2565,23 +2685,23 @@ def _code_artifact_metadata_shape_error(label: str, artifact: Mapping[str, Any])
if not _artifact_string_list(expectation.get("criteria_ids")) and not _artifact_string_list(
expectation.get("claimed_outcome_ids")
):
- return (
+ errors.append(
f"Artifact metadata terminal verifier expectation `{expectation_id or index}` for `{label}` "
"requires `criteria_ids` or `claimed_outcome_ids`."
)
for index, observation in enumerate(_artifact_rows(artifact.get("exploration_observations"))):
if observation.get("status") != "observed_not_verified":
- return (
+ errors.append(
f"Artifact metadata for `{label}` exploration observation {index} must be marked "
"`observed_not_verified` until authored execution and terminal verification pass."
)
if observation.get("checkpoint_next_mode") == "advance":
- return (
+ errors.append(
f"Artifact metadata for `{label}` exploration observation {index} cannot carry "
"`checkpoint_next_mode=advance`; record that as `diagnostic_only` evidence instead."
)
- return None
+ return errors
def _artifact_rows(value: Any) -> list[Mapping[str, Any]]:
@@ -2788,6 +2908,7 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[s
elif cred.secret_label:
entry["secret_label"] = cred.secret_label
serialized.append(entry)
+ _record_discovered_credentials_on_policy(ctx, credentials)
return {
"ok": True,
"data": {
@@ -2800,6 +2921,16 @@ async def _list_credentials(params: dict[str, Any], ctx: AgentContext) -> dict[s
}
+def _record_discovered_credentials_on_policy(ctx: AgentContext, credentials: list[Credential]) -> None:
+ request_policy = ctx.request_policy
+ if not isinstance(request_policy, RequestPolicy):
+ return
+ known_ids = {credential.credential_id for credential in request_policy.discovered_credentials}
+ for credential in credentials:
+ if credential.credential_id not in known_ids:
+ request_policy.discovered_credentials.append(credential)
+
+
# Block types that establish browser state (loaded page / authenticated
# session / navigation target). These are valid upstream anchors to walk back
# to when a downstream edit invalidates part of the chain.
@@ -5930,7 +6061,18 @@ async def _click_post_hook(
"url": url,
"title": title,
}
- _record_scouted_interaction(ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url)
+ navigated = bool(source_url) and bool(url) and source_url != url
+ role, accessible_name = await _resolve_scout_role_name(
+ ctx, data.get("selector", ""), allow_browser_read=not navigated
+ )
+ _record_scouted_interaction(
+ ctx,
+ tool_name="click",
+ selector=data.get("selector", ""),
+ source_url=source_url,
+ role=role,
+ accessible_name=accessible_name,
+ )
observation_step = _register_scout_interaction_observation(
ctx, tool_name="click", selector=data.get("selector", ""), source_url=source_url, url=url
)
@@ -6020,8 +6162,15 @@ async def _type_text_post_hook(
if landing_failure is not None:
return landing_failure
_mark_pending_browser_interaction_observation(ctx, tool_name="type_text", url=url)
+ role, accessible_name = await _resolve_scout_role_name(ctx, selector)
_record_scouted_interaction(
- ctx, tool_name="type_text", selector=selector, source_url=source_url, typed_length=typed_length
+ ctx,
+ tool_name="type_text",
+ selector=selector,
+ source_url=source_url,
+ typed_length=typed_length,
+ role=role,
+ accessible_name=accessible_name,
)
observation_step = _register_scout_interaction_observation(
ctx, tool_name="type_text", selector=selector, source_url=source_url, url=url
@@ -6105,12 +6254,15 @@ async def _select_option_post_hook(
"value": data.get("value", ""),
"url": url,
}
+ role, accessible_name = await _resolve_scout_role_name(ctx, data.get("selector", ""))
_record_scouted_interaction(
ctx,
tool_name="select_option",
selector=data.get("selector", ""),
source_url=source_url,
value=data.get("value", ""),
+ role=role,
+ accessible_name=accessible_name,
)
observation_step = _register_scout_interaction_observation(
ctx, tool_name="select_option", selector=data.get("selector", ""), source_url=source_url, url=url
diff --git a/tests/unit/test_copilot_code_artifact_metadata_violations.py b/tests/unit/test_copilot_code_artifact_metadata_violations.py
new file mode 100644
index 000000000..10892f6f8
--- /dev/null
+++ b/tests/unit/test_copilot_code_artifact_metadata_violations.py
@@ -0,0 +1,165 @@
+"""Tests for the code-artifact-metadata validator returning every violation at once.
+
+OSS-synced: only example.* / RFC-2606 placeholder targets and synthetic labels.
+"""
+
+from __future__ import annotations
+
+import re
+import textwrap
+
+from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
+
+
+def _code_block_yaml(label: str) -> str:
+ return textwrap.dedent(
+ f"""
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: {label}
+ code: |
+ await page.goto("https://example.com/")
+ """
+ ).strip()
+
+
+def _violation_count(error: str) -> int:
+ return len(re.findall(r"^\d+\.", error, flags=re.M))
+
+
+def _broken_metadata(label: str) -> dict:
+ return {
+ "block_label": label,
+ "artifact_id": "not-prefixed",
+ "declared_goal": "do the thing",
+ "claimed_outcomes": [{"id": "claim:x", "scope": "outcome", "text": "x", "status": "satisfied"}],
+ "page_dependencies": [{"id": "dependency:p", "scope": "page", "status": "satisfied"}],
+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
+ "terminal_verifier_expectations": [{"id": "exp", "text": "e"}],
+ "observation_refs": [{"observation_ref": "obs1", "status": "satisfied", "checkpoint_next_mode": "advance"}],
+ }
+
+
+class TestAccumulateAllViolations:
+ def test_every_violation_returned_at_once(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_broken_metadata("my_block")], _code_block_yaml("my_block")
+ )
+ assert normalized == {}
+ assert error is not None
+ # The 5+ sequential failures from the repair loop now surface together.
+ assert _violation_count(error) >= 5
+ assert error.startswith("Artifact metadata has ")
+ assert "fix all of them in one update" in error
+ assert "`artifact_id` to start with `code_artifact:`" in error
+ assert "requires `source_tool`" in error
+ assert "requires `depends_on`" in error
+ assert "is `satisfied` but has no" in error
+
+ def test_single_violation_is_not_numbered(self) -> None:
+ metadata = {
+ "block_label": "my_block",
+ "artifact_id": "not-prefixed",
+ "declared_goal": "g",
+ "claimed_outcomes": [
+ {
+ "id": "claim:x",
+ "scope": "outcome",
+ "text": "x",
+ "status": "observed_not_verified",
+ "depends_on": ["dependency:p"],
+ "covered_criteria": ["criterion:c"],
+ "observation_refs": ["obs1"],
+ }
+ ],
+ "page_dependencies": [
+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
+ ],
+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
+ "observation_refs": [
+ {
+ "observation_ref": "obs1",
+ "dependency_id": "dependency:p",
+ "status": "observed_not_verified",
+ "source_tool": "scout_interaction",
+ }
+ ],
+ }
+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
+ assert normalized == {}
+ assert error is not None
+ assert _violation_count(error) == 0
+ assert error == "Artifact metadata for `my_block` requires `artifact_id` to start with `code_artifact:`."
+
+ def test_violations_aggregate_across_multiple_artifacts(self) -> None:
+ yaml = textwrap.dedent(
+ """
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: block_one
+ code: |
+ await page.goto("https://example.com/")
+ - block_type: code
+ label: block_two
+ code: |
+ await page.goto("https://example.com/")
+ """
+ ).strip()
+ normalized, error = _normalize_code_artifact_metadata(
+ [_broken_metadata("block_one"), _broken_metadata("block_two")], yaml
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "block_one" in error
+ assert "block_two" in error
+
+ def test_unknown_label_does_not_short_circuit_other_artifacts(self) -> None:
+ normalized, error = _normalize_code_artifact_metadata(
+ [_broken_metadata("ghost_label"), _broken_metadata("my_block")], _code_block_yaml("my_block")
+ )
+ assert normalized == {}
+ assert error is not None
+ assert "does not reference an existing code block label" in error
+ # The second artifact's shape violations are still surfaced.
+ assert "requires `source_tool`" in error
+
+ def test_valid_metadata_passes(self) -> None:
+ metadata = {
+ "block_label": "my_block",
+ "artifact_id": "code_artifact:my_block",
+ "declared_goal": "g",
+ "claimed_outcomes": [
+ {
+ "id": "claim:x",
+ "scope": "outcome",
+ "text": "x",
+ "status": "observed_not_verified",
+ "depends_on": ["dependency:p"],
+ "covered_criteria": ["criterion:c"],
+ "observation_refs": ["obs1"],
+ }
+ ],
+ "page_dependencies": [
+ {"id": "dependency:p", "scope": "page", "status": "observed_not_verified", "observation_refs": ["obs1"]}
+ ],
+ "completion_criteria": [{"id": "criterion:c", "text": "c", "level": "terminal"}],
+ "terminal_verifier_expectations": [{"id": "exp", "text": "e", "criteria_ids": ["criterion:c"]}],
+ "observation_refs": [
+ {
+ "observation_ref": "obs1",
+ "dependency_id": "dependency:p",
+ "status": "observed_not_verified",
+ "source_tool": "scout_interaction",
+ }
+ ],
+ }
+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml("my_block"))
+ assert error is None
+ assert list(normalized.keys()) == ["my_block"]
+
+ def test_empty_metadata_is_noop(self) -> None:
+ assert _normalize_code_artifact_metadata(None, _code_block_yaml("my_block")) == ({}, None)
+ assert _normalize_code_artifact_metadata([], _code_block_yaml("my_block")) == ({}, None)
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
new file mode 100644
index 000000000..bfecc8e63
--- /dev/null
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -0,0 +1,647 @@
+"""Tests for the pure copilot code-block synthesizer.
+
+OSS-synced: only example.* / RFC-2606 placeholder targets.
+"""
+
+from __future__ import annotations
+
+import ast
+import keyword
+from typing import Any
+
+from skyvern.forge.sdk.copilot.code_block_preflight import preflight_code_block
+from skyvern.forge.sdk.copilot.code_block_synthesis import (
+ _SYNTHESIZED_BLOCK_LABEL,
+ build_synthesized_artifact_metadata,
+ render_synthesized_offer_text,
+ synthesize_code_block,
+)
+from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
+
+
+def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:
+ return {"tool_name": tool_name, **fields}
+
+
+class TestLocatorSynthesis:
+ def test_role_selector_emits_get_by_role(self) -> None:
+ # A `role=...` selector is an ARIA anchor (ref_to_selector form), not a native CSS path.
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='role=button[name="Add to cart"]',
+ source_url="https://example.com/product",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("button", name="Add to cart").click()' in result.code
+ assert "get_by_role" in result.code
+
+ def test_stable_id_selector_is_emitted_verbatim_not_get_by_role(self) -> None:
+ # Selector-first: a captured stable selector (id) wins over a get_by_role anchor, because the
+ # scout's a11y-name read may not reproduce on the raw page the code block runs against.
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector="#search-box",
+ source_url="https://example.com/",
+ typed_length=11,
+ role="textbox",
+ accessible_name="Search",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.locator("#search-box").fill(str(search))' in result.code
+ assert "get_by_role" not in result.code
+
+ def test_stable_attribute_selector_is_emitted_verbatim(self) -> None:
+ # [name=...] / [data-testid=...] etc. are stable identity anchors — kept verbatim.
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='[data-testid="add-to-cart"]',
+ source_url="https://example.com/product",
+ role="button",
+ accessible_name="Add to cart",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.locator("[data-testid=\\"add-to-cart\\"]").click()' in result.code
+ assert "get_by_role" not in result.code
+
+ def test_positional_nth_of_type_selector_uses_get_by_role_fallback(self) -> None:
+ # A positional CSS path is fragile; when a role/name anchor exists, prefer it.
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="div.list > button:nth-of-type(3)",
+ source_url="https://example.com/list",
+ role="button",
+ accessible_name="More",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("button", name="More").click()' in result.code
+
+ def test_nth_engine_chain_uses_get_by_role_anchor(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='role=button[name="More"] >> nth=2',
+ source_url="https://example.com/list",
+ role="button",
+ accessible_name="More",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("button", name="More").click()' in result.code
+
+ def test_positional_selector_without_role_name_keeps_selector_with_note(self) -> None:
+ result = synthesize_code_block(
+ [_interaction("click", selector="ul > li:nth-child(2)", source_url="https://example.com/results")]
+ )
+ assert result is not None
+ assert 'await page.locator("ul > li:nth-child(2)").click()' in result.code
+ assert any("low-confidence" in note for note in result.notes)
+
+ def test_stable_bare_css_without_role_name_is_emitted_verbatim_no_note(self) -> None:
+ result = synthesize_code_block(
+ [_interaction("click", selector=".results .item", source_url="https://example.com/results")]
+ )
+ assert result is not None
+ assert 'await page.locator(".results .item").click()' in result.code
+ assert not any("low-confidence" in note for note in result.notes)
+
+ def test_never_emits_first_or_last_callables(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='role=link[name="Next"] >> nth=0',
+ source_url="https://example.com/",
+ )
+ ]
+ )
+ assert result is not None
+ assert ".first" not in result.code
+ assert ".last" not in result.code
+
+
+class TestActionSynthesis:
+ def test_type_text_becomes_param_slot_fill(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Search"]',
+ source_url="https://example.com/",
+ typed_length=11,
+ role="textbox",
+ accessible_name="Search",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("textbox", name="Search").fill(str(search))' in result.code
+ assert result.parameters == [{"key": "search"}]
+ # Raw typed value is never captured.
+ assert "value" not in result.code
+
+ def test_goto_entry_url_and_load_state(self) -> None:
+ result = synthesize_code_block([_interaction("click", selector="#go", source_url="https://example.com/start")])
+ assert result is not None
+ lines = result.code.splitlines()
+ assert lines[0] == ' await page.goto("https://example.com/start")'
+ assert lines[1] == ' await page.wait_for_load_state("load")'
+
+ def test_press_enter_uses_keyboard_when_no_selector(self) -> None:
+ result = synthesize_code_block([_interaction("press_key", key="Enter")])
+ assert result is not None
+ assert 'await page.keyboard.press("Enter")' in result.code
+
+ def test_press_key_on_located_element(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "press_key",
+ selector='role=textbox[name="Search"]',
+ key="Enter",
+ role="textbox",
+ accessible_name="Search",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("textbox", name="Search").press("Enter")' in result.code
+
+ def test_select_option_emits_value(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "select_option",
+ selector='role=combobox[name="Size"]',
+ source_url="https://example.com/",
+ value="large",
+ role="combobox",
+ accessible_name="Size",
+ )
+ ]
+ )
+ assert result is not None
+ assert 'await page.get_by_role("combobox", name="Size").select_option("large")' in result.code
+
+ def test_select_option_without_value_is_dropped_with_note(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "select_option",
+ selector='role=combobox[name="Size"]',
+ source_url="https://example.com/",
+ role="combobox",
+ accessible_name="Size",
+ )
+ ]
+ )
+ assert result is None or "select_option" not in result.code
+ if result is not None:
+ assert any("select_option" in note for note in result.notes)
+
+
+class TestParamKeySafety:
+ @staticmethod
+ def _emitted_wrapper(code: str, param_keys: list[str]) -> str:
+ # Mirror block.py generate_async_user_function: the param keys become the wrapper signature.
+ signature = ", ".join(f"{key}=None" for key in param_keys)
+ body = "\n".join(f" {line.strip()}" for line in code.splitlines())
+ return f"async def wrapper({signature}):\n{body or ' pass'}"
+
+ def test_keyword_accessible_name_yields_bindable_identifier(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Class"]',
+ source_url="https://example.com/",
+ typed_length=4,
+ role="textbox",
+ accessible_name="Class",
+ )
+ ]
+ )
+ assert result is not None
+ keys = [p["key"] for p in result.parameters]
+ assert keys == ["class_field"]
+ assert all(key.isidentifier() and not keyword.iskeyword(key) for key in keys)
+ ast.parse(self._emitted_wrapper(result.code, keys))
+
+ def test_reserved_safe_var_name_is_suffixed(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Page"]',
+ source_url="https://example.com/",
+ typed_length=4,
+ role="textbox",
+ accessible_name="Page",
+ )
+ ]
+ )
+ assert result is not None
+ assert result.parameters == [{"key": "page_field"}]
+ assert "fill(str(page))" not in result.code
+ assert "fill(str(page_field))" in result.code
+
+ def test_leading_digit_name_is_valid_identifier(self) -> None:
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="2nd line"]',
+ source_url="https://example.com/",
+ typed_length=4,
+ role="textbox",
+ accessible_name="2nd line",
+ )
+ ]
+ )
+ assert result is not None
+ keys = [p["key"] for p in result.parameters]
+ assert keys and all(key.isidentifier() for key in keys)
+ ast.parse(self._emitted_wrapper(result.code, keys))
+
+
+class TestTrajectoryFidelity:
+ def test_two_same_selector_clicks_both_emitted(self) -> None:
+ # Regression: scout_trajectory is append-only/non-deduped, so a repeated
+ # click on the same selector must produce two clicks (the deduped list would drop one).
+ trajectory = [
+ _interaction(
+ "click",
+ selector='role=button[name="Add to cart"]',
+ source_url="https://example.com/p",
+ role="button",
+ accessible_name="Add to cart",
+ ),
+ _interaction(
+ "click",
+ selector='role=button[name="Add to cart"]',
+ source_url="https://example.com/p",
+ role="button",
+ accessible_name="Add to cart",
+ ),
+ ]
+ result = synthesize_code_block(trajectory)
+ assert result is not None
+ assert result.code.count('await page.get_by_role("button", name="Add to cart").click()') == 2
+
+ def test_same_name_disambiguation_keeps_distinct_param_keys(self) -> None:
+ trajectory = [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Name"]',
+ source_url="https://example.com/",
+ typed_length=3,
+ role="textbox",
+ accessible_name="Name",
+ ),
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Name"]',
+ source_url="https://example.com/",
+ typed_length=3,
+ role="textbox",
+ accessible_name="Name",
+ ),
+ ]
+ result = synthesize_code_block(trajectory)
+ assert result is not None
+ assert result.parameters == [{"key": "name"}, {"key": "name_2"}]
+ assert "fill(str(name))" in result.code
+ assert "fill(str(name_2))" in result.code
+
+ def test_param_keys_are_globally_unique_against_external_slug_collision(self) -> None:
+ # An externally-derived slug "name 2" produces base "name_2", which must not collide with the
+ # auto-suffix of two "name" fields. All keys are tracked in one global used-set.
+ trajectory = [
+ _interaction(
+ "type_text",
+ selector="#a",
+ source_url="https://example.com/",
+ typed_length=3,
+ role="textbox",
+ accessible_name="Name",
+ ),
+ _interaction(
+ "type_text",
+ selector="#b",
+ source_url="https://example.com/",
+ typed_length=3,
+ role="textbox",
+ accessible_name="name 2",
+ ),
+ _interaction(
+ "type_text",
+ selector="#c",
+ source_url="https://example.com/",
+ typed_length=3,
+ role="textbox",
+ accessible_name="Name",
+ ),
+ ]
+ result = synthesize_code_block(trajectory)
+ assert result is not None
+ keys = [p["key"] for p in result.parameters]
+ assert keys == ["name", "name_2", "name_3"]
+ assert len(set(keys)) == len(keys)
+ for key in keys:
+ assert f"fill(str({key}))" in result.code
+
+ def test_step_cap_truncates_at_twenty(self) -> None:
+ trajectory = [
+ _interaction("click", selector=f'role=button[name="b{i}"]', source_url="https://example.com/")
+ for i in range(25)
+ ]
+ result = synthesize_code_block(trajectory)
+ assert result is not None
+ assert result.code.count(".click()") == 20
+ assert any("truncated" in note for note in result.notes)
+
+
+class TestDeterminismAndEmpty:
+ def test_empty_trajectory_returns_none(self) -> None:
+ assert synthesize_code_block([]) is None
+
+ def test_byte_identical_per_trajectory(self) -> None:
+ trajectory = [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Search"]',
+ source_url="https://example.com/",
+ typed_length=4,
+ role="textbox",
+ accessible_name="Search",
+ ),
+ _interaction("press_key", key="Enter"),
+ _interaction(
+ "click",
+ selector='role=button[name="Go"]',
+ role="button",
+ accessible_name="Go",
+ ),
+ ]
+ first = synthesize_code_block(trajectory)
+ second = synthesize_code_block(trajectory)
+ assert first is not None and second is not None
+ assert first.code == second.code
+ assert first.parameters == second.parameters
+ assert first.notes == second.notes
+
+
+class TestLineBoundaryEscaping:
+ # str.splitlines() and several parsers treat each of these as a line boundary. An attacker-controlled
+ # page can plant one in an accessible name or option value; left unescaped it splits the emitted
+ # one-line literal across lines and corrupts the block (availability, not RCE — the leading quote
+ # precedes the payload and every attacker quote is escaped).
+ _BOUNDARY_CODEPOINTS = ("\x0b", "\x0c", "\x85", "
", "
")
+
+ @staticmethod
+ def _parses(code: str) -> ast.Module:
+ wrapper = "async def __wrapper__(payload=None):\n" + "\n".join(
+ f" {line.strip()}" for line in code.splitlines()
+ )
+ return ast.parse(wrapper)
+
+ def test_accessible_name_boundary_codepoints_keep_block_parseable(self) -> None:
+ for codepoint in self._BOUNDARY_CODEPOINTS:
+ name = f"Search{codepoint}payload"
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector=f'role=button[name="{name}"]',
+ source_url="https://example.com/",
+ role="button",
+ accessible_name=name,
+ )
+ ]
+ )
+ assert result is not None, f"codepoint U+{ord(codepoint):04X} produced no block"
+ # The block parses with no SyntaxError despite the raw line boundary in the name.
+ self._parses(result.code)
+ # The raw codepoint never reaches the emitted source; it survives only as a backslash escape,
+ # so the payload stays inert inside the single-line literal.
+ assert codepoint not in result.code, f"raw U+{ord(codepoint):04X} leaked into emitted code"
+ assert "payload" in result.code
+
+ def test_select_option_value_boundary_codepoint_keeps_block_parseable(self) -> None:
+ value = "small\x0bvalue"
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "select_option",
+ selector='role=combobox[name="Size"]',
+ source_url="https://example.com/",
+ value=value,
+ role="combobox",
+ accessible_name="Size",
+ )
+ ]
+ )
+ assert result is not None
+ self._parses(result.code)
+ assert "\x0b" not in result.code
+ assert "select_option" in result.code
+
+ def test_c0_control_codepoint_in_name_is_escaped(self) -> None:
+ # Belt-and-suspenders: the defensive C0/C1 control pass also escapes non-line-boundary controls.
+ name = "Search\x07bell"
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector=f'role=button[name="{name}"]',
+ source_url="https://example.com/",
+ role="button",
+ accessible_name=name,
+ )
+ ]
+ )
+ assert result is not None
+ self._parses(result.code)
+ assert "\x07" not in result.code
+
+
+class TestPreflightSurfacesSyntaxError:
+ def test_synthesized_block_round_trips_through_preflight(self) -> None:
+ # A well-formed synthesized block (boundary codepoint escaped) yields no syntax diagnostic.
+ result = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='role=button[name="Go\x0bx"]',
+ source_url="https://example.com/",
+ role="button",
+ accessible_name="Go\x0bx",
+ )
+ ]
+ )
+ assert result is not None
+ diagnostics = preflight_code_block(result.code, parameter_keys=())
+ assert not any(d.code == "SYNTAX_ERROR" for d in diagnostics)
+
+ def test_unparseable_block_surfaces_syntax_diagnostic(self) -> None:
+ # A malformed block must be caught at authoring time, not swallowed into a silent run-time failure.
+ diagnostics = preflight_code_block('await page.goto("unterminated)\n', parameter_keys=())
+ assert any(d.code == "SYNTAX_ERROR" for d in diagnostics)
+
+
+class TestRenderSynthesizedOfferText:
+ def test_renders_label_code_and_params(self) -> None:
+ synthesized = synthesize_code_block(
+ [
+ _interaction(
+ "type_text",
+ selector="#searcher_s",
+ source_url="https://example.com/",
+ typed_length=5,
+ role="textbox",
+ accessible_name="Search",
+ )
+ ]
+ )
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized)
+ assert text.startswith("SYNTHESIZED CODE BLOCK (offered once).")
+ assert _SYNTHESIZED_BLOCK_LABEL in text
+ assert "```python" in text
+ assert 'await page.locator("#searcher_s").fill(str(search))' in text
+ assert "Workflow parameters referenced (bind these): search." in text
+
+ def test_omits_param_line_when_no_parameters(self) -> None:
+ synthesized = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector='role=button[name="Go"]',
+ source_url="https://example.com/",
+ role="button",
+ accessible_name="Go",
+ )
+ ]
+ )
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized)
+ assert "Workflow parameters referenced" not in text
+
+ def test_includes_synthesis_notes_when_present(self) -> None:
+ synthesized = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="li:nth-of-type(3) > a",
+ source_url="https://example.com/",
+ )
+ ]
+ )
+ assert synthesized is not None
+ assert synthesized.notes
+ text = render_synthesized_offer_text(synthesized)
+ assert "Synthesis notes: " in text
+
+
+def _code_block_yaml(label: str) -> str:
+ return (
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ f" label: {label}\n"
+ " code: |\n"
+ ' await page.goto("https://example.com/")\n'
+ )
+
+
+_SCOUT_TRAJECTORY = [
+ {
+ "tool_name": "type_text",
+ "selector": "#search-box",
+ "source_url": "https://example.com/",
+ "typed_length": 5,
+ "role": "textbox",
+ "accessible_name": "Search",
+ },
+ {"tool_name": "press_key", "selector": "#search-box", "key": "Enter"},
+ {
+ "tool_name": "click",
+ "selector": 'role=button[name="Add to cart"]',
+ "role": "button",
+ "accessible_name": "Add to cart",
+ },
+]
+
+
+class TestSynthesizedArtifactMetadata:
+ def test_skeleton_passes_the_validator_with_placeholders(self) -> None:
+ # The skeleton passes the validator with only <fill> placeholders for the model-owned slots.
+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ normalized, error = _normalize_code_artifact_metadata([metadata], _code_block_yaml(_SYNTHESIZED_BLOCK_LABEL))
+ assert error is None
+ assert list(normalized.keys()) == [_SYNTHESIZED_BLOCK_LABEL]
+
+ def test_skeleton_never_asserts_satisfied_status(self) -> None:
+ # The scout never ran+verified the authored block, so the only honest status is observed_not_verified.
+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ statuses = [metadata["page_dependencies"][0]["status"], metadata["observation_refs"][0]["status"]]
+ statuses += [claim["status"] for claim in metadata["claimed_outcomes"]]
+ assert all(status == "observed_not_verified" for status in statuses)
+ assert "satisfied" not in str(metadata)
+
+ def test_skeleton_observation_ref_carries_scout_source_tool(self) -> None:
+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ observation_ref = metadata["observation_refs"][0]
+ assert observation_ref["source_tool"] == "scout_interaction"
+ assert observation_ref["dependency_id"] == metadata["page_dependencies"][0]["id"]
+
+ def test_skeleton_leaves_terminal_goal_for_the_model(self) -> None:
+ metadata = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ assert metadata["declared_goal"].startswith("<fill:")
+ assert metadata["completion_criteria"][0]["text"].startswith("<fill:")
+ assert metadata["claimed_outcomes"][0]["text"].startswith("<fill:")
+
+ def test_skeleton_is_byte_identical_per_trajectory(self) -> None:
+ import json
+
+ first = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ second = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
+ assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)
+
+ def test_skeleton_omits_url_hint_when_no_source_url(self) -> None:
+ metadata = build_synthesized_artifact_metadata([_interaction("press_key", key="Enter")])
+ assert "url_hint" not in metadata["page_dependencies"][0]
+ assert "current_url" not in metadata["observation_refs"][0]
+
+ def test_offer_text_embeds_metadata_when_trajectory_supplied(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY)
+ assert "code_artifact_metadata" in text
+ assert "```json" in text
+ assert "scout_interaction" in text
+ assert "returns every remaining violation at once" in text
+
+ def test_offer_text_omits_metadata_without_trajectory(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized)
+ assert "code_artifact_metadata" not in text
+ assert "```json" not in text
diff --git a/tests/unit/test_copilot_context.py b/tests/unit/test_copilot_context.py
index 75a79d102..de91cd093 100644
--- a/tests/unit/test_copilot_context.py
+++ b/tests/unit/test_copilot_context.py
@@ -34,6 +34,52 @@ def test_merge_turn_summary_caps_credentials_checked() -> None:
assert len(ctx.credentials_checked) == 40
+def test_merge_turn_summary_records_resolved_credential_ids() -> None:
+ ctx = StructuredContext()
+ activity = [
+ {
+ "tool": "list_credentials",
+ "summary": "Found 2 credential(s)",
+ "credentials": [
+ {"credential_id": "cred_amazon", "name": "Amazon"},
+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},
+ ],
+ }
+ ]
+ ctx.merge_turn_summary(activity)
+
+ by_id = {check.credential_id: check for check in ctx.credentials_checked}
+ assert set(by_id) == {"cred_amazon", "cred_quicken"}
+ assert all(check.found for check in ctx.credentials_checked)
+ assert by_id["cred_amazon"].credential_name == "Amazon"
+
+
+def test_resolved_credential_ids_survive_context_roundtrip() -> None:
+ ctx = StructuredContext()
+ ctx.merge_turn_summary(
+ [
+ {
+ "tool": "list_credentials",
+ "summary": "Found 1 credential(s)",
+ "credentials": [{"credential_id": "cred_amazon", "name": "Amazon"}],
+ }
+ ]
+ )
+
+ rehydrated = StructuredContext.from_json_str(ctx.to_json_str())
+
+ assert [check.credential_id for check in rehydrated.credentials_checked] == ["cred_amazon"]
+
+
+def test_merge_turn_summary_falls_back_to_summary_without_structured_credentials() -> None:
+ ctx = StructuredContext()
+ ctx.merge_turn_summary([{"tool": "list_credentials", "summary": "Found 0 credential(s)"}])
+
+ assert len(ctx.credentials_checked) == 1
+ assert ctx.credentials_checked[0].credential_id is None
+ assert ctx.credentials_checked[0].found is False
+
+
def test_merge_observed_acted_pages_uses_nested_evidence_url() -> None:
pages = _merge_observed_acted_pages(
[ObservedPage(url="https://example.com/old", had_bounded_schema=True, reached_via="navigate")],
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index a3fff1add..168f3ca73 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -117,6 +117,43 @@ async def test_on_tool_end_failed_whitelisted_tool_skips_preview() -> None:
assert "output_preview" not in entry
+async def test_on_tool_end_list_credentials_records_resolved_ids() -> None:
+ ctx = _FakeContext()
+ hooks = CopilotRunHooks(ctx)
+
+ output = _mcp_text_output(
+ {
+ "ok": True,
+ "data": {
+ "credentials": [
+ {"credential_id": "cred_amazon", "name": "Amazon", "username": "[email protected]"},
+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},
+ ],
+ "count": 2,
+ },
+ }
+ )
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("list_credentials"), output)
+
+ entry = ctx.tool_activity[0]
+ assert entry["credentials"] == [
+ {"credential_id": "cred_amazon", "name": "Amazon"},
+ {"credential_id": "cred_quicken", "name": "Quicken Classic"},
+ ]
+
+
+async def test_on_tool_end_list_credentials_empty_skips_field() -> None:
+ ctx = _FakeContext()
+ hooks = CopilotRunHooks(ctx)
+
+ output = _mcp_text_output({"ok": True, "data": {"credentials": [], "count": 0}})
+ await hooks.on_tool_end(_UNUSED, _UNUSED, _fake_tool("list_credentials"), output)
+
+ assert "credentials" not in ctx.tool_activity[0]
+
+
@pytest.mark.asyncio
async def test_on_tool_end_swallows_unserializable_output() -> None:
# json.dumps(default=str) can still raise if str() on the value raises --
@@ -725,6 +762,7 @@ class TestBrowserInteractionObservationHooks:
pending_browser_interaction_observation=None,
discovery_mcp_server=None,
scouted_interactions=[],
+ scout_trajectory=[],
pending_scout_source_url=None,
)
result = await _click_post_hook(
@@ -754,6 +792,7 @@ class TestBrowserInteractionObservationHooks:
),
discovery_mcp_server=None,
scouted_interactions=[],
+ scout_trajectory=[],
pending_scout_source_url=None,
)
@@ -781,6 +820,7 @@ class TestBrowserInteractionObservationHooks:
pending_browser_interaction_observation=None,
discovery_mcp_server=None,
scouted_interactions=[],
+ scout_trajectory=[],
pending_scout_source_url=None,
)
@@ -803,6 +843,7 @@ class TestScoutedInteractionCapture:
pending_browser_interaction_observation=None,
discovery_mcp_server=None,
scouted_interactions=[],
+ scout_trajectory=[],
observed_browser_urls=[],
pending_scout_source_url=source_url,
)
@@ -1047,3 +1088,51 @@ class TestScoutedInteractionCapture:
assert entry["reached_via"] == "interaction"
assert entry["evidence"]["interaction_selector"] == "#sort"
assert result["data"]["observation_step"] == entry["step"]
+
+
+class TestAssembleEnforcementMessages:
+ @staticmethod
+ def _screenshot_msg() -> dict[str, Any]:
+ return {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "screenshot"},
+ {"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
+ ],
+ }
+
+ @staticmethod
+ def _offer_msg() -> dict[str, Any]:
+ return {"role": "user", "content": "Here is a code block you can add."}
+
+ def test_screenshot_nudge_and_offer_ordering(self) -> None:
+ from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, _assemble_enforcement_messages
+
+ screenshot_msg = self._screenshot_msg()
+ offer_msg = self._offer_msg()
+ msgs = _assemble_enforcement_messages(screenshot_msg, "please finish the workflow", offer_msg)
+
+ screenshot_indices = [i for i, m in enumerate(msgs) if m is screenshot_msg]
+ assert screenshot_indices == [msgs.index(screenshot_msg)]
+ assert len(screenshot_indices) == 1
+
+ nudge_index = next(
+ i
+ for i, m in enumerate(msgs)
+ if isinstance(m.get("content"), str) and m["content"].startswith(NUDGE_SENTINEL)
+ )
+ assert nudge_index == len(msgs) - 1
+
+ offer_index = msgs.index(offer_msg)
+ assert offer_index < nudge_index
+
+ def test_offer_and_screenshot_without_nudge(self) -> None:
+ from skyvern.forge.sdk.copilot.enforcement import NUDGE_SENTINEL, _assemble_enforcement_messages
+
+ screenshot_msg = self._screenshot_msg()
+ offer_msg = self._offer_msg()
+ msgs = _assemble_enforcement_messages(screenshot_msg, None, offer_msg)
+
+ assert offer_msg in msgs
+ assert msgs.count(screenshot_msg) == 1
+ assert not any(isinstance(m.get("content"), str) and m["content"].startswith(NUDGE_SENTINEL) for m in msgs)
diff --git a/tests/unit/test_copilot_output_policy.py b/tests/unit/test_copilot_output_policy.py
index 18dbf96cb..909f7b8b0 100644
--- a/tests/unit/test_copilot_output_policy.py
+++ b/tests/unit/test_copilot_output_policy.py
@@ -1777,3 +1777,136 @@ def test_translate_to_agent_result_adds_unvalidated_affordance() -> None:
assert agent_result.updated_workflow is workflow
assert "Accept to save" in agent_result.user_response
assert "Reject to discard" in agent_result.user_response
+
+
+def _two_credential_workflow_yaml() -> str:
+ return """
+workflow_definition:
+ parameters:
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: site_a_credentials
+ default_value: cred_amazon
+ - parameter_type: workflow
+ workflow_parameter_type: credential_id
+ key: site_b_credentials
+ default_value: cred_quicken
+ blocks:
+ - block_type: login
+ label: login_a
+ url: https://login-a.authenticationtest.test/login
+ navigation_goal: Log in to site A.
+ parameter_keys:
+ - site_a_credentials
+ - block_type: login
+ label: login_b
+ url: https://login-b.authenticationtest.test/login
+ navigation_goal: Log in to site B.
+ parameter_keys:
+ - site_b_credentials
+"""
+
+
+def _discovered(credential_id: str, tested_url: str | None) -> object:
+ return SimpleNamespace(credential_id=credential_id, name=credential_id, tested_url=tested_url)
+
+
+def test_allows_discovered_bound_credential_with_no_resolved_credentials() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_workflow_yaml(navigation_goal="Log in."),
+ )
+
+ assert verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes
+
+
+def test_allows_two_discovered_bound_credentials_in_one_save() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[
+ _discovered("cred_amazon", "https://login-a.authenticationtest.test/login"),
+ _discovered("cred_quicken", "https://login-b.authenticationtest.test/login"),
+ ],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_two_credential_workflow_yaml(),
+ )
+
+ assert verdict.allowed
+
+
+def test_allows_resolved_and_discovered_credentials_together_in_one_save() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[_credential("cred_amazon", "https://login-a.authenticationtest.test/login")],
+ discovered_credentials=[_discovered("cred_quicken", "https://login-b.authenticationtest.test/login")],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_two_credential_workflow_yaml(),
+ )
+
+ assert verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE not in verdict.reason_codes
+
+
+def test_rejects_when_only_one_of_two_bound_credentials_is_discovered() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[_discovered("cred_amazon", "https://login-a.authenticationtest.test/login")],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_two_credential_workflow_yaml(),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes
+
+
+def test_rejects_discovered_credential_id_that_is_not_bound_in_workflow() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],
+ credential_input_kind="none",
+ ),
+ user_response="I used cred_safe for the login.",
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes
+
+
+def test_rejects_fabricated_credential_id_not_in_discovered_set() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[_discovered("cred_amazon", "https://login.example.test/login")],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_workflow_yaml(navigation_goal="Log in."),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.UNAPPROVED_CREDENTIAL_REFERENCE in verdict.reason_codes
+
+
+def test_rejects_discovered_credential_bound_to_new_origin() -> None:
+ verdict = evaluate_output_policy(
+ request_policy=_policy(
+ resolved_credentials=[],
+ discovered_credentials=[_discovered("cred_safe", "https://login.example.test/login")],
+ credential_input_kind="none",
+ ),
+ workflow_yaml=_workflow_yaml(url="https://evil.example.test/login"),
+ )
+
+ assert not verdict.allowed
+ assert OutputPolicyReason.CREDENTIAL_SCOPE_BROADENED in verdict.reason_codes
diff --git a/tests/unit/test_copilot_request_policy_workflow_credentials.py b/tests/unit/test_copilot_request_policy_workflow_credentials.py
index 39b7a7515..2c90badc1 100644
--- a/tests/unit/test_copilot_request_policy_workflow_credentials.py
+++ b/tests/unit/test_copilot_request_policy_workflow_credentials.py
@@ -1,8 +1,13 @@
from __future__ import annotations
import textwrap
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
-from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound
+import pytest
+
+from skyvern.forge.sdk.copilot.context import CredentialCheck, StructuredContext
+from skyvern.forge.sdk.copilot.request_policy import _workflow_credential_inputs_unbound, build_request_policy
def _yaml(body: str) -> str:
@@ -212,3 +217,70 @@ def test_malformed_or_empty_yaml_is_inert() -> None:
assert _workflow_credential_inputs_unbound("") == []
assert _workflow_credential_inputs_unbound("- not a workflow yaml\n") == []
assert _workflow_credential_inputs_unbound(":: broken yaml ::") == []
+
+
+def _discovered_context(*credential_ids: str) -> str:
+ structured = StructuredContext(
+ credentials_checked=[
+ CredentialCheck(credential_name=cid, credential_id=cid, found=True) for cid in credential_ids
+ ]
+ )
+ return structured.to_json_str()
+
+
+async def test_discovered_credentials_seed_approved_set_on_none_turn() -> None:
+ org_credentials = [
+ SimpleNamespace(credential_id="cred_amazon"),
+ SimpleNamespace(credential_id="cred_quicken"),
+ ]
+ with patch(
+ "skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids",
+ new=AsyncMock(return_value=org_credentials),
+ ):
+ policy = await build_request_policy(
+ user_message="yes, use both of those",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_discovered_context("cred_amazon", "cred_quicken"),
+ organization_id="o_test",
+ handler=None,
+ )
+
+ assert policy.credential_input_kind == "none"
+ assert [c.credential_id for c in policy.discovered_credentials] == ["cred_amazon", "cred_quicken"]
+
+
+async def test_discovered_credential_absent_from_org_is_not_approved() -> None:
+ with patch(
+ "skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids",
+ new=AsyncMock(return_value=[SimpleNamespace(credential_id="cred_amazon")]),
+ ):
+ policy = await build_request_policy(
+ user_message="yes",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context=_discovered_context("cred_amazon", "cred_ghost"),
+ organization_id="o_test",
+ handler=None,
+ )
+
+ assert [c.credential_id for c in policy.discovered_credentials] == ["cred_amazon"]
+
+
+async def test_no_discovered_credentials_leaves_approved_set_empty() -> None:
+ get_by_ids = AsyncMock(return_value=[])
+ with patch("skyvern.forge.app.DATABASE.credentials.get_credentials_by_ids", new=get_by_ids):
+ policy = await build_request_policy(
+ user_message="add a step to download the report",
+ workflow_yaml="",
+ chat_history=[],
+ global_llm_context="",
+ organization_id="o_test",
+ handler=None,
+ )
+
+ assert policy.discovered_credentials == []
+ get_by_ids.assert_not_awaited()
diff --git a/tests/unit/test_copilot_tools_credential_id_misbinding.py b/tests/unit/test_copilot_tools_credential_id_misbinding.py
index 95a9f8524..23b3cc5e5 100644
--- a/tests/unit/test_copilot_tools_credential_id_misbinding.py
+++ b/tests/unit/test_copilot_tools_credential_id_misbinding.py
@@ -5,10 +5,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.forge.sdk.copilot.request_policy import RequestPolicy
from skyvern.forge.sdk.copilot.runtime import AgentContext
from skyvern.forge.sdk.copilot.tools import (
_credential_id_misbinding_error_message,
_credential_id_misbinding_findings,
+ _list_credentials,
_update_workflow,
)
@@ -456,3 +458,60 @@ def test_error_message_groups_multiple_findings() -> None:
assert "navigation_goal" in message
assert "complete_criterion" in message
assert message.count("cred_111") >= 1
+
+
+class _ListedCredential:
+ def __init__(self, credential_id: str, name: str, tested_url: str | None = None) -> None:
+ self.credential_id = credential_id
+ self.name = name
+ self.tested_url = tested_url
+ self.credential_type = "password"
+ self.username = None
+ self.totp_type = None
+ self.card_last4 = None
+ self.card_brand = None
+ self.secret_label = None
+
+
+def _ctx_with_policy() -> MagicMock:
+ ctx = _ctx()
+ ctx.request_policy = RequestPolicy(credential_input_kind="none")
+ return ctx
+
+
+async def test_list_credentials_records_discovered_credentials_on_policy() -> None:
+ ctx = _ctx_with_policy()
+ listed = [_ListedCredential("cred_site_a", "Site A"), _ListedCredential("cred_site_b", "Site B")]
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)
+ ):
+ await _list_credentials({}, ctx)
+
+ assert [c.credential_id for c in ctx.request_policy.discovered_credentials] == ["cred_site_a", "cred_site_b"]
+
+
+async def test_list_credentials_unions_across_pages_without_duplicates() -> None:
+ ctx = _ctx_with_policy()
+ ctx.request_policy.discovered_credentials = [_ListedCredential("cred_site_a", "Site A")]
+ listed = [_ListedCredential("cred_site_a", "Site A"), _ListedCredential("cred_site_b", "Site B")]
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)
+ ):
+ await _list_credentials({"page": 2}, ctx)
+
+ assert [c.credential_id for c in ctx.request_policy.discovered_credentials] == ["cred_site_a", "cred_site_b"]
+
+
+async def test_list_credentials_without_request_policy_is_inert() -> None:
+ ctx = _ctx()
+ ctx.request_policy = None
+ listed = [_ListedCredential("cred_site_a", "Site A")]
+ with patch(
+ "skyvern.forge.sdk.copilot.tools.app.DATABASE.credentials.get_credentials", new=AsyncMock(return_value=listed)
+ ):
+ result = await _list_credentials({}, ctx)
+
+ assert result["ok"] is True
diff --git a/tests/unit/test_summarize_output.py b/tests/unit/test_summarize_output.py
index 8b1902836..4a7624cfc 100644
--- a/tests/unit/test_summarize_output.py
+++ b/tests/unit/test_summarize_output.py
@@ -1,5 +1,6 @@
"""Tests for the summarize-output endpoint and helpers."""
+import sys
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Iterator
@@ -60,10 +61,16 @@ class TestSummarizeOutputRequest:
SummarizeOutputRequest(output_json="{}", workflow_title="x" * 501)
def test_deeply_nested_json_rejected(self) -> None:
- # 10k levels of nesting exceeds Python's recursion limit inside json.loads.
+ # Rejection relies on json.loads overflowing the recursion limit; pin it so the test does not
+ # depend on the suite-global limit another test may have raised (e.g. via networkx).
deep = "[" * 10_000 + "]" * 10_000
- with pytest.raises(ValidationError):
- SummarizeOutputRequest(output_json=deep)
+ old_limit = sys.getrecursionlimit()
+ sys.setrecursionlimit(1000)
+ try:
+ with pytest.raises(ValidationError):
+ SummarizeOutputRequest(output_json=deep)
+ finally:
+ sys.setrecursionlimit(old_limit)
def _fake_org() -> Organization: