Skip to content

Automate browser based workflows with AI

feat: code-first browser recording (opt-in feature preview) (#7534)

Cindy Li59b739e13148
diff --git a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
index 10a1178f1..d18dc35e2 100644
--- a/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
+++ b/skyvern-frontend/src/routes/browserSessions/hooks/useProcessRecordingMutation.ts
@@ -1,10 +1,12 @@
import { useParams } from "react-router-dom";
import { useMutation } from "@tanstack/react-query";
import { useRef } from "react";
+import { useFeatureFlagEnabled } from "posthog-js/react";
import { getClient } from "@/api/AxiosClient";
import { toast } from "@/components/ui/use-toast";
import { useCredentialGetter } from "@/hooks/useCredentialGetter";
+import { RECORD_BROWSER_CODE_FIRST_FLAG } from "@/util/featureFlags";
import {
useRecordingStore,
type RecordingDraftStep,
@@ -34,6 +36,9 @@ const useProcessRecordingMutation = ({
const recordingStore = useRecordingStore();
const { workflowPermanentId } = useParams();
const mutationStartedAtRef = useRef<number | null>(null);
+ // Per-user opt-in preview; not enrolled reads as false (agent blocks).
+ const codeFirst =
+ useFeatureFlagEnabled(RECORD_BROWSER_CODE_FIRST_FLAG) ?? false;
const processRecordingMutation = useMutation({
mutationFn: async (
@@ -86,6 +91,7 @@ const useProcessRecordingMutation = ({
{
compressed_chunks: string[];
draft_steps?: Array<RecordingDraftStep>;
+ code_first: boolean;
},
{
data: {
@@ -96,6 +102,7 @@ const useProcessRecordingMutation = ({
>(`/browser_sessions/${browserSessionId}/process_recording`, {
compressed_chunks: compressedChunks,
workflow_permanent_id: workflowPermanentId,
+ code_first: codeFirst,
...(draftSteps !== null ? { draft_steps: draftSteps } : {}),
})
.then((response) => ({
diff --git a/skyvern-frontend/src/util/featureFlags.ts b/skyvern-frontend/src/util/featureFlags.ts
index 88ac03476..434942366 100644
--- a/skyvern-frontend/src/util/featureFlags.ts
+++ b/skyvern-frontend/src/util/featureFlags.ts
@@ -39,3 +39,6 @@ export const ANALYTICS_GROUP_BY_LENS_FLAG = "ANALYTICS_GROUP_BY_LENS";
// showing the editor to other orgs would promise a retry that never fires. Server-evaluated via
// /customer (see FeatureFlagProvider), keyed on organization_id like the backend gate.
export const CREDENTIAL_FALLBACK_RETRY_FLAG = "CREDENTIAL_FALLBACK_RETRY";
+
+// Opt-in preview (0% base): recordings synthesize code blocks instead of agent blocks.
+export const RECORD_BROWSER_CODE_FIRST_FLAG = "record_browser_code_first";
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 12b06819b..69d4dbada 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -1278,6 +1278,21 @@ def _trajectory_prefix_at_anchor(
return prefix, len(trajectory) - len(prefix)
+def synthesize_goto_code_block(url: str) -> SynthesizedCodeBlock | None:
+ """A goto-only block for a navigation with no captured interactions after it."""
+ url = (url or "").strip()
+ if not url:
+ return None
+ line = (
+ f"{_INDENT}await page.goto("
+ f"{_py_str(_scrub_url_for_code_literal(url))}, wait_until={_py_str(_DOMCONTENTLOADED)})"
+ )
+ return SynthesizedCodeBlock(
+ code=line + "\n",
+ steps=[{"description": f"Open {url}", "action_type": "goto_url", "line_start": 1, "line_end": 1}],
+ )
+
+
def synthesize_code_block(
trajectory: Sequence[Mapping[str, Any]],
*,
@@ -1624,6 +1639,22 @@ def synthesize_code_block(
emitted += 1
continue
+ if tool_name == "wait":
+ try:
+ duration_ms = int(interaction.get("duration_ms") or 0)
+ except (TypeError, ValueError):
+ duration_ms = 0
+ if duration_ms <= 0:
+ diagnostics.dropped_interactions.append(
+ {"trajectory_index": trajectory_index, "tool_name": tool_name, "reason_code": "missing_duration"}
+ )
+ continue
+ line_start = len(lines) + 1
+ lines.append(f"{action_indent}await page.wait_for_timeout({duration_ms})")
+ append_step(f"Wait {max(duration_ms // 1000, 1)}s", "wait", line_start)
+ emitted += 1
+ continue
+
locator = _locator_expr(
interaction,
notes,
@@ -1767,6 +1798,11 @@ def synthesize_code_block(
lines.append(f"{action_indent}await page.wait_for_load_state({_py_str(_DOMCONTENTLOADED)})")
record_emission(trajectory_index, tool_name, "select_option", locator, line_start=line_start)
append_step(f"Select {value} in {_step_target(interaction)}", "select_option", line_start)
+ elif tool_name == "hover" and not strict_selectors:
+ # Non-strict only: recording trajectories carry deliberate hovers; the
+ # strict-imposition envelope keeps treating hover as unsupported.
+ lines.append(f"{action_indent}await {locator}.hover()")
+ append_step(f"Hover over {_step_target(interaction)}", "hover", line_start)
else:
notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")
diagnostics.dropped_interactions.append(
diff --git a/skyvern/forge/sdk/routes/browser_sessions.py b/skyvern/forge/sdk/routes/browser_sessions.py
index 5da91b382..a0ed78019 100644
--- a/skyvern/forge/sdk/routes/browser_sessions.py
+++ b/skyvern/forge/sdk/routes/browser_sessions.py
@@ -372,6 +372,7 @@ async def process_recording(
compressed_chunks=recording_request.compressed_chunks,
workflow_permanent_id=recording_request.workflow_permanent_id,
draft_steps=recording_request.draft_steps,
+ code_first=recording_request.code_first,
)
return ProcessBrowserSessionRecordingResponse(blocks=blocks, parameters=parameters)
diff --git a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
index f63957c21..34b86d9cc 100644
--- a/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
+++ b/skyvern/forge/sdk/routes/streaming/channels/js/exfiltrate.js
@@ -4,6 +4,167 @@
console.log("[SYS] exfiltration: initializing");
window.__skyvern_exfiltration_initialized = true;
+ // Event types whose targets feed the recording state machines; only these
+ // pay for durable-locator computation (querySelectorAll uniqueness probes).
+ const LOCATOR_EVENT_TYPES = new Set([
+ "click",
+ "dblclick",
+ "contextmenu",
+ "focus",
+ "blur",
+ "input",
+ "change",
+ "keydown",
+ "mouseenter",
+ ]);
+
+ const cssEscape =
+ window.CSS && window.CSS.escape
+ ? window.CSS.escape.bind(window.CSS)
+ : (value) => String(value).replace(/[^a-zA-Z0-9_-]/g, (c) => `\\${c}`);
+
+ // The capture listener has no surrounding try/catch and this script is
+ // injected for every recording, so locator computation must degrade to
+ // null instead of breaking event capture.
+ const tryCapture = (fn) => {
+ try {
+ return fn();
+ } catch (err) {
+ return null;
+ }
+ };
+
+ const isUniqueSelector = (selector) => {
+ try {
+ return document.querySelectorAll(selector).length === 1;
+ } catch (err) {
+ return false;
+ }
+ };
+
+ // Framework-generated ids (numeric suffixes, uuid-ish, React ":r1:") churn
+ // across sessions and must not become replay selectors.
+ const looksGeneratedId = (id) =>
+ /^\d|[_-]\d{3,}|^[0-9a-fA-F-]{16,}$|^:|:$/.test(id);
+
+ const TEST_ATTRIBUTES = [
+ "data-testid",
+ "data-test-id",
+ "data-test",
+ "data-qa",
+ "data-cy",
+ ];
+
+ const getDurableSelector = (element) => {
+ if (!element || !element.tagName || !element.getAttribute) {
+ return null;
+ }
+
+ const id = element.id;
+ if (id && !looksGeneratedId(id)) {
+ const selector = `#${cssEscape(id)}`;
+ if (isUniqueSelector(selector)) return selector;
+ }
+
+ for (const attribute of TEST_ATTRIBUTES) {
+ const value = element.getAttribute(attribute);
+ if (value) {
+ const selector = `[${attribute}="${cssEscape(value)}"]`;
+ if (isUniqueSelector(selector)) return selector;
+ }
+ }
+
+ const tag = element.tagName.toLowerCase();
+ if (["input", "select", "textarea", "button"].includes(tag)) {
+ const name = element.getAttribute("name");
+ if (name) {
+ const selector = `${tag}[name="${cssEscape(name)}"]`;
+ if (isUniqueSelector(selector)) return selector;
+ }
+ }
+
+ // Structural fallback: tag + :nth-of-type at each level, anchored at the
+ // nearest stable-id ancestor (or body). Unique by construction; brittle
+ // under reordering, so it is last.
+ const segments = [];
+ let current = element;
+ let anchor = "";
+ let depth = 0;
+ while (current && current.tagName && depth < 8) {
+ const currentTag = current.tagName.toLowerCase();
+ if (currentTag === "body" || currentTag === "html") break;
+ const currentId = current.id;
+ if (
+ depth > 0 &&
+ currentId &&
+ !looksGeneratedId(currentId) &&
+ isUniqueSelector(`#${cssEscape(currentId)}`)
+ ) {
+ anchor = `#${cssEscape(currentId)}`;
+ break;
+ }
+ let nth = 1;
+ let sibling = current.previousElementSibling;
+ while (sibling) {
+ if (sibling.tagName === current.tagName) nth += 1;
+ sibling = sibling.previousElementSibling;
+ }
+ segments.unshift(`${currentTag}:nth-of-type(${nth})`);
+ current = current.parentElement;
+ depth += 1;
+ }
+ // Empty only when the target is body/html itself, which is never a
+ // useful replay target.
+ if (!segments.length) return null;
+ const selector = `${anchor ? `${anchor} > ` : ""}${segments.join(" > ")}`;
+ return isUniqueSelector(selector) ? selector : null;
+ };
+
+ const IMPLICIT_INPUT_ROLES = {
+ button: "button",
+ submit: "button",
+ reset: "button",
+ image: "button",
+ checkbox: "checkbox",
+ radio: "radio",
+ range: "slider",
+ number: "spinbutton",
+ search: "searchbox",
+ };
+
+ const getAriaRole = (element) => {
+ if (!element || !element.tagName || !element.getAttribute) {
+ return null;
+ }
+
+ const explicit = element.getAttribute("role");
+ if (explicit) return explicit.trim().split(/\s+/)[0];
+
+ const tag = element.tagName.toLowerCase();
+ switch (tag) {
+ case "a":
+ return element.hasAttribute("href") ? "link" : null;
+ case "button":
+ case "summary":
+ return "button";
+ case "select":
+ return element.multiple || element.size > 1 ? "listbox" : "combobox";
+ case "textarea":
+ return "textbox";
+ case "option":
+ return "option";
+ case "img":
+ return "img";
+ case "input": {
+ const type = (element.getAttribute("type") || "text").toLowerCase();
+ if (type === "hidden") return null;
+ return IMPLICIT_INPUT_ROLES[type] || "textbox";
+ }
+ default:
+ return null;
+ }
+ };
+
[
"click",
"mousedown",
@@ -102,6 +263,57 @@
return textSources.length > 0 ? textSources : [];
};
+ // Approximate accname computation, in spec priority order; Playwright's
+ // get_by_role(name=...) resolves the same sources for common controls.
+ const getAccessibleName = (element) => {
+ if (!element || !element.getAttribute) {
+ return null;
+ }
+
+ const candidates = [];
+
+ const ariaLabel = element.getAttribute("aria-label");
+ if (ariaLabel) candidates.push(ariaLabel);
+
+ const labelledBy = element.getAttribute("aria-labelledby");
+ if (labelledBy) {
+ const labelText = labelledBy
+ .split(/\s+/)
+ .map((id) => document.getElementById(id)?.textContent || "")
+ .join(" ");
+ candidates.push(labelText);
+ }
+
+ const labels = getAssociatedLabels(element);
+ if (labels) candidates.push(labels[0]);
+
+ const tag = element.tagName?.toLowerCase();
+ if (["button", "a", "summary", "option"].includes(tag)) {
+ candidates.push(element.innerText || element.textContent);
+ }
+ if (tag === "input") {
+ const type = (element.getAttribute("type") || "").toLowerCase();
+ if (["submit", "button", "reset"].includes(type)) {
+ candidates.push(element.value);
+ }
+ }
+
+ candidates.push(element.getAttribute("title"));
+ candidates.push(element.getAttribute("alt"));
+ candidates.push(element.getAttribute("placeholder"));
+
+ for (const candidate of candidates) {
+ const name = String(candidate || "")
+ .replace(/\s+/g, " ")
+ .trim();
+ if (name) return name.substring(0, 100);
+ }
+
+ return null;
+ };
+
+ const wantsLocator = LOCATOR_EVENT_TYPES.has(eventType);
+
const skyId = e.target?.dataset?.skyId || null;
if (!skyId && e.target?.tagName !== "HTML") {
@@ -162,6 +374,22 @@
text: getElementText(e.target),
labels: getAssociatedLabels(e.target),
skyId: e.target?.dataset?.skyId,
+ selector: wantsLocator
+ ? tryCapture(() => getDurableSelector(e.target))
+ : null,
+ role: wantsLocator
+ ? tryCapture(() => getAriaRole(e.target))
+ : null,
+ accessibleName: wantsLocator
+ ? tryCapture(() => getAccessibleName(e.target))
+ : null,
+ inputType:
+ e.target?.tagName === "INPUT" ||
+ e.target?.tagName === "SELECT" ||
+ e.target?.tagName === "TEXTAREA" ||
+ e.target?.tagName === "BUTTON"
+ ? e.target?.type || null
+ : null,
},
inputValue: ["input", "focus", "blur"].includes(eventType)
? e.target?.value
diff --git a/skyvern/schemas/browser_sessions.py b/skyvern/schemas/browser_sessions.py
index 34920c0ce..0c8fbcef6 100644
--- a/skyvern/schemas/browser_sessions.py
+++ b/skyvern/schemas/browser_sessions.py
@@ -93,6 +93,10 @@ class ProcessBrowserSessionRecordingRequest(BaseModel):
default=None,
description="Optional live interpretation drafts to commit instead of reprocessing the compressed recording.",
)
+ code_first: bool = Field(
+ default=False,
+ description="When true, synthesize deterministic code blocks from the recording instead of agent blocks.",
+ )
class ProcessBrowserSessionRecordingResponse(BaseModel):
diff --git a/skyvern/services/browser_recording/code_first.py b/skyvern/services/browser_recording/code_first.py
new file mode 100644
index 000000000..6c8253c6b
--- /dev/null
+++ b/skyvern/services/browser_recording/code_first.py
@@ -0,0 +1,318 @@
+"""
+Code-first recording conversion: captured actions -> deterministic Playwright code blocks.
+
+Reuses the Workflow Copilot trajectory synthesizer so recordings and copilot scouting
+share one code-generation substrate, parameter policy, and sandbox safety gate.
+"""
+
+import re
+import textwrap
+import typing as t
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+import structlog
+
+from skyvern.client.types.workflow_definition_yaml_blocks_item import WorkflowDefinitionYamlBlocksItem_Code
+from skyvern.client.types.workflow_definition_yaml_parameters_item import WorkflowDefinitionYamlParametersItem_Workflow
+from skyvern.forge.sdk.copilot.code_block_synthesis import synthesize_code_block, synthesize_goto_code_block
+from skyvern.forge.sdk.workflow.exceptions import InsecureCodeDetected
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.services.browser_recording.types import (
+ Action,
+ ActionClick,
+ ActionHover,
+ ActionInputText,
+ ActionUrlChange,
+ ActionWait,
+ RecordingDraftStep,
+)
+
+LOG = structlog.get_logger(__name__)
+
+# A navigation this soon after a click or text submit was caused by it; the emitted
+# click/press already waits for the load, so no goto is emitted for that navigation.
+CLICK_NAVIGATION_WINDOW_MS = 3000
+
+# Draft steps reference their source action by (kind, timestamp_start); tolerance
+# absorbs float round-tripping through JSON, not clock skew.
+DRAFT_TIMESTAMP_TOLERANCE_MS = 1.0
+
+ActionDraftPair = tuple[Action, RecordingDraftStep | None]
+
+# Interactions that silently vanish from a workflow if the synthesizer cannot
+# locate them; any unlocatable one forces the legacy (LLM agent block) fallback.
+# Dropped hovers are tolerated: they are usually incidental to a located click.
+_REQUIRED_LOCATOR_TOOLS = frozenset({"click", "type_text", "select_option"})
+
+CodeFirstResult = tuple[
+ list[WorkflowDefinitionYamlBlocksItem_Code],
+ list[WorkflowDefinitionYamlParametersItem_Workflow],
+]
+
+
+@dataclass
+class RecordingSegment:
+ source_url: str | None = None
+ pairs: list[ActionDraftPair] = field(default_factory=list)
+
+
+def apply_draft_overlay(
+ actions: list[Action],
+ draft_steps: list[RecordingDraftStep] | None,
+) -> list[ActionDraftPair]:
+ """Join user-edited draft steps back onto their source actions; a missing draft means deleted."""
+ if draft_steps is None:
+ return [(action, None) for action in actions]
+ if not draft_steps:
+ # An empty list means the user deleted every interpreted step.
+ return []
+
+ remaining = [draft for draft in draft_steps if draft.timestamp_start is not None]
+ if not remaining:
+ # Drafts without source timestamps cannot be joined back to actions;
+ # keep the raw actions rather than treating everything as deleted.
+ return [(action, None) for action in actions]
+
+ pairs: list[ActionDraftPair] = []
+ for action in actions:
+ matched: RecordingDraftStep | None = None
+ for index, draft in enumerate(remaining):
+ if draft.action_kind != action.kind:
+ continue
+ if abs((draft.timestamp_start or 0.0) - action.timestamp_start) <= DRAFT_TIMESTAMP_TOLERANCE_MS:
+ matched = remaining.pop(index)
+ break
+ if matched is None:
+ LOG.debug(
+ "record_browser.code_first_overlay_dropped_action",
+ action_kind=action.kind,
+ action_timestamp_start=action.timestamp_start,
+ )
+ continue
+ pairs.append((action, matched))
+
+ return pairs
+
+
+def segment_actions(pairs: list[ActionDraftPair]) -> list[RecordingSegment]:
+ """Split at user-initiated navigations; click-caused navigations stay inside their segment."""
+ segments = [RecordingSegment()]
+ last_interactive_end: float | None = None
+
+ for action, draft in pairs:
+ if isinstance(action, ActionUrlChange):
+ caused_by_interaction = (
+ last_interactive_end is not None
+ and action.timestamp_start - last_interactive_end <= CLICK_NAVIGATION_WINDOW_MS
+ )
+ if caused_by_interaction:
+ # One navigation per interaction: a later url_change inside the same
+ # window is a genuine user navigation and must start a new segment.
+ last_interactive_end = None
+ continue
+ url = ((draft.url or "").strip() if draft else "") or action.url
+ if segments[-1].pairs:
+ segments.append(RecordingSegment(source_url=url))
+ else:
+ segments[-1].source_url = url
+ continue
+
+ if isinstance(action, (ActionClick, ActionInputText)):
+ last_interactive_end = action.timestamp_end
+
+ segments[-1].pairs.append((action, draft))
+
+ if segments[0].source_url is None and segments[0].pairs:
+ segments[0].source_url = segments[0].pairs[0][0].url
+
+ return [segment for segment in segments if segment.pairs or segment.source_url]
+
+
+def _interaction_for_action(action: Action) -> dict[str, t.Any] | None:
+ target = action.target
+ base: dict[str, t.Any] = {}
+ if target.selector:
+ base["selector"] = target.selector
+ if target.role:
+ base["role"] = target.role
+ if target.accessible_name:
+ base["accessible_name"] = target.accessible_name
+
+ if isinstance(action, ActionClick):
+ return {"tool_name": "click", **base}
+ if isinstance(action, ActionHover):
+ return {"tool_name": "hover", **base}
+ if isinstance(action, ActionInputText):
+ if (target.tag_name or "").upper() == "SELECT":
+ return {"tool_name": "select_option", "value": action.input_value, **base}
+ interaction: dict[str, t.Any] = {"tool_name": "type_text", **base}
+ # A password typed during recording must become a parameter slot without
+ # a plaintext default; the value never reaches code or parameter defaults.
+ if (target.input_type or "").lower() != "password":
+ interaction["typed_value"] = action.input_value
+ return interaction
+ if isinstance(action, ActionWait):
+ return {"tool_name": "wait", "duration_ms": action.duration_ms}
+ return None
+
+
+def segment_trajectory(segment: RecordingSegment) -> list[dict[str, t.Any]]:
+ trajectory: list[dict[str, t.Any]] = []
+ for action, draft in segment.pairs:
+ interaction = _interaction_for_action(action)
+ if interaction is None:
+ continue
+ if interaction["tool_name"] == "wait" and draft is not None and draft.wait_sec:
+ interaction["duration_ms"] = int(draft.wait_sec) * 1000
+ trajectory.append(interaction)
+ if trajectory and segment.source_url:
+ trajectory[0] = {**trajectory[0], "source_url": segment.source_url}
+ return trajectory
+
+
+def _segment_label(segment: RecordingSegment, used: set[str]) -> str:
+ host = ""
+ if segment.source_url:
+ try:
+ host = urlparse(segment.source_url).netloc
+ except ValueError:
+ pass
+ base = re.sub(r"\W+", "_", f"recorded_{host}" if host else "recorded_steps").strip("_").lower()
+ label = base
+ suffix = 2
+ while label in used:
+ label = f"{base}_{suffix}"
+ suffix += 1
+ used.add(label)
+ return label
+
+
+def actions_to_code_first_blocks(
+ actions: list[Action],
+ draft_steps: list[RecordingDraftStep] | None,
+) -> CodeFirstResult | None:
+ """Convert recorded actions into code blocks and parameters; None means fall back to legacy blocks."""
+ if draft_steps is not None and not draft_steps:
+ # The user deleted every interpreted step; commit an empty workflow rather
+ # than falling back to blocks derived from the raw actions.
+ return [], []
+ pairs = apply_draft_overlay(actions, draft_steps)
+ segments = segment_actions(pairs)
+
+ blocks: list[WorkflowDefinitionYamlBlocksItem_Code] = []
+ # Global parameter key -> identity of the recorded field that minted it
+ # (selector, role, accessible name). The same field re-filled in a later
+ # segment reuses its key; a different same-labeled field gets a fresh key.
+ parameter_identities: dict[str, tuple[str, str, str] | None] = {}
+ used_labels: set[str] = set()
+
+ for segment in segments:
+ trajectory = segment_trajectory(segment)
+ for interaction in trajectory:
+ if interaction["tool_name"] in _REQUIRED_LOCATOR_TOOLS and not (
+ interaction.get("selector") or (interaction.get("role") and interaction.get("accessible_name"))
+ ):
+ LOG.info(
+ "record_browser.code_first_unlocatable_interaction",
+ tool_name=interaction["tool_name"],
+ )
+ return None
+ if trajectory:
+ synthesized = synthesize_code_block(trajectory)
+ elif segment.source_url:
+ synthesized = synthesize_goto_code_block(segment.source_url)
+ else:
+ synthesized = None
+ if synthesized is None:
+ continue
+
+ code = textwrap.dedent(synthesized.code)
+ identity_by_key: dict[str, tuple[str, str, str]] = {}
+ for trajectory_index, param_key in synthesized.diagnostics.typed_param_bindings:
+ if 0 <= trajectory_index < len(trajectory):
+ source = trajectory[trajectory_index]
+ identity_by_key.setdefault(
+ param_key,
+ (
+ str(source.get("selector") or ""),
+ str(source.get("role") or ""),
+ str(source.get("accessible_name") or ""),
+ ),
+ )
+
+ block_parameter_keys: list[str] = []
+ block_original_keys = {str(parameter.get("key") or "").strip() for parameter in synthesized.parameters}
+ renames: dict[str, str] = {}
+ for parameter in synthesized.parameters:
+ key = str(parameter.get("key") or "").strip()
+ if not key:
+ continue
+ if parameter.get("credential_id"):
+ # Recordings cannot bind credentials yet; the synthesizer only emits
+ # these for fill_credential_field interactions, which we never map.
+ LOG.warning("record_browser.code_first_unexpected_credential_parameter", parameter_key=key)
+ continue
+ identity = identity_by_key.get(key)
+ resolved_key = key
+ if key in parameter_identities and (identity is None or parameter_identities[key] != identity):
+ suffix = 2
+ while True:
+ candidate = f"{key}_{suffix}"
+ suffix += 1
+ if candidate in block_original_keys or candidate in renames.values():
+ continue
+ if candidate not in parameter_identities or (
+ identity is not None and parameter_identities[candidate] == identity
+ ):
+ resolved_key = candidate
+ break
+ renames[key] = resolved_key
+ parameter_identities.setdefault(resolved_key, identity)
+ block_parameter_keys.append(resolved_key)
+
+ for original_key, resolved_key in renames.items():
+ # A typed-text fill is the only emission that reads a string parameter,
+ # always as `str(<key>)`, so this rename cannot touch selectors. Rename
+ # targets never collide with this block's own keys, so one rename cannot
+ # cascade onto the fill of another parameter.
+ code = code.replace(f"str({original_key})", f"str({resolved_key})")
+
+ try:
+ CodeBlock.is_safe_code(code)
+ # ast.parse raises ValueError on e.g. null bytes; any failure inside the
+ # safety gate must fall back to legacy blocks, never surface as a 500.
+ except (SyntaxError, ValueError, InsecureCodeDetected):
+ LOG.warning(
+ "record_browser.code_first_safety_rejected",
+ dropped_interaction_count=len(synthesized.diagnostics.dropped_interactions),
+ exc_info=True,
+ )
+ return None
+
+ blocks.append(
+ WorkflowDefinitionYamlBlocksItem_Code(
+ label=_segment_label(segment, used_labels),
+ code=code,
+ parameter_keys=block_parameter_keys or None,
+ # Editor's convertToNode reads block.parameters.map(p => p.key); mirror the action block.
+ parameters=[{"key": key} for key in block_parameter_keys],
+ )
+ )
+
+ if not blocks:
+ return None
+
+ parameters = [
+ WorkflowDefinitionYamlParametersItem_Workflow(
+ key=key,
+ workflow_parameter_type="string",
+ # Like the legacy path, recorded values never persist as defaults: a
+ # secret typed into any field (not just type=password) must not land
+ # in a DB-stored, API-exposed default_value. The user binds values.
+ default_value="",
+ description="",
+ )
+ for key in parameter_identities
+ ]
+ return blocks, parameters
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index e5121a138..a605b9f56 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -23,6 +23,7 @@ from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler import LLMAPIHandler
from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+from skyvern.services.browser_recording.code_first import actions_to_code_first_blocks
from skyvern.services.browser_recording.types import (
Action,
ActionBlockable,
@@ -34,6 +35,7 @@ from skyvern.services.browser_recording.types import (
ExfiltratedConsoleEvent,
ExfiltratedEvent,
OutputBlock,
+ ProcessedBlock,
RecordingDraftStep,
)
@@ -609,10 +611,33 @@ class Processor:
self,
compressed_chunks: list[str],
draft_steps: list[RecordingDraftStep] | None = None,
- ) -> tuple[list[OutputBlock], list[WorkflowDefinitionYamlParametersItem_Workflow]]:
+ code_first: bool = False,
+ ) -> tuple[list[ProcessedBlock], list[WorkflowDefinitionYamlParametersItem_Workflow]]:
"""
Process the compressed browser session recording into workflow definition blocks.
"""
+ if code_first:
+ # Code-first always re-derives selector-bearing actions from raw events;
+ # draft steps carry no locators and act only as an edit overlay.
+ events = self.compressed_chunks_to_events(compressed_chunks)
+ actions = self.events_to_actions(events)
+ code_first_result = actions_to_code_first_blocks(actions, draft_steps)
+ if code_first_result is not None:
+ code_blocks, code_parameters = code_first_result
+ LOG.info(
+ "record_browser.process_recording_code_first",
+ recording_code_block_count=len(code_blocks),
+ recording_code_parameter_count=len(code_parameters),
+ recording_action_count=len(actions),
+ **self.identity,
+ )
+ return list(code_blocks), code_parameters
+ LOG.warning(
+ "record_browser.code_first_fallback_to_legacy",
+ recording_action_count=len(actions),
+ **self.identity,
+ )
+
# `is not None` (not truthiness): an empty list means the user deleted every
# live-interpreted step, which must not fall back to re-processing raw events.
if draft_steps is not None:
@@ -647,7 +672,8 @@ class BrowserSessionRecordingService:
workflow_permanent_id: str,
compressed_chunks: list[str],
draft_steps: list[RecordingDraftStep] | None = None,
- ) -> tuple[list[OutputBlock], list[WorkflowDefinitionYamlParametersItem_Workflow]]:
+ code_first: bool = False,
+ ) -> tuple[list[ProcessedBlock], list[WorkflowDefinitionYamlParametersItem_Workflow]]:
"""
Process compressed browser session recording events into workflow definition blocks.
"""
@@ -657,7 +683,7 @@ class BrowserSessionRecordingService:
workflow_permanent_id,
)
- return await processor.process(compressed_chunks, draft_steps=draft_steps)
+ return await processor.process(compressed_chunks, draft_steps=draft_steps, code_first=code_first)
async def smoke() -> None:
diff --git a/skyvern/services/browser_recording/state_machines/click.py b/skyvern/services/browser_recording/state_machines/click.py
index 4eb80b65d..608b5cb53 100644
--- a/skyvern/services/browser_recording/state_machines/click.py
+++ b/skyvern/services/browser_recording/state_machines/click.py
@@ -67,6 +67,10 @@ class StateMachineClick(StateMachine):
sky_id=self.target.skyId,
tag_name=self.target.tagName,
texts=self.target.text,
+ selector=self.target.selector,
+ role=self.target.role,
+ accessible_name=self.target.accessibleName,
+ input_type=self.target.inputType,
)
action = ActionClick(
diff --git a/skyvern/services/browser_recording/state_machines/hover.py b/skyvern/services/browser_recording/state_machines/hover.py
index a70ceb0dc..68da2e036 100644
--- a/skyvern/services/browser_recording/state_machines/hover.py
+++ b/skyvern/services/browser_recording/state_machines/hover.py
@@ -132,6 +132,10 @@ class StateMachineHover(StateMachine):
sky_id=self.target.skyId,
tag_name=self.target.tagName,
texts=self.target.text,
+ selector=self.target.selector,
+ role=self.target.role,
+ accessible_name=self.target.accessibleName,
+ input_type=self.target.inputType,
)
action = ActionHover(
diff --git a/skyvern/services/browser_recording/state_machines/input_text.py b/skyvern/services/browser_recording/state_machines/input_text.py
index 90d08e160..c1b799a16 100644
--- a/skyvern/services/browser_recording/state_machines/input_text.py
+++ b/skyvern/services/browser_recording/state_machines/input_text.py
@@ -118,6 +118,10 @@ class StateMachineInputText(StateMachine):
sky_id=self.target.skyId,
tag_name=self.target.tagName,
texts=self.target.text,
+ selector=self.target.selector,
+ role=self.target.role,
+ accessible_name=self.target.accessibleName,
+ input_type=self.target.inputType,
)
action = ActionInputText(
diff --git a/skyvern/services/browser_recording/types.py b/skyvern/services/browser_recording/types.py
index 1e1274108..adf26958d 100644
--- a/skyvern/services/browser_recording/types.py
+++ b/skyvern/services/browser_recording/types.py
@@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field
from skyvern.client.types.workflow_definition_yaml_blocks_item import (
WorkflowDefinitionYamlBlocksItem_Action,
+ WorkflowDefinitionYamlBlocksItem_Code,
WorkflowDefinitionYamlBlocksItem_GotoUrl,
WorkflowDefinitionYamlBlocksItem_Wait,
)
@@ -85,6 +86,12 @@ class ActionTarget(BaseModel):
sky_id: str | None = None
tag_name: str | None = None
texts: list[str] = []
+ # Durable-locator fields captured at event time for code-first synthesis.
+ # sky_id is runtime-assigned and never usable as a replay selector.
+ selector: str | None = None
+ role: str | None = None
+ accessible_name: str | None = None
+ input_type: str | None = None
class Mouse(BaseModel):
@@ -104,6 +111,10 @@ OutputBlock = t.Union[
WorkflowDefinitionYamlBlocksItem_Wait,
]
+# What process_recording can return: legacy interpretable blocks, plus code blocks
+# when code-first mode is enabled. The live interpreter only ever sees OutputBlock.
+ProcessedBlock = t.Union[OutputBlock, WorkflowDefinitionYamlBlocksItem_Code]
+
class RecordingDraftStep(BaseModel):
step_id: str
@@ -180,6 +191,10 @@ class EventTarget(BaseModel):
tagName: str | None = None
text: list[str] = []
value: str | int | None = None
+ selector: str | None = None
+ role: str | None = None
+ accessibleName: str | None = None
+ inputType: str | None = None
class MousePosition(BaseModel):
diff --git a/tests/unit/services/test_browser_recording_code_first.py b/tests/unit/services/test_browser_recording_code_first.py
new file mode 100644
index 000000000..d3d265bc7
--- /dev/null
+++ b/tests/unit/services/test_browser_recording_code_first.py
@@ -0,0 +1,422 @@
+import pytest
+
+from skyvern.services.browser_recording.code_first import (
+ actions_to_code_first_blocks,
+ apply_draft_overlay,
+ segment_actions,
+)
+from skyvern.services.browser_recording.service import Processor
+from skyvern.services.browser_recording.types import (
+ Action,
+ ActionClick,
+ ActionHover,
+ ActionInputText,
+ ActionKind,
+ ActionTarget,
+ ActionUrlChange,
+ ActionWait,
+ Mouse,
+ RecordingDraftStep,
+)
+
+ORG_ID = "org_123"
+PBS_ID = "pbs_123"
+WP_ID = "wpid_123"
+
+START_URL = "https://example.com/start"
+
+
+def make_target(**kwargs) -> ActionTarget:
+ return ActionTarget(mouse=Mouse(xp=0.5, yp=0.5), **kwargs)
+
+
+def make_click(ts: float, url: str = START_URL, **target_kwargs) -> ActionClick:
+ return ActionClick(
+ kind=ActionKind.CLICK.value,
+ target=make_target(**target_kwargs),
+ timestamp_start=ts,
+ timestamp_end=ts,
+ url=url,
+ )
+
+
+def make_input(ts: float, value: str, url: str = START_URL, **target_kwargs) -> ActionInputText:
+ return ActionInputText(
+ kind=ActionKind.INPUT_TEXT.value,
+ target=make_target(**target_kwargs),
+ timestamp_start=ts,
+ timestamp_end=ts,
+ url=url,
+ input_value=value,
+ )
+
+
+def make_url_change(ts: float, url: str) -> ActionUrlChange:
+ return ActionUrlChange(
+ kind=ActionKind.URL_CHANGE.value,
+ target=make_target(),
+ timestamp_start=ts,
+ timestamp_end=ts,
+ url=url,
+ )
+
+
+def make_wait(ts: float, duration_ms: int, url: str = START_URL) -> ActionWait:
+ return ActionWait(
+ kind=ActionKind.WAIT.value,
+ target=make_target(),
+ timestamp_start=ts,
+ timestamp_end=ts + duration_ms,
+ url=url,
+ duration_ms=duration_ms,
+ )
+
+
+def make_hover(ts: float, url: str = START_URL, **target_kwargs) -> ActionHover:
+ return ActionHover(
+ kind=ActionKind.HOVER.value,
+ target=make_target(**target_kwargs),
+ timestamp_start=ts,
+ timestamp_end=ts + 2000,
+ url=url,
+ )
+
+
+def draft_for(action: Action, **overrides) -> RecordingDraftStep:
+ block_type = {
+ ActionKind.URL_CHANGE: "goto_url",
+ ActionKind.WAIT: "wait",
+ }.get(action.kind, "action")
+ fields = {
+ "step_id": f"step_{action.timestamp_start}",
+ "action_kind": action.kind,
+ "block_type": block_type,
+ "label": "step",
+ "timestamp_start": action.timestamp_start,
+ "timestamp_end": action.timestamp_end,
+ }
+ fields.update(overrides)
+ return RecordingDraftStep(**fields)
+
+
+def test_click_and_type_synthesize_single_code_block() -> None:
+ actions: list[Action] = [
+ make_input(1000, "widgets", selector="#search", accessible_name="Search"),
+ make_click(2000, selector="#submit", role="button", accessible_name="Go"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert len(blocks) == 1
+ block = blocks[0]
+ assert block.block_type == "code"
+ assert f'await page.goto("{START_URL}"'.format(START_URL=START_URL) in block.code
+ assert 'await page.locator("#search").fill(str(search))' in block.code
+ assert 'await page.locator("#submit").click()' in block.code
+ assert block.parameter_keys == ["search"]
+ assert block.model_dump()["parameters"] == [{"key": "search"}]
+ assert len(parameters) == 1
+ assert parameters[0].key == "search"
+ # Recorded values never persist as defaults (legacy parity + secret safety).
+ assert parameters[0].default_value == ""
+ assert parameters[0].workflow_parameter_type == "string"
+ assert "widgets" not in block.code
+
+
+def test_password_input_becomes_parameter_without_default() -> None:
+ actions: list[Action] = [
+ make_input(1000, "hunter2", selector="#pw", accessible_name="Password", input_type="password"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert "hunter2" not in blocks[0].code
+ # `password` is sandbox-reserved, so the synthesizer names the slot `password_field`.
+ assert parameters[0].key == "password_field"
+ assert parameters[0].default_value == ""
+
+
+def test_select_element_maps_to_select_option() -> None:
+ actions: list[Action] = [
+ make_input(1000, "CA", selector="#state", tag_name="SELECT", accessible_name="State"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, _ = result
+ assert 'await page.locator("#state").select_option("CA")' in blocks[0].code
+
+
+def test_wait_and_hover_emit_deterministic_lines() -> None:
+ actions: list[Action] = [
+ make_hover(1000, selector="#menu", accessible_name="Menu"),
+ make_wait(4000, 6000),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, _ = result
+ assert 'await page.locator("#menu").hover()' in blocks[0].code
+ assert "await page.wait_for_timeout(6000)" in blocks[0].code
+
+
+def test_user_navigation_starts_new_segment() -> None:
+ actions: list[Action] = [
+ make_click(1000, selector="#one"),
+ # 20s after the click: a typed/user-initiated navigation, not click-caused.
+ make_url_change(21000, "https://other.example.com/page"),
+ make_click(22000, url="https://other.example.com/page", selector="#two"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, _ = result
+ assert len(blocks) == 2
+ assert '"https://other.example.com/page"' in blocks[1].code
+ assert 'await page.locator("#two").click()' in blocks[1].code
+
+
+def test_click_caused_navigation_stays_in_segment() -> None:
+ actions: list[Action] = [
+ make_click(1000, selector="#login"),
+ make_url_change(1500, "https://example.com/dashboard"),
+ make_click(2500, url="https://example.com/dashboard", selector="#profile"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, _ = result
+ assert len(blocks) == 1
+ assert "dashboard" not in blocks[0].code
+
+
+def test_goto_only_recording_emits_goto_code_block() -> None:
+ actions: list[Action] = [make_url_change(1000, "https://example.com/target")]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert len(blocks) == 1
+ assert 'await page.goto("https://example.com/target"' in blocks[0].code
+ assert blocks[0].model_dump()["parameters"] == []
+ assert parameters == []
+
+
+def test_actions_without_locators_fall_back_to_none() -> None:
+ actions: list[Action] = [make_click(1000), make_click(2000)]
+
+ assert actions_to_code_first_blocks(actions, None) is None
+
+
+def test_draft_overlay_deletion_drops_action() -> None:
+ kept = make_click(1000, selector="#keep", accessible_name="Keep")
+ deleted = make_click(2000, selector="#delete", accessible_name="Delete")
+
+ result = actions_to_code_first_blocks([kept, deleted], [draft_for(kept)])
+
+ assert result is not None
+ blocks, _ = result
+ assert "#keep" in blocks[0].code
+ assert "#delete" not in blocks[0].code
+
+
+def test_empty_draft_steps_commits_empty_workflow() -> None:
+ actions: list[Action] = [make_click(1000, selector="#keep", accessible_name="Keep")]
+
+ assert apply_draft_overlay(actions, []) == []
+ assert actions_to_code_first_blocks(actions, []) == ([], [])
+
+
+def test_user_navigation_after_click_caused_navigation_starts_segment() -> None:
+ actions: list[Action] = [
+ make_click(1000, selector="#login"),
+ # Click-caused navigation: consumed, no goto emitted.
+ make_url_change(1500, "https://example.com/dashboard"),
+ # A second navigation inside the click window is user-initiated.
+ make_url_change(2500, "https://other.example.com/manual"),
+ make_click(3000, url="https://other.example.com/manual", selector="#two"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, _ = result
+ assert len(blocks) == 2
+ assert '"https://other.example.com/manual"' in blocks[1].code
+
+
+def test_safety_gate_value_error_falls_back_to_legacy(monkeypatch: pytest.MonkeyPatch) -> None:
+ def raise_value_error(code: str) -> None:
+ raise ValueError("source code string cannot contain null bytes")
+
+ monkeypatch.setattr(
+ "skyvern.services.browser_recording.code_first.CodeBlock.is_safe_code",
+ staticmethod(raise_value_error),
+ )
+ actions: list[Action] = [make_click(1000, selector="#go", accessible_name="Go")]
+
+ assert actions_to_code_first_blocks(actions, None) is None
+
+
+def test_draft_overlay_without_timestamps_keeps_all_actions() -> None:
+ actions: list[Action] = [make_click(1000, selector="#keep")]
+ draft = draft_for(actions[0], timestamp_start=None, timestamp_end=None)
+
+ pairs = apply_draft_overlay(actions, [draft])
+
+ assert [action for action, _ in pairs] == actions
+
+
+def test_draft_overlay_wait_seconds_override() -> None:
+ # The wait sits after a located click: a leading wait is absorbed by the
+ # synthesizer's entry-replay visibility wait rather than emitted.
+ click = make_click(1000, selector="#go")
+ wait = make_wait(2000, 6000)
+
+ result = actions_to_code_first_blocks([click, wait], [draft_for(click), draft_for(wait, wait_sec=12)])
+
+ assert result is not None
+ blocks, _ = result
+ assert "await page.wait_for_timeout(12000)" in blocks[0].code
+
+
+def test_colliding_parameter_keys_are_renamed_across_segments() -> None:
+ actions: list[Action] = [
+ make_input(1000, "first", selector="#q1", accessible_name="Search"),
+ make_url_change(21000, "https://other.example.com/page"),
+ make_input(22000, "second", url="https://other.example.com/page", selector="#q2", accessible_name="Search"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert len(blocks) == 2
+ assert "fill(str(search))" in blocks[0].code
+ assert "fill(str(search_2))" in blocks[1].code
+ assert {parameter.key for parameter in parameters} == {"search", "search_2"}
+ assert all(parameter.default_value == "" for parameter in parameters)
+
+
+def test_same_field_across_segments_reuses_parameter_key() -> None:
+ actions: list[Action] = [
+ make_input(1000, "first", selector="#q", accessible_name="Search"),
+ make_url_change(21000, "https://other.example.com/page"),
+ make_input(22000, "second", url="https://other.example.com/page", selector="#q", accessible_name="Search"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert len(blocks) == 2
+ assert "fill(str(search))" in blocks[0].code
+ assert "fill(str(search))" in blocks[1].code
+ assert [parameter.key for parameter in parameters] == ["search"]
+
+
+def test_same_labeled_fields_in_one_segment_do_not_conflate_after_rename() -> None:
+ # Segment 1 claims `search`; segment 2 has two distinct same-labeled fields
+ # locally deduped to `search`/`search_2`. The rename of segment 2's `search`
+ # must not cascade onto its `search_2` fill.
+ actions: list[Action] = [
+ make_input(1000, "first", selector="#q1", accessible_name="Search"),
+ make_url_change(21000, "https://other.example.com/page"),
+ make_input(22000, "second", url="https://other.example.com/page", selector="#q2", accessible_name="Search"),
+ make_input(23000, "third", url="https://other.example.com/page", selector="#q3", accessible_name="Search"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert len(blocks) == 2
+ assert 'page.locator("#q2").fill(str(search_3))' in blocks[1].code
+ assert 'page.locator("#q3").fill(str(search_2))' in blocks[1].code
+ keys = [parameter.key for parameter in parameters]
+ assert sorted(keys) == ["search", "search_2", "search_3"]
+
+
+def test_typed_secret_in_non_password_field_never_reaches_default_value() -> None:
+ actions: list[Action] = [
+ make_input(1000, "sk-secret-token", selector="#otp", accessible_name="One-time code", input_type="text"),
+ ]
+
+ result = actions_to_code_first_blocks(actions, None)
+
+ assert result is not None
+ blocks, parameters = result
+ assert "sk-secret-token" not in blocks[0].code
+ assert all("sk-secret-token" not in (parameter.default_value or "") for parameter in parameters)
+ assert all(parameter.default_value == "" for parameter in parameters)
+
+
+def test_segment_actions_uses_first_action_url_as_entry() -> None:
+ click = make_click(1000, selector="#one")
+
+ segments = segment_actions([(click, None)])
+
+ assert len(segments) == 1
+ assert segments[0].source_url == START_URL
+
+
+async def test_process_code_first_returns_code_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
+ actions: list[Action] = [make_click(1000, selector="#submit", accessible_name="Go")]
+ monkeypatch.setattr(Processor, "compressed_chunks_to_events", lambda self, chunks: [])
+ monkeypatch.setattr(
+ Processor,
+ "events_to_actions",
+ lambda self, events, machines=None, initial_actions=None: actions,
+ )
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ blocks, parameters = await processor.process(["chunk"], code_first=True)
+
+ assert len(blocks) == 1
+ assert blocks[0].block_type == "code"
+ assert parameters == []
+
+
+async def test_process_code_first_falls_back_to_legacy_when_synthesis_empty(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(Processor, "compressed_chunks_to_events", lambda self, chunks: [])
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ blocks, parameters = await processor.process(["chunk"], code_first=True)
+
+ assert blocks == []
+ assert parameters == []
+
+
+async def test_process_code_first_prefers_draft_overlay_over_drafts_to_blocks(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ kept = make_click(1000, selector="#keep", accessible_name="Keep")
+ deleted = make_click(2000, selector="#delete", accessible_name="Delete")
+ monkeypatch.setattr(Processor, "compressed_chunks_to_events", lambda self, chunks: [])
+ monkeypatch.setattr(
+ Processor,
+ "events_to_actions",
+ lambda self, events, machines=None, initial_actions=None: [kept, deleted],
+ )
+
+ processor = Processor(PBS_ID, ORG_ID, WP_ID)
+ blocks, _ = await processor.process(["chunk"], draft_steps=[draft_for(kept)], code_first=True)
+
+ assert len(blocks) == 1
+ assert blocks[0].block_type == "code"
+ assert "#keep" in blocks[0].code
+ assert "#delete" not in blocks[0].code
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index 4d292c1ad..a433243d0 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -1689,7 +1689,6 @@ class TestStepEmission:
def test_skipped_interactions_emit_no_step(self) -> None:
trajectory = [
_interaction("click", selector="#go", source_url="https://example.com/"),
- _interaction("hover", selector="#menu"),
_interaction("select_option", selector="#size"),
_interaction("click"),
_interaction("press_key", key=""),
@@ -1698,6 +1697,39 @@ class TestStepEmission:
assert block is not None
assert [s["action_type"] for s in block.steps] == ["goto_url", "click"]
+ def test_hover_emits_step_in_non_strict_mode(self) -> None:
+ trajectory = [
+ _interaction("click", selector="#go", source_url="https://example.com/"),
+ _interaction("hover", selector="#menu"),
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["goto_url", "click", "hover"]
+ assert 'await page.locator("#menu").hover()' in block.code
+
+ def test_hover_stays_unsupported_in_strict_mode(self) -> None:
+ block = synthesize_code_block(
+ [
+ _interaction("click", selector="#open", source_url="https://example.com/"),
+ _interaction("hover", selector="#menu"),
+ ],
+ strict_selectors=True,
+ )
+ assert block is not None
+ assert {"trajectory_index": 1, "tool_name": "hover", "reason_code": "unsupported_tool"} in (
+ block.diagnostics.dropped_interactions
+ )
+
+ def test_wait_emits_timeout_step(self) -> None:
+ trajectory = [
+ _interaction("click", selector="#go", source_url="https://example.com/"),
+ {"tool_name": "wait", "duration_ms": 6000},
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["goto_url", "click", "wait"]
+ assert "await page.wait_for_timeout(6000)" in block.code
+
def test_no_entry_url_means_no_goto_step(self) -> None:
block = synthesize_code_block([_interaction("press_key", key="Enter")])
assert block is not None