Skip to content

Automate browser based workflows with AI

feat(SKY-10837): code-first code block backend (schema, recording proxy, copilot fills, run rendering) (#6593)

Aaron Perez269ebb10fcfa
diff --git a/skyvern/config.py b/skyvern/config.py
index 72e442e64..df91a14a4 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -173,6 +173,9 @@ class Settings(BaseSettings):
# Default-off companion to code-block mode. When enabled, Copilot can impose
# strict scout-trajectory synthesis at the persist seam for the narrow PR-1 envelope.
WORKFLOW_COPILOT_CODE_BLOCK_IMPOSE_SYNTHESIS: bool = False
+ # Flag-gate for the optional LLM pass that rewrites code-block step descriptions.
+ # Off (default) = deterministic templated descriptions; On = LLM rewrites descriptions only.
+ WORKFLOW_COPILOT_CODE_BLOCK_STEP_DESCRIPTIONS_LLM: bool = False
PORT: int = 8000
ALLOWED_ORIGINS: list[str] = ["*"]
BLOCKED_HOSTS: list[str] = ["localhost"]
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 038ae80d8..252766e67 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -110,6 +110,7 @@ from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
from skyvern.forge.sdk.workflow.models.block import (
ActionBlock,
BaseTaskBlock,
+ CodeBlock,
ValidationBlock,
)
from skyvern.forge.sdk.workflow.models.workflow import Workflow, WorkflowRun, WorkflowRunStatus
@@ -557,6 +558,39 @@ class ForgeAgent:
)
return task, step
+ async def create_task_and_step_from_code_block(
+ self,
+ code_block: CodeBlock,
+ organization_id: str | None,
+ workflow_run_id: str,
+ task_url: str,
+ ) -> tuple[Task, Step]:
+ """Container task v1 for a code block's recorded actions, also the seat for future agent fallback."""
+ task_order, task_retry = await BaseTaskBlock.get_task_order(workflow_run_id, 0)
+ task = await app.DATABASE.tasks.create_task(
+ url=task_url,
+ title=code_block.label,
+ navigation_goal=code_block.prompt,
+ data_extraction_goal=None,
+ navigation_payload=None,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ order=task_order,
+ retry=task_retry,
+ )
+ task = await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.running,
+ )
+ step = await app.DATABASE.tasks.create_step(
+ task.task_id,
+ order=0,
+ retry_index=0,
+ organization_id=organization_id,
+ )
+ return task, step
+
async def create_task(self, task_request: TaskRequest, organization_id: str) -> Task:
webhook_callback_url = str(task_request.webhook_callback_url) if task_request.webhook_callback_url else None
totp_verification_url = str(task_request.totp_verification_url) if task_request.totp_verification_url else None
diff --git a/skyvern/forge/prompts/skyvern/workflow-copilot-code-block-steps.j2 b/skyvern/forge/prompts/skyvern/workflow-copilot-code-block-steps.j2
new file mode 100644
index 000000000..9555b11e5
--- /dev/null
+++ b/skyvern/forge/prompts/skyvern/workflow-copilot-code-block-steps.j2
@@ -0,0 +1,21 @@
+You are labeling a Playwright/SkyvernPage code block for a non-technical reader.
+
+You are given the block's goal and an ordered list of detected actions. Each action already has a
+fixed `action_type` and the code line(s) it covers. Rewrite ONLY the `description` of each action into
+a short, plain-language sentence (max ~8 words, imperative, no code, no selectors-as-jargon). Do not
+change `action_type`, `line_start`, or `line_end`, and do not add or remove actions.
+
+Goal: {{ goal }}
+
+Code:
+```python
+{{ code }}
+```
+
+Actions (JSON):
+```json
+{{ actions_json }}
+```
+
+Return ONLY a JSON array of objects with keys `line_start` and `description`, one per input action,
+in the same order. Example: [{"line_start": 2, "description": "Open the homepage"}]
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 04db6f3d5..8c9d1b042 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -541,7 +541,8 @@ def _synthesized_block_offer_prompt(ctx: CopilotContext | None) -> str:
trajectory_len=len(ctx.scout_trajectory),
code_len=len(synthesized.code),
)
- return "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory)
+ goal = ctx.block_goal_main_goal or ctx.user_message or ""
+ return "\n\n" + render_synthesized_offer_text(synthesized, ctx.scout_trajectory, goal=goal)
def _build_dynamic_system_prompt(tool_usage_guide: str, config: CopilotConfig) -> Callable[[object, object], str]:
@@ -3036,6 +3037,7 @@ async def _run_copilot_turn_impl(
prior_copilot_workflow_yaml=prior_copilot_workflow_yaml,
block_authoring_policy=copilot_config.block_authoring_policy,
impose_synthesized_code_block=copilot_config.impose_synthesized_code_block,
+ target_block_label=getattr(chat_request, "target_block_label", None),
)
# Fail loud if a future caller skips the kwarg and gets a fresh UUID from
# the default_factory — the envelope and terminal frames would then carry
@@ -3210,10 +3212,21 @@ async def _run_copilot_turn_impl(
if ctx.turn_context_packet.repeated_reply_context is not None:
repeated_reply_warning = ctx.turn_context_packet.repeated_reply_context.rendered_summary
+ scoped_global_llm_context = safe_global_llm_context
+ if ctx.target_block_label:
+ # Defang the user-supplied label before embedding it in the instruction: collapse
+ # whitespace and drop quotes so it can't break out of the string or inject directives.
+ safe_target_block_label = re.sub(r"\s+", " ", ctx.target_block_label).replace('"', "").strip()[:200]
+ scoped_global_llm_context = (
+ f'CRITICAL: Regenerate ONLY the block labeled "{safe_target_block_label}". '
+ "Preserve every other block's code, goal, steps, and configuration exactly as-is.\n\n"
+ f"{safe_global_llm_context}"
+ )
+
user_message = _build_user_context(
workflow_yaml=safe_workflow_yaml,
chat_history_text=safe_chat_history_text,
- global_llm_context=safe_global_llm_context,
+ global_llm_context=scoped_global_llm_context,
debug_run_info_text=redact_raw_secrets_for_prompt(debug_run_info_text),
user_message=agent_user_message,
user_workflow_change_summary=user_workflow_change_summary,
diff --git a/skyvern/forge/sdk/copilot/code_block_steps.py b/skyvern/forge/sdk/copilot/code_block_steps.py
new file mode 100644
index 000000000..05311eec7
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/code_block_steps.py
@@ -0,0 +1,333 @@
+from __future__ import annotations
+
+import ast
+import asyncio
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Iterator
+
+import structlog
+import yaml
+
+from skyvern.config import settings
+from skyvern.forge.prompts import prompt_engine
+
+LOG = structlog.get_logger()
+
+# Playwright/SkyvernPage method name -> step action_type (values are ActionType members;
+# kept as string literals so this module does not import skyvern.webeye, matching code_block_synthesis.py).
+_METHOD_ACTION_TYPES: dict[str, str] = {
+ "goto": "goto_url",
+ "click": "click",
+ "dblclick": "click",
+ "check": "click",
+ "uncheck": "click",
+ "tap": "click",
+ "fill": "input_text",
+ "type": "input_text",
+ "press": "keypress",
+ "press_sequentially": "input_text",
+ "select_option": "select_option",
+ "set_input_files": "upload_file",
+ "go_back": "go_back",
+ "reload": "reload_page",
+ "wait_for_timeout": "wait",
+}
+# Awaited calls that are sync/no-op helpers — never surfaced as their own step.
+_IGNORED_METHODS: frozenset[str] = frozenset(
+ {"wait_for_load_state", "wait_for_selector", "wait_for_url", "wait_for_function"}
+)
+
+_STRING_LITERAL = re.compile(r"""^\s*['"](.*)['"]\s*$""", re.DOTALL)
+_NAME_KWARG = re.compile(r"""name\s*=\s*['"]([^'"]+)['"]""")
+
+
+@dataclass
+class CodeActionSpan:
+ action_type: str
+ line_start: int
+ line_end: int
+ method: str
+ receiver: str # source of the call receiver, e.g. "page" or "page.get_by_role('link', name='Login')"
+ first_arg: str | None # source of the first call arg, if any
+
+
+def analyze_code_actions(code: str) -> list[CodeActionSpan]:
+ """Find browser-action calls in `code` and map each to an action_type + exact line range."""
+ if not code or not code.strip():
+ return []
+ try:
+ tree = ast.parse(code)
+ except SyntaxError:
+ return []
+
+ spans: list[CodeActionSpan] = []
+ for node in ast.walk(tree):
+ if not isinstance(node, ast.Await) or not isinstance(node.value, ast.Call):
+ continue
+ call = node.value
+ if not isinstance(call.func, ast.Attribute):
+ continue
+ method = call.func.attr
+ if method in _IGNORED_METHODS:
+ continue
+ action_type = _METHOD_ACTION_TYPES.get(method)
+ if action_type is None:
+ continue
+ spans.append(
+ CodeActionSpan(
+ action_type=action_type,
+ line_start=node.lineno,
+ line_end=getattr(node, "end_lineno", None) or node.lineno,
+ method=method,
+ receiver=_safe_unparse(call.func.value),
+ first_arg=_safe_unparse(call.args[0]) if call.args else None,
+ )
+ )
+ spans.sort(key=lambda s: (s.line_start, s.line_end))
+ return spans
+
+
+def _safe_unparse(node: ast.AST) -> str:
+ try:
+ return ast.unparse(node)
+ except Exception:
+ return ""
+
+
+def _string_value(arg: str | None) -> str | None:
+ """Return the inner text of a simple string-literal call argument, else None."""
+ if not arg:
+ return None
+ m = _STRING_LITERAL.match(arg)
+ return m.group(1) if m else None
+
+
+def _target_label(receiver: str, first_arg: str | None) -> str:
+ """A short human label for the element a call acts on."""
+ # Prefer an accessible name from get_by_role/get_by_label/get_by_text(name=... or "literal").
+ name_kwarg = _NAME_KWARG.search(receiver)
+ if name_kwarg:
+ return f'"{name_kwarg.group(1)}"'
+ for getter in ("get_by_label", "get_by_text", "get_by_placeholder"):
+ idx = receiver.find(f"{getter}(")
+ if idx != -1:
+ inner = receiver[idx + len(getter) + 1 :]
+ lit = _string_value(inner.split(",", 1)[0].rstrip(")"))
+ if lit:
+ return f'"{lit}"'
+ return "the element"
+
+
+def _describe(span: CodeActionSpan) -> str:
+ value = _string_value(span.first_arg)
+ if span.action_type == "goto_url":
+ return f"Open {value}" if value else "Open the page"
+ if span.action_type == "click":
+ return f"Click {_target_label(span.receiver, span.first_arg)}"
+ if span.action_type == "input_text":
+ return f"Type into {_target_label(span.receiver, span.first_arg)}"
+ if span.action_type == "select_option":
+ target = _target_label(span.receiver, span.first_arg)
+ return f"Select {value} in {target}" if value else f"Select an option in {target}"
+ if span.action_type == "keypress":
+ return f"Press {value}" if value else "Press a key"
+ if span.action_type == "upload_file":
+ return f"Upload a file to {_target_label(span.receiver, span.first_arg)}"
+ if span.action_type == "wait":
+ return "Wait"
+ if span.action_type == "go_back":
+ return "Go back"
+ if span.action_type == "reload_page":
+ return "Reload the page"
+ return "Run a step"
+
+
+def derive_code_block_steps(code: str, goal: str | None = None) -> list[dict[str, Any]]:
+ """Derive the ordered plain-language steps for a code block from its code (deterministic)."""
+ return [
+ {
+ "description": _describe(span),
+ "action_type": span.action_type,
+ "line_start": span.line_start,
+ "line_end": span.line_end,
+ }
+ for span in analyze_code_actions(code)
+ ]
+
+
+def _iter_code_block_dicts(node: Any) -> Iterator[dict[str, Any]]:
+ """Yield every code-block dict anywhere in the workflow structure (handles nested loop_blocks)."""
+ if isinstance(node, dict):
+ if node.get("block_type") == "code" and isinstance(node.get("code"), str):
+ yield node
+ for value in node.values():
+ yield from _iter_code_block_dicts(value)
+ elif isinstance(node, list):
+ for item in node:
+ yield from _iter_code_block_dicts(item)
+
+
+_STEP_DESC_PROMPT = "workflow-copilot-code-block-steps"
+_STEP_DESC_TIMEOUT_SECONDS = 20
+
+
+async def refine_step_descriptions(
+ code: str,
+ goal: str | None,
+ steps: list[dict[str, Any]],
+ handler: Any | None,
+) -> list[dict[str, Any]]:
+ """Rewrite only the `description` of each step via an LLM; return steps unchanged on any failure."""
+ if not steps or handler is None:
+ return steps
+ actions_json = json.dumps([{"line_start": s["line_start"], "action_type": s["action_type"]} for s in steps])
+ try:
+ prompt = prompt_engine.load_prompt(
+ template=_STEP_DESC_PROMPT,
+ goal=(goal or "").strip(),
+ code=code,
+ actions_json=actions_json,
+ )
+ raw = await asyncio.wait_for(
+ handler(prompt=prompt, prompt_name=_STEP_DESC_PROMPT),
+ timeout=_STEP_DESC_TIMEOUT_SECONDS,
+ )
+ except Exception as exc:
+ LOG.info("code-block step description pass failed; keeping templated descriptions", error=str(exc))
+ return steps
+
+ overrides = _parse_description_overrides(raw)
+ if not overrides:
+ return steps
+ return [{**s, "description": overrides.get(s["line_start"], s["description"])} for s in steps]
+
+
+def _parse_description_overrides(raw: Any) -> dict[int, str]:
+ """Parse the LLM response into {line_start: description}; tolerant of dict/str/list shapes."""
+ payload = raw
+ if isinstance(raw, dict):
+ payload = raw.get("content") or raw.get("response") or raw
+ if isinstance(payload, str):
+ try:
+ payload = json.loads(payload)
+ except json.JSONDecodeError:
+ return {}
+ if not isinstance(payload, list):
+ return {}
+ result: dict[int, str] = {}
+ for item in payload:
+ if not isinstance(item, dict):
+ continue
+ raw_line = item.get("line_start")
+ # LLMs occasionally stringify the integer; accept "2" as 2 but reject bools/non-numerics.
+ if isinstance(raw_line, bool) or not isinstance(raw_line, (int, str)):
+ continue
+ try:
+ line_start = int(raw_line)
+ except (TypeError, ValueError):
+ continue
+ description = str(item.get("description") or "").strip()
+ if description:
+ result[line_start] = description
+ return result
+
+
+def derive_code_block_steps_in_yaml(workflow_yaml: str) -> str:
+ """Return workflow_yaml with each code block's `steps` filled from its `code` when absent.
+
+ Deterministic, sync, and refinement-safe: a block that already carries steps
+ (e.g. an LLM description pass) is left untouched, so this can run on the shared
+ YAML->Workflow seam without clobbering richer steps produced upstream."""
+ try:
+ data = yaml.safe_load(workflow_yaml)
+ except yaml.YAMLError:
+ return workflow_yaml
+ if not isinstance(data, (dict, list)):
+ return workflow_yaml
+
+ changed = False
+ for block in _iter_code_block_dicts(data):
+ if block.get("steps"):
+ continue
+ block["steps"] = derive_code_block_steps(block["code"], block.get("prompt"))
+ changed = True
+
+ if not changed:
+ return workflow_yaml
+ return yaml.safe_dump(data, sort_keys=False)
+
+
+def fill_code_block_prompts_in_yaml(
+ workflow_yaml: str,
+ *,
+ prior_yaml: str | None = None,
+ fallback_goals: dict[str, str] | None = None,
+) -> str:
+ """Return workflow_yaml with each code block's `prompt` (goal) filled when absent.
+
+ The editor treats a code block as code-first (plain view + steps) only when it
+ carries a `prompt`; the model authors the goal as artifact `declared_goal`, not on
+ the block, and code regeneration replaces the whole block YAML and drops it. Prefer
+ the prior block's prompt by label (exact user text, preserved across regen), then a
+ fallback goal by label (e.g. the model's `declared_goal`)."""
+ try:
+ data = yaml.safe_load(workflow_yaml)
+ except yaml.YAMLError:
+ return workflow_yaml
+ if not isinstance(data, (dict, list)):
+ return workflow_yaml
+
+ prior_prompts: dict[str, str] = {}
+ if prior_yaml:
+ try:
+ prior_data = yaml.safe_load(prior_yaml)
+ except yaml.YAMLError:
+ prior_data = None
+ if isinstance(prior_data, (dict, list)):
+ for block in _iter_code_block_dicts(prior_data):
+ label = block.get("label")
+ prompt = block.get("prompt")
+ if isinstance(label, str) and isinstance(prompt, str) and prompt:
+ prior_prompts[label] = prompt
+
+ fallback_goals = fallback_goals or {}
+ changed = False
+ for block in _iter_code_block_dicts(data):
+ if block.get("prompt"):
+ continue
+ label = block.get("label")
+ if not isinstance(label, str):
+ continue
+ goal = prior_prompts.get(label) or fallback_goals.get(label)
+ if goal:
+ block["prompt"] = goal
+ changed = True
+
+ if not changed:
+ return workflow_yaml
+ return yaml.safe_dump(data, sort_keys=False)
+
+
+async def apply_derived_code_block_steps(workflow_yaml: str, handler: Any | None = None) -> str:
+ """Return workflow_yaml with each code block's `steps` recomputed from its `code`."""
+ try:
+ data = yaml.safe_load(workflow_yaml)
+ except yaml.YAMLError:
+ return workflow_yaml
+ if not isinstance(data, (dict, list)):
+ return workflow_yaml
+
+ use_llm = settings.WORKFLOW_COPILOT_CODE_BLOCK_STEP_DESCRIPTIONS_LLM and handler is not None
+ changed = False
+ for block in _iter_code_block_dicts(data):
+ steps = derive_code_block_steps(block["code"], block.get("prompt"))
+ if use_llm:
+ steps = await refine_step_descriptions(block["code"], block.get("prompt"), steps, handler)
+ block["steps"] = steps
+ changed = True
+
+ if not changed:
+ return workflow_yaml
+ return yaml.safe_dump(data, sort_keys=False)
diff --git a/skyvern/forge/sdk/copilot/code_block_synthesis.py b/skyvern/forge/sdk/copilot/code_block_synthesis.py
index 12af9db4c..69af890f4 100644
--- a/skyvern/forge/sdk/copilot/code_block_synthesis.py
+++ b/skyvern/forge/sdk/copilot/code_block_synthesis.py
@@ -17,6 +17,7 @@ from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from skyvern.forge.sdk.copilot.composition_evidence import SCOUT_INTERACTION_EVIDENCE_TOOL
+from skyvern.utils.strings import escape_code_fences
_MAX_STEPS = 60
_INDENT = " "
@@ -91,6 +92,7 @@ class SynthesizedCodeBlock:
parameters: list[dict[str, str]] = field(default_factory=list)
notes: list[str] = field(default_factory=list)
diagnostics: SynthesisDiagnostics = field(default_factory=SynthesisDiagnostics)
+ steps: list[dict[str, Any]] = field(default_factory=list)
# str.splitlines() and several parsers treat these codepoints as line boundaries, so a raw one in a
@@ -280,6 +282,17 @@ def _unique_key(base: str, used: set[str]) -> str:
return candidate
+def _step_target(interaction: Mapping[str, Any]) -> str:
+ """Plain-language anchor for a step description: accessible name, else selector, else role."""
+ name = str(interaction.get("accessible_name") or "").strip()
+ if name:
+ return name
+ selector = str(interaction.get("selector") or "").strip()
+ if selector:
+ return selector
+ return str(interaction.get("role") or "").strip() or "the element"
+
+
def _param_key(interaction: Mapping[str, Any], used: set[str]) -> str:
name = str(interaction.get("accessible_name") or "").strip()
role = str(interaction.get("role") or "").strip()
@@ -314,10 +327,22 @@ def synthesize_code_block(
notes: list[str] = []
parameters: list[dict[str, str]] = []
diagnostics = SynthesisDiagnostics()
+ steps: list[dict[str, Any]] = []
used_param_keys: set[str] = set()
typed_param_keys: dict[tuple[str, str, str, str], str] = {}
credential_param_keys: dict[str, str] = {}
+ def append_step(description: str, action_type: str, line_start: int) -> None:
+ steps.append(
+ {
+ "description": description,
+ "action_type": action_type,
+ "line_start": line_start,
+ # last line emitted for this step; append_step always runs before the next step's lines.
+ "line_end": len(lines),
+ }
+ )
+
entry_url = ""
entry_index = -1
for index, interaction in enumerate(trajectory):
@@ -329,8 +354,10 @@ def synthesize_code_block(
if entry_url:
if entry_index > 0:
notes.append("entry URL taken from a later interaction; earlier steps had no source_url")
+ line_start = len(lines) + 1
lines.append(f"{_INDENT}await page.goto({_py_str(_scrub_url_for_code_literal(entry_url))})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ append_step(f"Open {entry_url}", "goto_url", line_start)
emitted = 0
for trajectory_index, interaction in enumerate(trajectory):
@@ -359,6 +386,7 @@ def synthesize_code_block(
if interaction.get("selector")
else ""
)
+ line_start = len(lines) + 1
if locator:
lines.append(f"{_INDENT}await {locator}.press({_py_str(key)})")
else:
@@ -373,6 +401,7 @@ def synthesize_code_block(
continue
lines.append(f"{_INDENT}await page.keyboard.press({_py_str(key)})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ append_step(f"Press {key}", "keypress", line_start)
emitted += 1
continue
@@ -387,9 +416,11 @@ def synthesize_code_block(
if not locator:
continue
+ line_start = len(lines) + 1
if tool_name == "click":
lines.append(f"{_INDENT}await {locator}.click()")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ append_step(f"Click {_step_target(interaction)}", "click", line_start)
elif tool_name == "type_text":
typed_identity = _typed_value_identity(interaction)
param_key = typed_param_keys.get(typed_identity) if typed_identity is not None else None
@@ -403,6 +434,7 @@ def synthesize_code_block(
if typed_identity is not None:
typed_param_keys[typed_identity] = param_key
lines.append(f"{_INDENT}await {locator}.fill(str({param_key}))")
+ append_step(f"Type into {_step_target(interaction)}", "input_text", line_start)
elif tool_name == CREDENTIAL_FILL_TOOL_NAME:
credential_id = str(interaction.get("credential_id") or "").strip()
credential_field = str(interaction.get("credential_field") or "").strip()
@@ -432,6 +464,7 @@ def synthesize_code_block(
continue
lines.append(f"{_INDENT}await {locator}.select_option({_py_str(value)})")
lines.append(f'{_INDENT}await page.wait_for_load_state("load")')
+ append_step(f"Select {value} in {_step_target(interaction)}", "select_option", line_start)
else:
notes.append(f"skipped unsupported interaction tool_name={tool_name!r}")
diagnostics.dropped_interactions.append(
@@ -445,7 +478,7 @@ def synthesize_code_block(
diagnostics.emitted_interaction_count = emitted
code = "\n".join(lines) + "\n"
- return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes, diagnostics=diagnostics)
+ return SynthesizedCodeBlock(code=code, parameters=parameters, notes=notes, diagnostics=diagnostics, steps=steps)
# Model-owned slots the synthesizer cannot prove; the model fills these.
@@ -544,6 +577,7 @@ def _render_artifact_metadata_block(metadata: Mapping[str, Any]) -> str:
def render_synthesized_offer_text(
synthesized: SynthesizedCodeBlock,
trajectory: Sequence[Mapping[str, Any]] | None = None,
+ goal: str | None = None,
) -> str:
"""Render the offer body the copilot sees for a synthesized block (pure)."""
param_keys = [p.get("key", "") for p in synthesized.parameters if p.get("key") and not p.get("credential_id")]
@@ -581,6 +615,17 @@ def render_synthesized_offer_text(
)
if synthesized.notes:
parts.append("Synthesis notes: " + "; ".join(synthesized.notes) + ".")
+ if synthesized.steps:
+ # escape_quotes + whitespace collapse keep the goal inside its quoted span in the prompt.
+ goal_text = " ".join(escape_code_fences(goal or "", escape_quotes=True).split())
+ goal_part = f' and set the block\'s `prompt` field to "{goal_text}"' if goal_text else ""
+ parts.append(
+ "On the same block, set the `steps` field to the JSON list below VERBATIM (plain-language "
+ f"annotations mapping each step to the code lines it covers){goal_part}."
+ )
+ parts.append("```json")
+ parts.append(json.dumps(synthesized.steps, indent=2, sort_keys=True))
+ parts.append("```")
if trajectory:
metadata = build_synthesized_artifact_metadata(trajectory)
parts.append(
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index e2c22c8ae..086adc7a8 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -370,6 +370,7 @@ class CopilotContext(AgentContext):
request_policy: RequestPolicy | None = None
block_authoring_policy: BlockAuthoringPolicy = BlockAuthoringPolicy.STANDARD
impose_synthesized_code_block: bool = False
+ target_block_label: str | None = None
turn_intent: TurnIntent | None = None
turn_context_packet: TurnContextPacket | None = None
latest_diagnosis_repair_contract: DiagnosisRepairContract | None = None
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 0f6e2f8f6..ca57552ae 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -1578,7 +1578,8 @@ def _maybe_synthesized_block_offer_msg(ctx: Any) -> dict[str, Any] | None:
return None
ctx.synthesized_block_offered = True
- return {"role": "user", "content": render_synthesized_offer_text(synthesized, trajectory)}
+ goal = getattr(ctx, "block_goal_main_goal", "") or getattr(ctx, "user_message", "") or ""
+ return {"role": "user", "content": render_synthesized_offer_text(synthesized, trajectory, goal=goal)}
def _assemble_enforcement_messages(
diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.py
index 73a9fcb6d..cfa67f362 100644
--- a/skyvern/forge/sdk/copilot/model_resolver.py
+++ b/skyvern/forge/sdk/copilot/model_resolver.py
@@ -25,6 +25,7 @@ from agents.models.interface import Model
from agents.run_config import RunConfig
from skyvern.config import settings
+from skyvern.forge.sdk.api.llm.api_handler_factory import LLMAPIHandlerFactory
from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMConfigError
from skyvern.forge.sdk.api.llm.litellm_transport import configure_litellm_transport
@@ -162,6 +163,13 @@ def resolve_model_config(
configure_litellm_transport()
llm_key = llm_key_override or getattr(llm_api_handler, "llm_key", None) or settings.LLM_KEY
+ # GitHub Copilot's OPENAI_COMPATIBLE handler rewrites .llm_key to the bare model name; restore the registry key.
+ if (
+ llm_key == settings.OPENAI_COMPATIBLE_MODEL_NAME
+ and LLMAPIHandlerFactory.is_github_copilot_endpoint()
+ and not LLMConfigRegistry.is_registered(llm_key)
+ ):
+ llm_key = settings.OPENAI_COMPATIBLE_MODEL_KEY
config = LLMConfigRegistry.get_config(llm_key)
if isinstance(config, LLMRouterConfig):
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 8d8c12a1b..37948deb8 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -12,10 +12,12 @@ import structlog
import yaml
from pydantic import AliasChoices, BaseModel, Field, ValidationError
+from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.copilot.attribution import resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.blocker_signal import clear_terminal_evidence_on_workflow_edit
from skyvern.forge.sdk.copilot.code_block_preflight import sandbox_unresolved_name_diagnostics
+from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, fill_code_block_prompts_in_yaml
from skyvern.forge.sdk.copilot.code_block_synthesis import (
artifact_dependency_id,
artifact_observation_ref_id,
@@ -34,6 +36,7 @@ from skyvern.forge.sdk.copilot.enforcement import (
_completion_contract_unknown_due_to_policy_fallback,
_goal_likely_needs_more_blocks,
)
+from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.loop_detection import clear_failed_step_tracker_for_tools_in_ctx
from skyvern.forge.sdk.copilot.outcome_verification_trace import record_code_artifact_violations
from skyvern.forge.sdk.copilot.output_policy import (
@@ -1961,11 +1964,29 @@ async def _update_workflow(
return {"ok": False, "error": composition_evidence_error}
try:
+ # A code block renders code-first (goal + plain step timeline) only when it
+ # carries a `prompt`; the model authors the goal as artifact `declared_goal`
+ # and code regeneration drops it, so carry the goal onto the block here.
+ artifact_metadata = getattr(ctx, "code_artifact_metadata", None)
+ fallback_goals = {
+ label: str(meta["declared_goal"]).strip()
+ for label, meta in (artifact_metadata or {}).items()
+ if isinstance(meta, dict) and str(meta.get("declared_goal") or "").strip()
+ }
+ workflow_yaml = fill_code_block_prompts_in_yaml(
+ workflow_yaml, prior_yaml=prior_yaml, fallback_goals=fallback_goals
+ )
+ # Derive plain-language steps from each code block's code so the editor timeline
+ # mirrors the actual code (deterministic action_type + line ranges).
+ steps_handler = None
+ if settings.WORKFLOW_COPILOT_CODE_BLOCK_STEP_DESCRIPTIONS_LLM:
+ steps_handler = await resolve_main_copilot_handler(ctx.workflow_permanent_id, ctx.organization_id)
+ workflow_yaml_with_steps = await apply_derived_code_block_steps(workflow_yaml, handler=steps_handler)
workflow = _process_workflow_yaml(
workflow_id=ctx.workflow_id,
workflow_permanent_id=ctx.workflow_permanent_id,
organization_id=ctx.organization_id,
- workflow_yaml=workflow_yaml,
+ workflow_yaml=workflow_yaml_with_steps,
)
_record_workflow_proxy_location_span(workflow_yaml, workflow)
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index e13fd3ebb..4189f6a61 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -26,6 +26,7 @@ from skyvern.forge.sdk.artifact.models import Artifact, ArtifactType, LogEntityT
from skyvern.forge.sdk.copilot.agent import run_copilot_agent
from skyvern.forge.sdk.copilot.attribution import is_copilot_born_initial_write, resolve_copilot_created_by_stamp
from skyvern.forge.sdk.copilot.block_type_aliases import normalize_copilot_block_type_alias
+from skyvern.forge.sdk.copilot.code_block_steps import apply_derived_code_block_steps, derive_code_block_steps_in_yaml
from skyvern.forge.sdk.copilot.completion_criteria_store import (
CRITERIA_SET_STATUS_ACTIVE,
StoredCriteriaSet,
@@ -1372,6 +1373,11 @@ def _process_workflow_yaml(
organization_id: str,
workflow_yaml: str,
) -> Workflow:
+ # Single seam every copilot YAML->Workflow conversion passes through, so code
+ # blocks get their plain-view steps regardless of which path produced the YAML
+ # (the update_workflow tool derives them upstream; the inline REPLACE_WORKFLOW
+ # fallbacks would otherwise surface "No steps yet").
+ workflow_yaml = derive_code_block_steps_in_yaml(workflow_yaml)
workflow_yaml_request = _normalize_copilot_yaml(workflow_yaml)
updated_workflow_definition = convert_workflow_definition(
@@ -2411,6 +2417,8 @@ async def workflow_copilot_apply_proposed_workflow(
detail="Proposed workflow has no copilot YAML to apply",
)
+ copilot_yaml = await apply_derived_code_block_steps(copilot_yaml)
+
try:
yaml_request = _normalize_copilot_yaml(copilot_yaml)
except (yaml.YAMLError, ValidationError) as e:
diff --git a/skyvern/forge/sdk/schemas/workflow_copilot.py b/skyvern/forge/sdk/schemas/workflow_copilot.py
index 1934b3897..06766809f 100644
--- a/skyvern/forge/sdk/schemas/workflow_copilot.py
+++ b/skyvern/forge/sdk/schemas/workflow_copilot.py
@@ -108,6 +108,13 @@ class WorkflowCopilotChatRequest(BaseModel):
"Optional; legacy clients omit it and cancel becomes a no-op for those requests."
),
)
+ target_block_label: str | None = Field(
+ None,
+ description=(
+ "When set, the copilot regenerates only this code block from its goal and leaves every "
+ "other block unchanged. Used by the block-level Generate action."
+ ),
+ )
class WorkflowCopilotCancelRequest(BaseModel):
diff --git a/skyvern/forge/sdk/workflow/exceptions.py b/skyvern/forge/sdk/workflow/exceptions.py
index f3d743970..8d43fe174 100644
--- a/skyvern/forge/sdk/workflow/exceptions.py
+++ b/skyvern/forge/sdk/workflow/exceptions.py
@@ -157,6 +157,14 @@ class WorkflowDefinitionHasUndefinedParameters(WorkflowDefinitionValidationExcep
)
+class InvalidCodeBlockStep(WorkflowDefinitionValidationException):
+ def __init__(self, block_label: str, step_index: int, detail: str) -> None:
+ super().__init__(
+ f"Invalid step at index {step_index} in code block '{block_label}': {detail}",
+ status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
+ )
+
+
class InvalidWaitBlockTime(SkyvernException):
def __init__(self, max_sec: int) -> None:
super().__init__(f"Invalid wait time for wait block, it should be a number between 0 and {max_sec}.")
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index 57afea01b..d928e2a9d 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -86,7 +86,7 @@ from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.db.exceptions import NotFoundError
from skyvern.forge.sdk.experimentation.llm_prompt_config import get_llm_handler_for_prompt_type
-from skyvern.forge.sdk.models import Step
+from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.task_v2 import TaskV2Status
from skyvern.forge.sdk.schemas.tasks import Task, TaskOutput, TaskStatus
@@ -121,6 +121,11 @@ from skyvern.forge.sdk.workflow.models._jinja import (
_json_type_filter,
jinja_json_finalize_strict_env,
)
+from skyvern.forge.sdk.workflow.models.code_block_recorder import (
+ CODE_BLOCK_FILENAME,
+ RecordingPage,
+ user_code_line_from_exception,
+)
from skyvern.forge.sdk.workflow.models.parameter import (
PARAMETER_TYPE,
AWSSecretParameter,
@@ -131,6 +136,7 @@ from skyvern.forge.sdk.workflow.models.parameter import (
WorkflowParameterType,
)
from skyvern.schemas.runs import RunEngine
+from skyvern.schemas.steps import AgentStepOutput
from skyvern.schemas.workflows import (
AIFallbackMode,
BlockResult,
@@ -146,6 +152,8 @@ from skyvern.utils.strings import generate_random_string
from skyvern.utils.templating import get_missing_variables
from skyvern.utils.token_counter import count_tokens
from skyvern.utils.url_validators import prepend_scheme_and_validate_url
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action, ActionStatus
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.utils.page import SkyvernFrame
@@ -3351,6 +3359,14 @@ class Credential(SimpleNamespace):
pass
+class CodeBlockStep(BaseModel):
+ title: str | None = None
+ description: str | None = None
+ action_type: ActionType = ActionType.NULL_ACTION
+ line_start: int | None = None
+ line_end: int | None = None
+
+
class CodeBlockOTPError(Exception):
"""Sanitized OTP-primitive error: never includes the identifier, URL, code, or seed."""
@@ -3458,6 +3474,8 @@ class CodeBlock(Block):
code: str
parameters: list[PARAMETER_TYPE] = []
+ prompt: str | None = None
+ steps: list[CodeBlockStep] | None = None
# Dangerous attribute names that must never be accessed in user code.
# This blocks subprocess creation, OS access, and sandbox-escape primitives.
@@ -3598,7 +3616,7 @@ class CodeBlock(Block):
}
def generate_async_user_function(
- self, code: str, page: Page, parameters: dict[str, Any] | None = None
+ self, code: str, page: Page | RecordingPage, parameters: dict[str, Any] | None = None
) -> Callable[[], Awaitable[dict[str, Any]]]:
# SECURITY: validate before exec(). The AST check must run on the raw
# user code so it can block dunder identifiers like __capture_locals.
@@ -3622,7 +3640,9 @@ async def wrapper({default_args}):
safe_vars["page"] = page
safe_vars["__capture_locals"] = locals
safe_vars["__param_defaults"] = parameter_defaults
- exec(full_code, safe_vars, runtime_variables)
+ # Compile under a recognizable filename so tracebacks map back to user code lines.
+ compiled_code = compile(full_code, CODE_BLOCK_FILENAME, "exec")
+ exec(compiled_code, safe_vars, runtime_variables) # nosemgrep
user_function = runtime_variables["wrapper"]
if not parameter_defaults:
return user_function
@@ -3657,6 +3677,10 @@ async def wrapper({default_args}):
def format_potential_template_parameters(self, workflow_run_context: WorkflowRunContext) -> None:
self.code = self.format_block_parameter_template_from_workflow_run_context(self.code, workflow_run_context)
+ if self.prompt:
+ self.prompt = self.format_block_parameter_template_from_workflow_run_context(
+ self.prompt, workflow_run_context
+ )
async def _register_downloaded_files(
self,
@@ -3860,13 +3884,136 @@ async def wrapper({default_args}):
secret_value = workflow_run_context.get_original_secret_value_or_none(value)
parameter_values[parameter.key] = secret_value if secret_value is not None else value
+ workflow_run_block = None
+
+ # A prompt-bearing code block gets a task v1 + step so its recorded calls render through
+ # the standard action/artifact timeline and the agent can later take over on failure.
+ # Promptless blocks have no task and persist neither actions nor screenshots.
+ task: Task | None = None
+ step: Step | None = None
+ if self.prompt:
+ task, step = await app.agent.create_task_and_step_from_code_block(
+ code_block=self,
+ organization_id=organization_id,
+ workflow_run_id=workflow_run_id,
+ task_url=page.url,
+ )
+
+ screenshot_tasks: list[asyncio.Task[None]] = []
+
+ async def _screenshot_sink(action: Action) -> None:
+ # No task means no action row will reference the screenshot, so skip it rather than orphan an artifact.
+ nonlocal workflow_run_block
+ if task is None:
+ return
+ if action.action_type not in (ActionType.GOTO_URL, ActionType.CLICK, ActionType.INPUT_TEXT):
+ return
+ # page.screenshot() shares the CDP channel with the user's page calls, so it must run synchronously
+ # in the user-await chain (a backgrounded capture races the next action and clips a mid-nav frame);
+ # only the page-free S3 upload is deferred off the critical path.
+ try:
+ if workflow_run_block is None:
+ workflow_run_block = await app.DATABASE.observer.get_workflow_run_block(
+ workflow_run_block_id=workflow_run_block_id, organization_id=organization_id
+ )
+ run_block = workflow_run_block
+ screenshot = await page.screenshot(timeout=settings.BROWSER_SCREENSHOT_TIMEOUT_MS)
+ except Exception:
+ LOG.warning(
+ "Code block screenshot capture failed",
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+ return
+
+ async def _upload() -> None:
+ try:
+ action.screenshot_artifact_id = await app.ARTIFACT_MANAGER.create_workflow_run_block_artifact(
+ workflow_run_block=run_block,
+ artifact_type=ArtifactType.SCREENSHOT_ACTION,
+ data=screenshot,
+ )
+ except Exception:
+ LOG.warning(
+ "Code block screenshot upload failed",
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+
+ screenshot_tasks.append(asyncio.create_task(_upload()))
+
+ async def _drain_screenshots() -> None:
+ if screenshot_tasks:
+ await asyncio.gather(*screenshot_tasks, return_exceptions=True)
+
+ recording_page = RecordingPage(page, on_action=_screenshot_sink)
+
+ async def _persist_recorded_actions(recorded: list[Action]) -> None:
+ # Best-effort like the screenshot sink: recording must never change block outcome.
+ await _drain_screenshots()
+ if not recorded or task is None or step is None:
+ return
+ try:
+ masked = workflow_run_context.mask_secrets_in_data([a.model_dump(mode="json") for a in recorded])
+ for raw in masked:
+ action = Action.model_validate(raw)
+ action.task_id = task.task_id
+ action.step_id = step.step_id
+ action.step_order = step.order
+ action.organization_id = organization_id
+ await app.DATABASE.workflow_params.create_action(action)
+ except Exception:
+ LOG.warning(
+ "Failed to persist recorded code block actions",
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+
+ finalized = False
+
+ async def _finalize_code_block_task(success: bool) -> None:
+ # Finalize both task and step on every exit path (incl. CancelledError via the finally); idempotent.
+ nonlocal finalized
+ if task is None or finalized:
+ return
+ finalized = True
+ try:
+ await app.DATABASE.tasks.update_task(
+ task_id=task.task_id,
+ organization_id=organization_id,
+ status=TaskStatus.completed if success else TaskStatus.failed,
+ )
+ if step is not None:
+ await app.DATABASE.tasks.update_step(
+ task_id=task.task_id,
+ step_id=step.step_id,
+ status=StepStatus.completed if success else StepStatus.failed,
+ output=AgentStepOutput(action_results=[]) if success else None,
+ is_last=True,
+ organization_id=organization_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to finalize code block task status",
+ workflow_run_block_id=workflow_run_block_id,
+ exc_info=True,
+ )
+
try:
- user_function = self.generate_async_user_function(self.code, page, parameter_values)
+ if task is not None:
+ await app.DATABASE.observer.update_workflow_run_block(
+ workflow_run_block_id=workflow_run_block_id,
+ task_id=task.task_id,
+ organization_id=organization_id,
+ )
+ user_function = self.generate_async_user_function(self.code, recording_page, parameter_values)
result = await self.execute_user_function_with_timeout(
user_function,
settings.CODE_BLOCK_EXECUTION_TIMEOUT_SECONDS,
)
except InsecureCodeDetected as e:
+ await _drain_screenshots()
+ await _finalize_code_block_task(success=False)
return await self.build_block_result(
success=False,
failure_reason=str(e),
@@ -3876,6 +4023,8 @@ async def wrapper({default_args}):
organization_id=organization_id,
)
except asyncio.TimeoutError:
+ await _persist_recorded_actions(recording_page.recorded_actions())
+ await _finalize_code_block_task(success=False)
return await self.build_block_result(
success=False,
failure_reason=(
@@ -3889,9 +4038,25 @@ async def wrapper({default_args}):
)
except Exception as e:
exc = CustomizedCodeException(e)
+ failing_line = user_code_line_from_exception(e)
# User code can raise an exception carrying a resolved secret (e.g.
# `raise Exception(await cred.otp())`); mask before it reaches the persisted reason.
failure_reason = workflow_run_context.mask_secrets_in_data(exc.message)
+ recorded = recording_page.recorded_actions()
+ if recording_page.last_recorded_exception() is not e:
+ # The exception did not come from a recorded page call; add a synthetic failure row.
+ recorded.append(
+ Action(
+ action_type=ActionType.NULL_ACTION,
+ status=ActionStatus.failed,
+ action_order=len(recorded),
+ description=f"code error at line {failing_line}" if failing_line else "code error",
+ response=(failure_reason or "")[:500],
+ output={"code_line": failing_line},
+ )
+ )
+ await _persist_recorded_actions(recorded)
+ await _finalize_code_block_task(success=False)
return await self.build_block_result(
success=False,
failure_reason=failure_reason,
@@ -3901,6 +4066,13 @@ async def wrapper({default_args}):
organization_id=organization_id,
)
+ else:
+ await _persist_recorded_actions(recording_page.recorded_actions())
+ await _finalize_code_block_task(success=True)
+ finally:
+ # Safety net for paths the except arms miss (CancelledError, update_workflow_run_block failure).
+ await _finalize_code_block_task(success=False)
+
result = json.loads(
json.dumps(result, default=lambda value: f"Object '{type(value)}' is not JSON serializable")
)
diff --git a/skyvern/forge/sdk/workflow/models/code_block_recorder.py b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
new file mode 100644
index 000000000..19a7e5ef6
--- /dev/null
+++ b/skyvern/forge/sdk/workflow/models/code_block_recorder.py
@@ -0,0 +1,230 @@
+from __future__ import annotations
+
+import sys
+import time
+from types import FrameType
+from typing import Any, Awaitable, Callable
+
+import structlog
+
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action, ActionStatus
+
+LOG = structlog.get_logger()
+
+CODE_BLOCK_FILENAME = "<code_block>"
+# full_code = "\nasync def wrapper(...):\n<user code from line 3>"; user line = frame line - 2
+CODE_LINE_OFFSET = 2
+
+_PAGE_ACTION_MAP: dict[str, ActionType] = {
+ "goto": ActionType.GOTO_URL,
+ "go_back": ActionType.GO_BACK,
+ "go_forward": ActionType.GO_FORWARD,
+ "reload": ActionType.RELOAD_PAGE,
+}
+_LOCATOR_ACTION_MAP: dict[str, ActionType] = {
+ "click": ActionType.CLICK,
+ "dblclick": ActionType.CLICK,
+ "fill": ActionType.INPUT_TEXT,
+ "type": ActionType.INPUT_TEXT,
+ "press": ActionType.KEYPRESS,
+ "select_option": ActionType.SELECT_OPTION,
+ "check": ActionType.CHECKBOX,
+ "uncheck": ActionType.CHECKBOX,
+ "hover": ActionType.HOVER,
+ "set_input_files": ActionType.UPLOAD_FILE,
+}
+# Sync locator factories on both Page and Locator; their results must stay wrapped.
+_LOCATOR_FACTORY_METHODS = frozenset(
+ {
+ "get_by_role",
+ "get_by_text",
+ "get_by_label",
+ "get_by_placeholder",
+ "get_by_alt_text",
+ "get_by_title",
+ "get_by_test_id",
+ "filter",
+ }
+)
+
+OnAction = Callable[[Action], Awaitable[None]]
+
+
+def _frame_user_line() -> int | None:
+ # Walk f_back instead of inspect.stack(), which reads source for every frame; this runs on
+ # every recorded action, on the await chain that counts against the code block timeout.
+ frame: FrameType | None = sys._getframe()
+ while frame is not None:
+ if frame.f_code.co_filename == CODE_BLOCK_FILENAME:
+ return max(frame.f_lineno - CODE_LINE_OFFSET, 1)
+ frame = frame.f_back
+ return None
+
+
+def user_code_line_from_exception(exc: BaseException) -> int | None:
+ tb = exc.__traceback__
+ line: int | None = None
+ while tb is not None:
+ if tb.tb_frame.f_code.co_filename == CODE_BLOCK_FILENAME:
+ line = max(tb.tb_lineno - CODE_LINE_OFFSET, 1)
+ tb = tb.tb_next
+ return line
+
+
+def _describe(name: str, target: str | None, args: tuple[Any, ...]) -> str:
+ arg = next((str(a) for a in args if isinstance(a, (str, int, float))), None)
+ parts = [name]
+ if target:
+ parts.append(target)
+ if arg is not None and arg != target:
+ parts.append(arg[:200])
+ return " ".join(parts)
+
+
+def _factory_selector(name: str, args: tuple[Any, ...]) -> str:
+ arg = next((str(a) for a in args if isinstance(a, (str, int, float))), None)
+ return f"{name}({arg})" if arg is not None else name
+
+
+class _Recorder:
+ def __init__(self, on_action: OnAction | None = None) -> None:
+ self.actions: list[Action] = []
+ self.last_exception: BaseException | None = None
+ self._on_action = on_action
+
+ async def record(
+ self,
+ action_type: ActionType,
+ name: str,
+ target: str | None,
+ call: Callable[[], Awaitable[Any]],
+ args: tuple[Any, ...],
+ ) -> Any:
+ started = time.monotonic()
+ # Input values may be credentials (incl. derived TOTP codes); never describe them.
+ describe_args = () if action_type == ActionType.INPUT_TEXT else args
+ action = Action(
+ action_type=action_type,
+ status=ActionStatus.completed,
+ action_order=len(self.actions),
+ description=_describe(name, target, describe_args),
+ output={"code_line": _frame_user_line()},
+ )
+ try:
+ result = await call()
+ except BaseException as exc:
+ action.status = ActionStatus.failed
+ action.response = str(exc)[:500]
+ self.last_exception = exc
+ raise
+ finally:
+ duration_ms = int((time.monotonic() - started) * 1000)
+ if isinstance(action.output, dict):
+ action.output["duration_ms"] = duration_ms
+ self.actions.append(action)
+ if self._on_action is not None:
+ try:
+ await self._on_action(action)
+ except Exception:
+ LOG.warning("Code block action sink failed", action_type=action_type, exc_info=True)
+ return result
+
+
+class RecordingLocator:
+ def __init__(self, locator: Any, recorder: _Recorder, selector: str | None) -> None:
+ self.__locator = locator
+ self.__recorder = recorder
+ self.__selector = selector
+
+ def locator(self, selector: str, **kwargs: Any) -> RecordingLocator:
+ return RecordingLocator(self.__locator.locator(selector, **kwargs), self.__recorder, selector)
+
+ @property
+ def first(self) -> RecordingLocator:
+ return RecordingLocator(self.__locator.first, self.__recorder, self.__selector)
+
+ def nth(self, index: int) -> RecordingLocator:
+ return RecordingLocator(self.__locator.nth(index), self.__recorder, self.__selector)
+
+ def __getattr__(self, name: str) -> Any:
+ attr = getattr(self.__locator, name)
+ if name in _LOCATOR_FACTORY_METHODS and callable(attr):
+
+ def factory(*args: Any, **kwargs: Any) -> RecordingLocator:
+ return RecordingLocator(attr(*args, **kwargs), self.__recorder, _factory_selector(name, args))
+
+ return factory
+ action_type = _LOCATOR_ACTION_MAP.get(name)
+ if action_type is None or not callable(attr):
+ return attr
+
+ async def recorded(*args: Any, **kwargs: Any) -> Any:
+ return await self.__recorder.record(
+ action_type, f"locator.{name}", self.__selector, lambda: attr(*args, **kwargs), args
+ )
+
+ return recorded
+
+
+class RecordingKeyboard:
+ def __init__(self, keyboard: Any, recorder: _Recorder) -> None:
+ self.__keyboard = keyboard
+ self.__recorder = recorder
+
+ def __getattr__(self, name: str) -> Any:
+ attr = getattr(self.__keyboard, name)
+ if name != "press" or not callable(attr):
+ return attr
+
+ async def recorded(*args: Any, **kwargs: Any) -> Any:
+ return await self.__recorder.record(
+ ActionType.KEYPRESS, "keyboard.press", None, lambda: attr(*args, **kwargs), args
+ )
+
+ return recorded
+
+
+class RecordingPage:
+ """Proxy around a playwright page that records mapped calls as Actions.
+
+ Name-mangled private state keeps casual user code away from recorder
+ internals, but the mangled `_RecordingPage__*` form is still reachable;
+ treat recordings as telemetry, not a tamper-proof audit trail.
+ """
+
+ def __init__(self, page: Any, on_action: OnAction | None = None) -> None:
+ self.__page = page
+ self.__recorder = _Recorder(on_action)
+
+ def recorded_actions(self) -> list[Action]:
+ return list(self.__recorder.actions)
+
+ def last_recorded_exception(self) -> BaseException | None:
+ return self.__recorder.last_exception
+
+ def locator(self, selector: str, **kwargs: Any) -> RecordingLocator:
+ return RecordingLocator(self.__page.locator(selector, **kwargs), self.__recorder, selector)
+
+ @property
+ def keyboard(self) -> RecordingKeyboard:
+ return RecordingKeyboard(self.__page.keyboard, self.__recorder)
+
+ def __getattr__(self, name: str) -> Any:
+ attr = getattr(self.__page, name)
+ if name in _LOCATOR_FACTORY_METHODS and callable(attr):
+
+ def factory(*args: Any, **kwargs: Any) -> RecordingLocator:
+ return RecordingLocator(attr(*args, **kwargs), self.__recorder, _factory_selector(name, args))
+
+ return factory
+ # Record direct page-level interactions (page.click/fill/press/...) with the same
+ # redaction as the locator path, alongside navigation calls.
+ action_type = _LOCATOR_ACTION_MAP.get(name) or _PAGE_ACTION_MAP.get(name)
+ if action_type is None or not callable(attr):
+ return attr
+
+ async def recorded(*args: Any, **kwargs: Any) -> Any:
+ return await self.__recorder.record(action_type, f"page.{name}", None, lambda: attr(*args, **kwargs), args)
+
+ return recorded
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 31edb2a1f..c97e189b8 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -472,6 +472,8 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
"version",
"model",
]
+ # `steps` is a plain-language annotation, not execution input, so editing it must not bust the cached script.
+ code_block_annotation_fields = ("steps",)
# Use BFS to recursively remove fields from all nested objects
@@ -486,6 +488,9 @@ def _get_workflow_definition_core_data(workflow_definition: WorkflowDefinition)
for field in fields_to_remove:
if field: # Skip empty string
current_obj.pop(field, None)
+ if current_obj.get("block_type") == BlockType.CODE.value:
+ for field in code_block_annotation_fields:
+ current_obj.pop(field, None)
# Add all nested dictionaries and lists to queue for processing
for value in current_obj.values():
diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
index 839e8de21..3d4a72ca9 100644
--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py
+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py
@@ -3,6 +3,7 @@ from datetime import UTC, datetime
from typing import Any, cast
import structlog
+from pydantic import ValidationError
from skyvern.config import settings
from skyvern.constants import DEFAULT_LOGIN_PROMPT
@@ -20,6 +21,7 @@ from skyvern.forge.sdk.db.id import (
)
from skyvern.forge.sdk.workflow.exceptions import (
ContextParameterSourceNotDefined,
+ InvalidCodeBlockStep,
InvalidWaitBlockTime,
InvalidWorkflowDefinition,
WorkflowDefinitionHasDuplicateBlockLabels,
@@ -33,6 +35,7 @@ from skyvern.forge.sdk.workflow.models.block import (
BlockTypeVar,
BranchCondition,
CodeBlock,
+ CodeBlockStep,
ConditionalBlock,
DownloadToS3Block,
ExtractionBlock,
@@ -510,10 +513,26 @@ def block_yaml_to_block(
branch_conditions=branch_conditions,
)
elif block_yaml.block_type == BlockType.CODE:
+ code_block_steps: list[CodeBlockStep] | None = None
+ if block_yaml.steps:
+ code_block_steps = []
+ for step_index, step_yaml in enumerate(block_yaml.steps):
+ try:
+ code_block_steps.append(CodeBlockStep(**step_yaml.model_dump()))
+ except ValidationError as exc:
+ first_error = exc.errors()[0]
+ field = ".".join(str(part) for part in first_error["loc"])
+ raise InvalidCodeBlockStep(
+ block_label=block_yaml.label,
+ step_index=step_index,
+ detail=f"{field}: {first_error['msg']}",
+ ) from exc
return CodeBlock(
**base_kwargs,
code=block_yaml.code,
parameters=_resolve_block_parameters(block_yaml, parameters),
+ prompt=block_yaml.prompt,
+ steps=code_block_steps,
)
elif block_yaml.block_type == BlockType.TEXT_PROMPT:
return TextPromptBlock(
diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.py
index 2f5e4ff16..7288a43e7 100644
--- a/skyvern/schemas/workflows.py
+++ b/skyvern/schemas/workflows.py
@@ -795,6 +795,15 @@ class ConditionalBlockYAML(BlockYAML):
return self
+class CodeBlockStepYAML(BaseModel):
+ title: str | None = None
+ description: str | None = None
+ # str (not ActionType) so this module does not import skyvern.webeye; the converter coerces to the enum.
+ action_type: str = "null_action"
+ line_start: int | None = None
+ line_end: int | None = None
+
+
class CodeBlockYAML(BlockYAML):
# There is a mypy bug with Literal. Without the type: ignore, mypy will raise an error:
# Parameter 1 of Literal[...] cannot be of type "Any"
@@ -804,6 +813,8 @@ class CodeBlockYAML(BlockYAML):
code: str
parameter_keys: list[str] | None = None
+ prompt: str | None = None
+ steps: list[CodeBlockStepYAML] | None = None
@model_validator(mode="before")
@classmethod
diff --git a/tests/unit/test_code_block_steps.py b/tests/unit/test_code_block_steps.py
new file mode 100644
index 000000000..4e63cd302
--- /dev/null
+++ b/tests/unit/test_code_block_steps.py
@@ -0,0 +1,287 @@
+import pytest
+import yaml
+
+from skyvern.forge.sdk.copilot.code_block_steps import (
+ analyze_code_actions,
+ apply_derived_code_block_steps,
+ derive_code_block_steps,
+ derive_code_block_steps_in_yaml,
+ fill_code_block_prompts_in_yaml,
+ refine_step_descriptions,
+)
+from skyvern.webeye.actions.action_types import ActionType
+
+
+def test_analyze_maps_playwright_calls_to_action_types_with_line_ranges():
+ code = (
+ "async def run(page):\n"
+ " await page.goto('https://example.com/')\n"
+ " await page.wait_for_load_state('load')\n"
+ " await page.get_by_role('link', name='Login').click()\n"
+ " await page.get_by_label('Username').fill(str(username))\n"
+ " await page.get_by_label('Country').select_option('US')\n"
+ " await page.keyboard.press('Enter')\n"
+ )
+ spans = analyze_code_actions(code)
+ assert [(s.action_type, s.line_start) for s in spans] == [
+ ("goto_url", 2),
+ ("click", 4),
+ ("input_text", 5),
+ ("select_option", 6),
+ ("keypress", 7),
+ ]
+
+
+def test_analyze_skips_noise_and_returns_empty_on_syntax_error():
+ # wait_for_load_state is paired sync noise, never its own step.
+ assert analyze_code_actions("async def run(page):\n await page.wait_for_load_state('load')\n") == []
+ assert analyze_code_actions("def broken(:\n") == []
+
+
+def test_derive_steps_returns_dicts_with_templated_descriptions():
+ code = (
+ "async def run(page):\n"
+ " await page.goto('https://example.com/')\n"
+ " await page.get_by_role('button', name='Submit').click()\n"
+ " await page.get_by_label('Email').fill(str(email))\n"
+ )
+ steps = derive_code_block_steps(code)
+ assert steps == [
+ {"description": "Open https://example.com/", "action_type": "goto_url", "line_start": 2, "line_end": 2},
+ {"description": 'Click "Submit"', "action_type": "click", "line_start": 3, "line_end": 3},
+ {"description": 'Type into "Email"', "action_type": "input_text", "line_start": 4, "line_end": 4},
+ ]
+
+
+def test_derive_steps_empty_code_is_empty():
+ assert derive_code_block_steps("") == []
+ assert derive_code_block_steps("x = 1\n") == []
+
+
+async def test_apply_sets_steps_on_code_blocks_and_leaves_others_untouched():
+ src = {
+ "workflow_definition": {
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "block_1",
+ "code": "async def run(page):\n await page.goto('https://x.com/')\n",
+ },
+ {"block_type": "task", "label": "t1", "url": "https://x.com"},
+ {
+ "block_type": "for_loop",
+ "label": "loop",
+ "loop_blocks": [
+ {
+ "block_type": "code",
+ "label": "inner",
+ "code": "async def run(page):\n await page.get_by_role('button', name='Go').click()\n",
+ },
+ ],
+ },
+ ]
+ }
+ }
+ out = yaml.safe_load(await apply_derived_code_block_steps(yaml.safe_dump(src)))
+ blocks = out["workflow_definition"]["blocks"]
+ assert blocks[0]["steps"] == [
+ {"description": "Open https://x.com/", "action_type": "goto_url", "line_start": 2, "line_end": 2}
+ ]
+ assert "steps" not in blocks[1] # non-code block untouched
+ assert blocks[2]["loop_blocks"][0]["steps"][0]["action_type"] == "click" # nested code block annotated
+
+
+async def test_apply_is_noop_on_unparseable_yaml():
+ assert await apply_derived_code_block_steps("::not yaml::") == "::not yaml::"
+
+
+def test_derive_in_yaml_fills_steps_when_absent():
+ src = (
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: block_2\n"
+ " code: |\n"
+ " await page.goto('https://x.com/')\n"
+ " await page.get_by_role('link', name='login').click()\n"
+ )
+ out = yaml.safe_load(derive_code_block_steps_in_yaml(src))
+ steps = out["workflow_definition"]["blocks"][0]["steps"]
+ assert [s["action_type"] for s in steps] == ["goto_url", "click"]
+
+
+def test_derive_in_yaml_preserves_existing_steps():
+ # An LLM-refined steps list must survive untouched; deterministic derivation
+ # is a fallback only when steps are absent.
+ refined = [{"description": "Open the homepage", "action_type": "goto_url", "line_start": 1, "line_end": 1}]
+ src = {
+ "workflow_definition": {
+ "blocks": [
+ {
+ "block_type": "code",
+ "label": "block_1",
+ "code": "await page.goto('https://x.com/')\n",
+ "steps": refined,
+ }
+ ]
+ }
+ }
+ out = yaml.safe_load(derive_code_block_steps_in_yaml(yaml.safe_dump(src)))
+ assert out["workflow_definition"]["blocks"][0]["steps"] == refined
+
+
+def test_derive_in_yaml_noop_on_unparseable():
+ assert derive_code_block_steps_in_yaml("::not yaml::") == "::not yaml::"
+
+
+def test_fill_prompts_preserves_prior_block_prompt_across_regen():
+ # Regenerating a code block replaces the whole block YAML, dropping the goal.
+ # Without the prompt the editor renders the legacy code-only layout, so the
+ # block's prior prompt must be carried forward (exact user text).
+ prior = (
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: block_1\n"
+ " prompt: Build an agent to find the top post on the site\n"
+ " code: 'x = 1'\n"
+ )
+ regenerated = (
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: block_1\n"
+ " code: |\n"
+ " await page.goto('https://example.com/')\n"
+ )
+ out = yaml.safe_load(fill_code_block_prompts_in_yaml(regenerated, prior_yaml=prior))
+ assert out["workflow_definition"]["blocks"][0]["prompt"] == "Build an agent to find the top post on the site"
+
+
+def test_fill_prompts_falls_back_to_declared_goal_for_new_block():
+ new = "workflow_definition:\n blocks:\n - block_type: code\n label: block_2\n code: 'x = 1'\n"
+ out = yaml.safe_load(
+ fill_code_block_prompts_in_yaml(new, prior_yaml=None, fallback_goals={"block_2": "Search the catalog"})
+ )
+ assert out["workflow_definition"]["blocks"][0]["prompt"] == "Search the catalog"
+
+
+def test_fill_prompts_prefers_prior_over_fallback_and_preserves_existing():
+ prior = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n prompt: Exact user text\n code: 'x=1'\n"
+ new = (
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: b\n"
+ " code: 'y=2'\n"
+ " - block_type: code\n"
+ " label: c\n"
+ " prompt: Already set\n"
+ " code: 'z=3'\n"
+ )
+ out = yaml.safe_load(
+ fill_code_block_prompts_in_yaml(new, prior_yaml=prior, fallback_goals={"b": "model goal", "c": "ignored"})
+ )
+ blocks = out["workflow_definition"]["blocks"]
+ assert blocks[0]["prompt"] == "Exact user text" # prior beats fallback
+ assert blocks[1]["prompt"] == "Already set" # existing prompt untouched
+
+
+def test_fill_prompts_noop_without_sources():
+ new = "workflow_definition:\n blocks:\n - block_type: code\n label: b\n code: 'x=1'\n"
+ out = yaml.safe_load(fill_code_block_prompts_in_yaml(new))
+ assert "prompt" not in out["workflow_definition"]["blocks"][0]
+
+
+def test_process_workflow_yaml_derives_code_block_steps_for_replace_path():
+ # Regression: the inline REPLACE_WORKFLOW path (v1 and v2) builds the
+ # frontend-facing workflow via _process_workflow_yaml without first deriving
+ # steps, so a generated code block surfaced as "No steps yet" in the plain
+ # editor view while the update_workflow tool path showed them.
+ from skyvern.forge.sdk.routes.workflow_copilot import _process_workflow_yaml
+
+ yaml_str = (
+ "title: HN Login\n"
+ "workflow_definition:\n"
+ " parameters: []\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: block_2\n"
+ " prompt: Go to the site and log in\n"
+ " code: |\n"
+ " await page.goto('https://example.com/')\n"
+ " await page.get_by_role('link', name='login').click()\n"
+ )
+ wf = _process_workflow_yaml(
+ workflow_id="w_1",
+ workflow_permanent_id="wpid_1",
+ organization_id="o_1",
+ workflow_yaml=yaml_str,
+ )
+ block = wf.workflow_definition.blocks[0]
+ assert block.steps is not None
+ assert [s.action_type for s in block.steps] == [ActionType.GOTO_URL, ActionType.CLICK]
+
+
+async def test_apply_derived_steps_on_copilot_yaml_shape():
+ # Mirrors the _copilot_yaml payload that apply-proposed-workflow reads from
+ # the stashed proposal. Steps must be populated so manual-accept persists them.
+ copilot_yaml = (
+ "title: Search\n"
+ "workflow_definition:\n"
+ " blocks:\n"
+ " - block_type: code\n"
+ " label: do_search\n"
+ " code: |\n"
+ " await page.goto('https://example.com/')\n"
+ " await page.get_by_label('Query').fill(str(query))\n"
+ )
+ enriched = yaml.safe_load(await apply_derived_code_block_steps(copilot_yaml))
+ steps = enriched["workflow_definition"]["blocks"][0]["steps"]
+ assert len(steps) == 2
+ assert steps[0]["action_type"] == "goto_url"
+ assert steps[1]["action_type"] == "input_text"
+
+
+async def test_refine_overrides_descriptions_by_line_start():
+ steps = [
+ {"description": "Open https://x.com/", "action_type": "goto_url", "line_start": 2, "line_end": 2},
+ {"description": 'Click "Go"', "action_type": "click", "line_start": 3, "line_end": 3},
+ ]
+
+ async def handler(prompt: str, prompt_name: str):
+ return '[{"line_start": 2, "description": "Open the homepage"}, {"line_start": 3, "description": "Start the search"}]'
+
+ out = await refine_step_descriptions("code", None, steps, handler=handler)
+ assert [s["description"] for s in out] == ["Open the homepage", "Start the search"]
+ assert [s["action_type"] for s in out] == ["goto_url", "click"] # labels & lines unchanged
+
+
+def test_multiline_call_span_covers_all_lines():
+ code = "async def run(page):\n await page.get_by_label('Email').fill(\n str(email)\n )\n"
+ spans = analyze_code_actions(code)
+ assert spans[0].action_type == "input_text"
+ assert spans[0].line_start == 2 and spans[0].line_end == 4
+
+
+def test_get_by_role_without_name_falls_back_to_the_element():
+ code = "async def run(page):\n await page.get_by_role('button').click()\n"
+ steps = derive_code_block_steps(code)
+ assert steps[0]["description"] == "Click the element"
+
+
+async def test_refine_falls_back_to_input_on_handler_failure():
+ steps = [{"description": "Open https://x.com/", "action_type": "goto_url", "line_start": 2, "line_end": 2}]
+
+ async def handler(prompt: str, prompt_name: str):
+ raise RuntimeError("llm down")
+
+ out = await refine_step_descriptions("code", None, steps, handler=handler)
+ assert out == steps # unchanged on failure
+
+ assert await refine_step_descriptions("code", None, steps, handler=None) == steps # no handler -> unchanged
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index c6b8533d7..051b259f9 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -6,6 +6,7 @@ OSS-synced: only example.* / RFC-2606 placeholder targets.
from __future__ import annotations
import ast
+import json
import keyword
import sys
from typing import Any
@@ -19,6 +20,7 @@ from skyvern.forge.sdk.copilot.code_block_synthesis import (
synthesize_code_block,
)
from skyvern.forge.sdk.copilot.tools import _normalize_code_artifact_metadata
+from skyvern.forge.sdk.workflow.models.block import CodeBlockStep
def _interaction(tool_name: str, **fields: Any) -> dict[str, Any]:
@@ -545,6 +547,140 @@ class TestDeterminismAndEmpty:
assert first.notes == second.notes
+class TestStepEmission:
+ def test_synthesize_emits_goal_ready_steps(self) -> None:
+ trajectory = [
+ _interaction("click", selector="#go", source_url="https://example.com"),
+ _interaction("type_text", selector="#q", typed_length=5),
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["goto_url", "click", "input_text"]
+ assert all(s["description"] for s in block.steps)
+
+ def test_step_line_spans_cover_every_emitted_line(self) -> None:
+ trajectory = [
+ _interaction("click", selector="#go", source_url="https://example.com/start"),
+ _interaction(
+ "type_text",
+ selector="#q",
+ typed_length=5,
+ role="textbox",
+ accessible_name="Search",
+ ),
+ _interaction("press_key", key="Enter"),
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ code_lines = block.code.splitlines()
+ goto_step, click_step, fill_step, key_step = block.steps
+ assert (goto_step["line_start"], goto_step["line_end"]) == (1, 2)
+ assert code_lines[goto_step["line_start"] - 1].lstrip().startswith("await page.goto(")
+ assert (click_step["line_start"], click_step["line_end"]) == (3, 4)
+ assert ".click()" in code_lines[click_step["line_start"] - 1]
+ assert (fill_step["line_start"], fill_step["line_end"]) == (5, 5)
+ assert ".fill(" in code_lines[fill_step["line_start"] - 1]
+ assert key_step["action_type"] == "keypress"
+ assert (key_step["line_start"], key_step["line_end"]) == (6, 7)
+ assert "press" in code_lines[key_step["line_start"] - 1]
+ # Spans are contiguous and cover the whole block.
+ assert block.steps[0]["line_start"] == 1
+ assert block.steps[-1]["line_end"] == len(code_lines)
+ for previous, current in zip(block.steps, block.steps[1:]):
+ assert current["line_start"] == previous["line_end"] + 1
+
+ def test_select_option_and_press_key_action_types(self) -> None:
+ trajectory = [
+ _interaction(
+ "select_option",
+ selector='role=combobox[name="Size"]',
+ source_url="https://example.com/",
+ value="large",
+ role="combobox",
+ accessible_name="Size",
+ ),
+ _interaction("press_key", key="Enter"),
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["goto_url", "select_option", "keypress"]
+
+ def test_skipped_interactions_emit_no_step(self) -> None:
+ trajectory = [
+ _interaction("click", selector="#go", source_url="https://example.com/"),
+ _interaction("hover", selector="#menu"),
+ _interaction("select_option", selector="#size"),
+ _interaction("click"),
+ _interaction("press_key", key=""),
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["goto_url", "click"]
+
+ def test_no_entry_url_means_no_goto_step(self) -> None:
+ block = synthesize_code_block([_interaction("press_key", key="Enter")])
+ assert block is not None
+ assert [s["action_type"] for s in block.steps] == ["keypress"]
+
+ def test_step_descriptions_prefer_accessible_name_over_selector(self) -> None:
+ block = synthesize_code_block(
+ [
+ _interaction(
+ "click",
+ selector="div.list > button:nth-of-type(3)",
+ source_url="https://example.com/",
+ role="button",
+ accessible_name="Add to cart",
+ )
+ ]
+ )
+ assert block is not None
+ click_step = next(s for s in block.steps if s["action_type"] == "click")
+ assert "Add to cart" in click_step["description"]
+ assert "nth-of-type" not in click_step["description"]
+
+ def test_entry_url_step_description_carries_url(self) -> None:
+ block = synthesize_code_block([_interaction("click", selector="#go", source_url="https://example.com/start")])
+ assert block is not None
+ goto_step = block.steps[0]
+ assert goto_step["action_type"] == "goto_url"
+ assert "https://example.com/start" in goto_step["description"]
+
+ def test_steps_are_byte_identical_per_trajectory(self) -> None:
+ trajectory = [
+ _interaction(
+ "type_text",
+ selector='role=textbox[name="Search"]',
+ source_url="https://example.com/",
+ typed_length=4,
+ role="textbox",
+ accessible_name="Search",
+ ),
+ _interaction("press_key", key="Enter"),
+ ]
+ first = synthesize_code_block(trajectory)
+ second = synthesize_code_block(trajectory)
+ assert first is not None and second is not None
+ assert first.steps == second.steps
+
+ def test_truncated_trajectory_caps_steps_with_code(self) -> None:
+ trajectory = [
+ _interaction("click", selector=f'role=button[name="b{i}"]', source_url="https://example.com/")
+ for i in range(_MAX_STEPS + 5)
+ ]
+ block = synthesize_code_block(trajectory)
+ assert block is not None
+ click_steps = [s for s in block.steps if s["action_type"] == "click"]
+ assert len(click_steps) == _MAX_STEPS
+ assert block.steps[-1]["line_end"] == len(block.code.splitlines())
+
+ def test_steps_validate_against_code_block_step_schema(self) -> None:
+ block = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert block is not None
+ validated = [CodeBlockStep(**step) for step in block.steps]
+ assert all(step.line_start is not None and step.line_end is not None for step in validated)
+
+
class TestLineBoundaryEscaping:
# str.splitlines() and several parsers treat each of these as a line boundary. An attacker-controlled
# page can plant one in an accessible name or option value; left unescaped it splits the emitted
@@ -697,6 +833,46 @@ class TestRenderSynthesizedOfferText:
assert "Synthesis notes: " in text
+class TestOfferTextGoalAndSteps:
+ def test_offer_text_carries_steps_json_and_goal(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(
+ synthesized, _SCOUT_TRAJECTORY, goal="Search the catalog and add the item to the cart"
+ )
+ assert "`steps`" in text
+ assert "`prompt`" in text
+ assert "Search the catalog and add the item to the cart" in text
+ assert '"action_type": "goto_url"' in text
+ assert '"action_type": "input_text"' in text
+
+ def test_offer_text_omits_goal_mention_without_goal(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY)
+ assert "`steps`" in text
+ assert "`prompt`" not in text
+
+ def test_offer_text_goal_quotes_and_newlines_stay_in_quoted_span(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY, goal='find the "best" deal\nand report it')
+ assert "`prompt` field to \"find the 'best' deal and report it\"" in text
+
+ def test_offer_text_goal_code_fences_are_neutralized(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY, goal="do this\n```python\nx\n```")
+ assert "\n```python\nx\n```" not in text
+
+ def test_offer_text_steps_json_matches_synthesized_steps(self) -> None:
+ synthesized = synthesize_code_block(_SCOUT_TRAJECTORY)
+ assert synthesized is not None
+ text = render_synthesized_offer_text(synthesized, _SCOUT_TRAJECTORY)
+ rendered = json.dumps(synthesized.steps, indent=2, sort_keys=True)
+ assert rendered in text
+
+
def _code_block_yaml(label: str) -> str:
return (
"workflow_definition:\n"
@@ -756,8 +932,6 @@ class TestSynthesizedArtifactMetadata:
assert metadata["claimed_outcomes"][0]["text"].startswith("<fill:")
def test_skeleton_is_byte_identical_per_trajectory(self) -> None:
- import json
-
first = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
second = build_synthesized_artifact_metadata(_SCOUT_TRAJECTORY)
assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)
@@ -781,7 +955,7 @@ class TestSynthesizedArtifactMetadata:
assert synthesized is not None
text = render_synthesized_offer_text(synthesized)
assert "code_artifact_metadata" not in text
- assert "```json" not in text
+ assert "observed_not_verified" not in text
class TestCredentialFillSynthesis:
diff --git a/tests/unit/test_copilot_model_resolver.py b/tests/unit/test_copilot_model_resolver.py
index 17731ed01..699216a8b 100644
--- a/tests/unit/test_copilot_model_resolver.py
+++ b/tests/unit/test_copilot_model_resolver.py
@@ -440,3 +440,60 @@ class TestModelResolver:
joined = " ".join(str(record.get("event", "")) for record in logs)
assert "unrouted" not in joined
+
+ def test_github_copilot_endpoint_restores_openai_compatible_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ """GitHub Copilot via OPENAI_COMPATIBLE exposes the handler's .llm_key as the bare
+ model name ("gpt-4o"), which is not a registry key. The resolver must restore the
+ OPENAI_COMPATIBLE key so the githubcopilot api_base/api_key thread through instead of
+ resolving to a credential-less OpenAI model that 401s."""
+ from skyvern.config import settings
+ from skyvern.schemas.llm import LiteLLMParams, LLMConfig
+
+ openai_compatible_config = LLMConfig(
+ model_name="openai/gpt-4o",
+ required_env_vars=[],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ litellm_params=LiteLLMParams(
+ api_key="gho_test",
+ api_base="https://api.githubcopilot.com",
+ api_version=None,
+ model_info={"model_name": "openai/gpt-4o"},
+ ),
+ )
+
+ seen_keys: list[str] = []
+
+ def fake_get_config(key: str) -> LLMConfig:
+ seen_keys.append(key)
+ return openai_compatible_config
+
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.model_resolver.LLMConfigRegistry.get_config",
+ fake_get_config,
+ )
+ # The rewritten label "gpt-4o" is not a registered registry key.
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.model_resolver.LLMConfigRegistry.is_registered",
+ lambda key: False,
+ )
+ # Force the GitHub Copilot endpoint branch on.
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.copilot.model_resolver.LLMAPIHandlerFactory.is_github_copilot_endpoint",
+ lambda: True,
+ )
+ monkeypatch.setattr(settings, "OPENAI_COMPATIBLE_MODEL_KEY", "OPENAI_COMPATIBLE")
+ monkeypatch.setattr(settings, "OPENAI_COMPATIBLE_MODEL_NAME", "gpt-4o")
+
+ from skyvern.forge.sdk.copilot.model_resolver import resolve_model_config
+
+ handler = MagicMock()
+ handler.llm_key = "gpt-4o" # the rewritten observability label
+
+ model_name, run_config, llm_key, _ = resolve_model_config(handler)
+
+ assert llm_key == "OPENAI_COMPATIBLE"
+ assert seen_keys == ["OPENAI_COMPATIBLE"]
+ assert model_name == "openai/gpt-4o"
+ assert run_config.model_provider._base_url == "https://api.githubcopilot.com"
+ assert run_config.model_provider._api_key == "gho_test"
diff --git a/tests/unit/workflow/test_cache_invalidation.py b/tests/unit/workflow/test_cache_invalidation.py
index 41ee25ef3..d6711b85b 100644
--- a/tests/unit/workflow/test_cache_invalidation.py
+++ b/tests/unit/workflow/test_cache_invalidation.py
@@ -7,10 +7,11 @@ do not trigger cache invalidation.
from datetime import datetime, timezone
-from skyvern.forge.sdk.workflow.models.block import BlockType, TaskBlock
+from skyvern.forge.sdk.workflow.models.block import BlockType, CodeBlock, CodeBlockStep, TaskBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.forge.sdk.workflow.models.workflow import WorkflowDefinition
from skyvern.forge.sdk.workflow.service import _get_workflow_definition_core_data
+from skyvern.webeye.actions.action_types import ActionType
def make_output_parameter(key: str) -> OutputParameter:
@@ -39,6 +40,22 @@ def make_task_block(label: str, model: dict | None = None) -> TaskBlock:
)
+def make_code_block(
+ label: str,
+ goal: str | None = None,
+ steps: list[CodeBlockStep] | None = None,
+) -> CodeBlock:
+ """Create a test code block with optional code-first annotation fields."""
+ return CodeBlock(
+ label=label,
+ block_type=BlockType.CODE,
+ output_parameter=make_output_parameter(f"{label}_output"),
+ code="x = 1",
+ prompt=goal,
+ steps=steps,
+ )
+
+
class TestCacheInvalidation:
"""Tests for the _get_workflow_definition_core_data function."""
@@ -125,6 +142,54 @@ class TestCacheInvalidation:
f"Core data should be identical regardless of model. Definition 0 vs {i} differ."
)
+ def test_code_block_annotation_edits_excluded_from_comparison(self) -> None:
+ """Editing display/derived code-block annotations must not invalidate the cached script."""
+ plain = make_code_block("code1")
+ annotated = make_code_block(
+ "code1",
+ steps=[CodeBlockStep(description="Open the page", action_type=ActionType.GOTO_URL)],
+ )
+
+ core_data_plain = _get_workflow_definition_core_data(WorkflowDefinition(parameters=[], blocks=[plain]))
+ core_data_annotated = _get_workflow_definition_core_data(WorkflowDefinition(parameters=[], blocks=[annotated]))
+
+ assert core_data_plain == core_data_annotated, (
+ "Code block steps are a display/derived annotation and should be excluded from comparison."
+ )
+
+ def test_code_block_annotation_fields_not_in_core_data(self) -> None:
+ block = make_code_block(
+ "code1",
+ steps=[CodeBlockStep(description="Click go", action_type=ActionType.CLICK)],
+ )
+ core_data = _get_workflow_definition_core_data(WorkflowDefinition(parameters=[], blocks=[block]))
+
+ for block_data in core_data.get("blocks", []):
+ assert "steps" not in block_data
+
+ def test_code_block_goal_change_still_detected(self) -> None:
+ """A goal reprompt regenerates code and steps, so a goal edit must keep invalidating."""
+ definition1 = WorkflowDefinition(parameters=[], blocks=[make_code_block("code1", goal="Goal A")])
+ definition2 = WorkflowDefinition(parameters=[], blocks=[make_code_block("code1", goal="Goal B")])
+
+ core_data1 = _get_workflow_definition_core_data(definition1)
+ core_data2 = _get_workflow_definition_core_data(definition2)
+
+ assert core_data1 != core_data2, "Code block goal changes must still trigger cache invalidation"
+
+ def test_task_block_criteria_changes_still_detected(self) -> None:
+ """Criteria on task-family blocks are runtime-consumed; edits there must keep invalidating."""
+ block1 = make_task_block("task1")
+ block1.complete_criterion = "Criterion A"
+
+ block2 = make_task_block("task1")
+ block2.complete_criterion = "Criterion B"
+
+ core_data1 = _get_workflow_definition_core_data(WorkflowDefinition(parameters=[], blocks=[block1]))
+ core_data2 = _get_workflow_definition_core_data(WorkflowDefinition(parameters=[], blocks=[block2]))
+
+ assert core_data1 != core_data2, "Task block criteria changes must still trigger cache invalidation"
+
def test_timestamps_excluded_from_comparison(self) -> None:
"""Verify that timestamps are properly excluded from comparison."""
# Create two blocks with different timestamps
diff --git a/tests/unit/workflow/test_code_block_recorder.py b/tests/unit/workflow/test_code_block_recorder.py
new file mode 100644
index 000000000..c4e26b372
--- /dev/null
+++ b/tests/unit/workflow/test_code_block_recorder.py
@@ -0,0 +1,593 @@
+"""Tests for the RecordingPage proxy that records code block playwright calls as actions."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from datetime import datetime, timezone
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.sdk.models import StepStatus
+from skyvern.forge.sdk.schemas.tasks import TaskStatus
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.forge.sdk.workflow.models.code_block_recorder import (
+ CODE_BLOCK_FILENAME,
+ CODE_LINE_OFFSET,
+ RecordingPage,
+ user_code_line_from_exception,
+)
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
+from skyvern.schemas.workflows import BlockStatus
+from skyvern.webeye.actions.action_types import ActionType
+from skyvern.webeye.actions.actions import Action, ActionStatus
+
+
+class FakeLocator:
+ def __init__(self) -> None:
+ self.calls: list[str] = []
+
+ def locator(self, selector): # noqa: ANN001, ANN201
+ return self
+
+ def get_by_text(self, text, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return self
+
+ @property
+ def first(self): # noqa: ANN201
+ return self
+
+ async def click(self, **kwargs): # noqa: ANN003, ANN201
+ self.calls.append("click")
+
+ async def fill(self, value, **kwargs): # noqa: ANN001, ANN003, ANN201
+ self.calls.append(f"fill:{value}")
+
+ async def type(self, value, **kwargs): # noqa: ANN001, ANN003, ANN201
+ self.calls.append(f"type:{value}")
+
+ async def select_option(self, value, **kwargs): # noqa: ANN001, ANN003, ANN201
+ self.calls.append(f"select:{value}")
+
+ async def press(self, key, **kwargs): # noqa: ANN001, ANN003, ANN201
+ self.calls.append(f"press:{key}")
+
+ def filter(self, **kwargs): # noqa: ANN003, ANN201
+ return self
+
+
+class FakeKeyboard:
+ async def press(self, key, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+
+class FakePage:
+ def __init__(self) -> None:
+ self.inner = FakeLocator()
+ self.keyboard = FakeKeyboard()
+ self.url = "about:blank"
+
+ async def goto(self, url, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ async def wait_for_load_state(self, state="load", **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ def locator(self, selector): # noqa: ANN001, ANN201
+ return self.inner
+
+ async def click(self, selector, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ async def fill(self, selector, value, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return None
+
+ def get_by_role(self, role, **kwargs): # noqa: ANN001, ANN003, ANN201
+ return self.inner
+
+ async def screenshot(self, **kwargs): # noqa: ANN003, ANN201
+ return b"img"
+
+
+async def test_records_goto_click_fill_with_types_and_order() -> None:
+ page = RecordingPage(FakePage())
+ await page.goto("https://example.com")
+ await page.locator("#q").fill("hello")
+ await page.locator("#go").click()
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [
+ ActionType.GOTO_URL,
+ ActionType.INPUT_TEXT,
+ ActionType.CLICK,
+ ]
+ assert [a.action_order for a in recorded] == [0, 1, 2]
+ assert all(a.status == ActionStatus.completed for a in recorded)
+ assert recorded[0].description == "page.goto https://example.com"
+
+
+async def test_unmapped_calls_and_attributes_pass_through_unrecorded() -> None:
+ fake = FakePage()
+ page = RecordingPage(fake)
+ await page.wait_for_load_state("networkidle")
+ assert page.url == "about:blank"
+ assert page.recorded_actions() == []
+
+
+async def test_keyboard_press_records_keypress_action() -> None:
+ page = RecordingPage(FakePage())
+ await page.keyboard.press("Enter")
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.KEYPRESS]
+ assert recorded[0].description and "Enter" in recorded[0].description
+
+
+async def test_get_by_role_click_is_recorded() -> None:
+ page = RecordingPage(FakePage())
+ await page.get_by_role("button", name="Go").click()
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.CLICK]
+ assert recorded[0].description == "locator.click get_by_role(button)"
+
+
+async def test_locator_get_by_chain_is_recorded() -> None:
+ page = RecordingPage(FakePage())
+ await page.locator("#form").get_by_text("Submit").click()
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.CLICK]
+ assert recorded[0].description == "locator.click get_by_text(Submit)"
+
+
+async def test_direct_page_actions_are_recorded_with_redaction() -> None:
+ page = RecordingPage(FakePage())
+ await page.click("#submit")
+ await page.fill("#email", "[email protected]")
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.CLICK, ActionType.INPUT_TEXT]
+ # Input values may be credentials; the fill value must never reach the description.
+ assert all("[email protected]" not in (a.description or "") for a in recorded)
+
+
+async def test_filter_locator_chain_click_is_recorded() -> None:
+ page = RecordingPage(FakePage())
+ await page.get_by_role("button", name="Go").filter(has_text="Submit").click()
+ recorded = page.recorded_actions()
+ assert [a.action_type for a in recorded] == [ActionType.CLICK]
+
+
+async def test_failed_call_records_failed_action_and_reraises() -> None:
+ class ExplodingLocator(FakeLocator):
+ async def click(self, **kwargs): # noqa: ANN003, ANN201
+ raise RuntimeError("element detached")
+
+ fake = FakePage()
+ fake.inner = ExplodingLocator()
+ page = RecordingPage(fake)
+ with pytest.raises(RuntimeError):
+ await page.locator("#x").click()
+ recorded = page.recorded_actions()
+ assert recorded[-1].action_type == ActionType.CLICK
+ assert recorded[-1].status == ActionStatus.failed
+ assert "element detached" in (recorded[-1].response or "")
+
+
+async def test_on_action_sink_receives_each_action_and_errors_are_swallowed() -> None:
+ seen: list[ActionType] = []
+
+ async def sink(action) -> None: # noqa: ANN001
+ seen.append(action.action_type)
+ raise RuntimeError("sink failure must not break recording")
+
+ page = RecordingPage(FakePage(), on_action=sink)
+ await page.goto("https://example.com")
+ await page.locator("#go").click()
+ assert seen == [ActionType.GOTO_URL, ActionType.CLICK]
+ assert len(page.recorded_actions()) == 2
+
+
+def test_user_code_line_from_exception_unwraps_wrapper_offset() -> None:
+ code = "raise ValueError('boom')"
+ full_code = f"\nasync def wrapper():\n {code}\n return None\n"
+ namespace: dict = {}
+ exec(compile(full_code, CODE_BLOCK_FILENAME, "exec"), {}, namespace)
+ with pytest.raises(ValueError) as exc_info:
+ asyncio.run(namespace["wrapper"]())
+ line = user_code_line_from_exception(exc_info.value)
+ assert line == 3 - CODE_LINE_OFFSET # frame line 3 -> user line 1
+
+
+async def test_generated_user_function_exception_maps_to_user_code_line() -> None:
+ block = _make_code_block("x = 1")
+ user_function = block.generate_async_user_function("x = 1\nraise Exception('boom')", FakePage())
+ with pytest.raises(Exception, match="boom") as exc_info:
+ await user_function()
+ assert user_code_line_from_exception(exc_info.value) == 2
+
+
+async def test_input_values_are_elided_from_descriptions() -> None:
+ page = RecordingPage(FakePage())
+ await page.locator("#pw").fill("hunter2-credential")
+ await page.locator("#user").type("alice-credential")
+ recorded = page.recorded_actions()
+ dumped = json.dumps([a.model_dump(mode="json") for a in recorded])
+ assert "hunter2-credential" not in dumped
+ assert "alice-credential" not in dumped
+ assert recorded[0].description == "locator.fill #pw"
+ assert recorded[1].description == "locator.type #user"
+
+
+def _make_code_block(code: str, goal: str | None = None) -> CodeBlock:
+ now = datetime.now(timezone.utc)
+ output_parameter = OutputParameter(
+ parameter_type=ParameterType.OUTPUT,
+ key="code_output",
+ description="test output",
+ output_parameter_id="op_code",
+ workflow_id="w_test",
+ created_at=now,
+ modified_at=now,
+ )
+ return CodeBlock(label="code_1", code=code, prompt=goal, output_parameter=output_parameter)
+
+
+class _FakeTask:
+ def __init__(self) -> None:
+ self.task_id = "tsk_code"
+ self.organization_id = "o_test"
+
+
+class _FakeStep:
+ def __init__(self) -> None:
+ self.step_id = "stp_code"
+ self.order = 0
+
+
+class FakeWorkflowRunContext:
+ """Minimal context for CodeBlock.execute; masking delegates to the real implementation."""
+
+ values: dict = {}
+ workflow_run_outputs: list = []
+ include_secrets_in_templates = False
+ workflow_title = "Test Workflow"
+ workflow_id = "w_test"
+ workflow_permanent_id = "wpid_test"
+ workflow_run_id = "wr_test"
+ browser_session_id = None
+
+ def __init__(self, secrets: dict[str, str] | None = None) -> None:
+ self.secrets = secrets or {}
+
+ def get_block_metadata(self, label): # noqa: ANN001, ANN201
+ return {}
+
+ def build_workflow_run_summary(self) -> str:
+ return ""
+
+ def get_value(self, key): # noqa: ANN001, ANN201
+ return self.values.get(key)
+
+ def get_original_secret_value_or_none(self, value): # noqa: ANN001, ANN201
+ return None
+
+ def mask_secrets_in_data(self, data, mask="*****"): # noqa: ANN001, ANN201
+ return WorkflowRunContext.mask_secrets_in_data(self, data, mask) # type: ignore[arg-type]
+
+ async def register_output_parameter_value_post_execution(self, parameter, value): # noqa: ANN001, ANN201
+ return None
+
+
+def _patch_execute_environment(
+ monkeypatch: pytest.MonkeyPatch,
+ page: FakePage,
+ context: FakeWorkflowRunContext,
+) -> dict[str, AsyncMock]:
+ class FakeBrowserState:
+ async def get_working_page(self): # noqa: ANN201
+ return page
+
+ async def validate_code_block(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return None
+
+ async def get_browser_state(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return FakeBrowserState()
+
+ async def record_output(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
+ return None
+
+ mocks = {
+ "get_workflow_run_block": AsyncMock(return_value=object()),
+ "update_workflow_run_block": AsyncMock(return_value=None),
+ "create_artifact": AsyncMock(return_value="artifact_1"),
+ "create_task_and_step": AsyncMock(return_value=(_FakeTask(), _FakeStep())),
+ "create_action": AsyncMock(return_value=None),
+ "update_task": AsyncMock(return_value=None),
+ "update_step": AsyncMock(return_value=None),
+ }
+ monkeypatch.setattr(
+ "skyvern.forge.sdk.workflow.models.block.app.AGENT_FUNCTION.validate_code_block", validate_code_block
+ )
+ monkeypatch.setattr(CodeBlock, "get_or_create_browser_state", get_browser_state)
+ monkeypatch.setattr(CodeBlock, "get_workflow_run_context", lambda *args: context)
+ monkeypatch.setattr(CodeBlock, "record_output_parameter_value", record_output)
+ monkeypatch.setattr(app.DATABASE.observer, "get_workflow_run_block", mocks["get_workflow_run_block"])
+ monkeypatch.setattr(app.DATABASE.observer, "update_workflow_run_block", mocks["update_workflow_run_block"])
+ monkeypatch.setattr(app.ARTIFACT_MANAGER, "create_workflow_run_block_artifact", mocks["create_artifact"])
+ monkeypatch.setattr(app.agent, "create_task_and_step_from_code_block", mocks["create_task_and_step"], raising=False)
+ monkeypatch.setattr(app.DATABASE.workflow_params, "create_action", mocks["create_action"])
+ monkeypatch.setattr(app.DATABASE.tasks, "update_task", mocks["update_task"])
+ monkeypatch.setattr(app.DATABASE.tasks, "update_step", mocks["update_step"])
+ return mocks
+
+
+def _created_actions(mocks: dict[str, AsyncMock]) -> list[Action]:
+ return [call.args[0] for call in mocks["create_action"].await_args_list]
+
+
+async def test_goal_code_block_creates_task_and_links_block(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A code block with a goal spins up a task v1 + step and links it to the run block."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("value = 'ok'", goal="log into the portal")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ assert mocks["create_task_and_step"].await_count == 1
+ linked = [
+ call.kwargs.get("task_id")
+ for call in mocks["update_workflow_run_block"].await_args_list
+ if call.kwargs.get("task_id") is not None
+ ]
+ assert linked == ["tsk_code"]
+
+
+async def test_create_task_and_step_from_code_block_maps_goal_to_task(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The container task carries the code block goal as its navigation goal so the agent can resume it."""
+ create_task = AsyncMock(return_value=_FakeTask())
+ update_task = AsyncMock(return_value=_FakeTask())
+ create_step = AsyncMock(return_value=_FakeStep())
+ monkeypatch.setattr(app.DATABASE.tasks, "get_last_task_for_workflow_run", AsyncMock(return_value=None))
+ monkeypatch.setattr(app.DATABASE.tasks, "create_task", create_task)
+ monkeypatch.setattr(app.DATABASE.tasks, "update_task", update_task)
+ monkeypatch.setattr(app.DATABASE.tasks, "create_step", create_step)
+
+ block = _make_code_block("x = 1", goal="log into the portal")
+ task, step = await ForgeAgent().create_task_and_step_from_code_block(
+ code_block=block,
+ organization_id="o_test",
+ workflow_run_id="wr_test",
+ task_url="https://example.com/login",
+ )
+
+ assert task.task_id == "tsk_code"
+ assert step.step_id == "stp_code"
+ assert create_task.await_args.kwargs["navigation_goal"] == "log into the portal"
+ assert create_task.await_args.kwargs["url"] == "https://example.com/login"
+ assert update_task.await_args.kwargs["status"] == TaskStatus.running
+ assert create_step.await_args.kwargs["order"] == 0
+
+
+async def test_goal_code_block_marks_task_completed_on_success(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The container task must not dangle in 'running'; success drives it to completed."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("value = 'ok'", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ statuses = [call.kwargs.get("status") for call in mocks["update_task"].await_args_list]
+ assert TaskStatus.completed in statuses
+ step_statuses = [call.kwargs.get("status") for call in mocks["update_step"].await_args_list]
+ assert StepStatus.completed in step_statuses
+
+
+async def test_goal_code_block_marks_task_failed_on_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ """A failing code block drives its container task to failed, not a stuck 'running'."""
+
+ class ExplodingLocator(FakeLocator):
+ async def click(self, **kwargs): # noqa: ANN003, ANN201
+ raise RuntimeError("element detached")
+
+ page = FakePage()
+ page.inner = ExplodingLocator()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("await page.locator('#x').click()", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is False
+ statuses = [call.kwargs.get("status") for call in mocks["update_task"].await_args_list]
+ assert TaskStatus.failed in statuses
+ step_statuses = [call.kwargs.get("status") for call in mocks["update_step"].await_args_list]
+ assert StepStatus.failed in step_statuses
+
+
+async def test_goal_code_block_finalizes_step_on_cancellation(monkeypatch: pytest.MonkeyPatch) -> None:
+ """An asyncio.CancelledError (copilot orphan-cancel) must still finalize task + step, not dangle in 'running'/'created'."""
+
+ class CancellingLocator(FakeLocator):
+ async def click(self, **kwargs): # noqa: ANN003, ANN201
+ raise asyncio.CancelledError()
+
+ page = FakePage()
+ page.inner = CancellingLocator()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("await page.locator('#x').click()", goal="go")
+ with pytest.raises(asyncio.CancelledError):
+ await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ task_statuses = [call.kwargs.get("status") for call in mocks["update_task"].await_args_list]
+ step_statuses = [call.kwargs.get("status") for call in mocks["update_step"].await_args_list]
+ assert TaskStatus.failed in task_statuses
+ assert StepStatus.failed in step_statuses
+
+
+async def test_goalless_code_block_creates_no_task_or_actions(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Without a goal there is no task to hang actions on, so none are created or persisted."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("await page.locator('#go').click()\nvalue = 'ok'")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ assert mocks["create_task_and_step"].await_count == 0
+ assert mocks["create_action"].await_count == 0
+
+
+async def test_goalless_code_block_skips_screenshots(monkeypatch: pytest.MonkeyPatch) -> None:
+ """No task means screenshots would have no action row to anchor to, so don't take orphan ones."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("await page.locator('#go').click()\nvalue = 'ok'")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ assert mocks["create_artifact"].await_count == 0
+
+
+async def test_recorded_calls_persist_as_actions_on_the_step(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Each recorded playwright call becomes a real Action row tied to the task/step."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block("await page.goto('https://example.com')\nawait page.locator('#go').click()", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ actions = _created_actions(mocks)
+ assert [a.action_type for a in actions] == [ActionType.GOTO_URL, ActionType.CLICK]
+ assert all(a.task_id == "tsk_code" and a.step_id == "stp_code" and a.step_order == 0 for a in actions)
+ assert [a.action_order for a in actions] == [0, 1]
+
+
+async def test_backgrounded_screenshots_are_drained_and_linked_before_persist(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Screenshots upload off the user-await chain, but the drain must finish before
+ actions persist so each persisted action carries its screenshot artifact id.
+ The upload is made slow so this fails if the pre-persist drain is dropped: the
+ action would serialize before the still-running upload sets its id."""
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ async def slow_upload(**kwargs: object) -> str:
+ # Outlasts the microsecond gap between the user function ending and persist's
+ # serialization, so only an explicit drain can complete it in time.
+ await asyncio.sleep(0.05)
+ return "artifact_1"
+
+ mocks["create_artifact"].side_effect = slow_upload
+
+ block = _make_code_block("await page.goto('https://example.com')\nawait page.locator('#go').click()", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ # Both screenshot-eligible actions captured an artifact off the await chain...
+ assert mocks["create_artifact"].await_count == 2
+ # ...and the drain ran before persistence, so every persisted action links its screenshot.
+ actions = _created_actions(mocks)
+ assert [a.action_type for a in actions] == [ActionType.GOTO_URL, ActionType.CLICK]
+ assert all(a.screenshot_artifact_id == "artifact_1" for a in actions)
+
+
+async def test_persist_failure_does_not_fail_the_block(monkeypatch: pytest.MonkeyPatch) -> None:
+ page = FakePage()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+ mocks["create_action"].side_effect = RuntimeError("db unavailable")
+
+ block = _make_code_block("await page.locator('#go').click()\nvalue = 'ok'", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is True
+ assert result.status == BlockStatus.completed
+ assert result.output_parameter_value is not None
+ assert result.output_parameter_value["value"] == "ok"
+
+
+async def test_caught_page_failure_then_unrelated_raise_persists_synthetic_action(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A swallowed page failure must not steal attribution from a later unrelated raise."""
+
+ class ExplodingLocator(FakeLocator):
+ async def click(self, **kwargs): # noqa: ANN003, ANN201
+ raise RuntimeError("element detached")
+
+ page = FakePage()
+ page.inner = ExplodingLocator()
+ context = FakeWorkflowRunContext()
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ code = "try:\n await page.locator('#x').click()\nexcept Exception:\n pass\nraise Exception('later failure')"
+ block = _make_code_block(code, goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.success is False
+ actions = _created_actions(mocks)
+ assert actions[-2].action_type == ActionType.CLICK
+ assert actions[-2].status == ActionStatus.failed
+ assert actions[-1].action_type == ActionType.NULL_ACTION
+ assert actions[-1].status == ActionStatus.failed
+ assert isinstance(actions[-1].output, dict) and actions[-1].output["code_line"] == 5
+ assert "later failure" in (actions[-1].response or "")
+
+
+async def test_persisted_actions_never_contain_secret_values(monkeypatch: pytest.MonkeyPatch) -> None:
+ secret = "s3cr3t-credential-value"
+
+ class ExplodingLocator(FakeLocator):
+ async def fill(self, value, **kwargs): # noqa: ANN001, ANN003, ANN201
+ raise RuntimeError(f"cannot fill element with {value}")
+
+ page = FakePage()
+ page.inner = ExplodingLocator()
+ context = FakeWorkflowRunContext(secrets={"cred": secret})
+ mocks = _patch_execute_environment(monkeypatch, page, context)
+
+ block = _make_code_block(f"await page.locator('#pw').fill('{secret}')", goal="go")
+ result = await block.execute(workflow_run_id="wr_test", workflow_run_block_id="wrb_test", organization_id="o_test")
+
+ assert result.status == BlockStatus.failed
+ actions = _created_actions(mocks)
+ assert actions
+ dumped = json.dumps([a.model_dump(mode="json") for a in actions])
+ assert secret not in dumped
diff --git a/tests/unit/workflow/test_code_block_schema.py b/tests/unit/workflow/test_code_block_schema.py
new file mode 100644
index 000000000..e8dcd544a
--- /dev/null
+++ b/tests/unit/workflow/test_code_block_schema.py
@@ -0,0 +1,117 @@
+from unittest.mock import MagicMock
+
+import pytest
+
+from skyvern.forge.sdk.workflow.context_manager import WorkflowRunContext
+from skyvern.forge.sdk.workflow.exceptions import WorkflowDefinitionValidationException
+from skyvern.forge.sdk.workflow.models.block import CodeBlock, CodeBlockStep
+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter
+from skyvern.forge.sdk.workflow.workflow_definition_converter import block_yaml_to_block
+from skyvern.schemas.workflows import CodeBlockYAML
+from skyvern.webeye.actions.action_types import ActionType
+
+
+def _output_parameter() -> OutputParameter:
+ return OutputParameter(
+ output_parameter_id="op_1",
+ key="code_output",
+ workflow_id="w_1",
+ created_at="2026-01-01T00:00:00",
+ modified_at="2026-01-01T00:00:00",
+ )
+
+
+def test_code_block_yaml_accepts_code_first_fields() -> None:
+ block_yaml = CodeBlockYAML(
+ label="code_1",
+ code="x = 1",
+ prompt="Open {{ url }} and read the total",
+ steps=[
+ {"description": "Open the page", "action_type": "goto_url", "line_start": 1, "line_end": 2},
+ {"description": "Read the total", "action_type": "extract"},
+ ],
+ )
+ assert block_yaml.prompt == "Open {{ url }} and read the total"
+ assert block_yaml.steps is not None and block_yaml.steps[0].action_type == "goto_url"
+
+
+def test_code_block_yaml_defaults_keep_legacy_shape() -> None:
+ block_yaml = CodeBlockYAML(label="code_1", code="x = 1")
+ assert block_yaml.prompt is None
+ assert block_yaml.steps is None
+
+
+def test_converter_coerces_step_action_type_to_enum() -> None:
+ block_yaml = CodeBlockYAML(
+ label="code_1",
+ code="x = 1",
+ steps=[
+ {"description": "Open the page", "action_type": "goto_url"},
+ {"description": "No explicit action type"},
+ ],
+ )
+ block = block_yaml_to_block(block_yaml, {"code_1_output": _output_parameter()})
+ assert isinstance(block, CodeBlock)
+ assert block.steps is not None
+ assert block.steps[0].action_type is ActionType.GOTO_URL
+ assert block.steps[1].action_type is ActionType.NULL_ACTION
+
+
+def test_converter_rejects_invalid_step_action_type_with_block_context() -> None:
+ block_yaml = CodeBlockYAML(
+ label="code_1",
+ code="x = 1",
+ steps=[
+ {"description": "Open the page", "action_type": "goto_url"},
+ {"description": "Bad step", "action_type": "not_a_real_action"},
+ ],
+ )
+ with pytest.raises(WorkflowDefinitionValidationException) as exc_info:
+ block_yaml_to_block(block_yaml, {"code_1_output": _output_parameter()})
+ message = str(exc_info.value)
+ assert "code_1" in message
+ assert "index 1" in message
+ assert "action_type" in message
+
+
+def test_code_block_model_roundtrip() -> None:
+ block = CodeBlock(
+ label="code_1",
+ output_parameter=_output_parameter(),
+ code="x = 1",
+ prompt="g",
+ steps=[CodeBlockStep(description="d", action_type=ActionType.CLICK)],
+ )
+ dumped = block.model_dump()
+ assert dumped["prompt"] == "g"
+ assert dumped["steps"][0]["action_type"] == "click"
+
+
+def _make_workflow_run_context(values: dict | None = None) -> WorkflowRunContext:
+ ctx = WorkflowRunContext(
+ workflow_title="test",
+ workflow_id="w_1",
+ workflow_permanent_id="wpid_1",
+ workflow_run_id="wr_1",
+ aws_client=MagicMock(),
+ )
+ if values:
+ ctx.values.update(values)
+ return ctx
+
+
+def test_format_potential_template_parameters_renders_prompt() -> None:
+ # Regression: prompt must be jinja-rendered before it reaches the task v1
+ # (mirrors the task block), not passed through raw.
+ block = CodeBlock(
+ label="code_1",
+ output_parameter=_output_parameter(),
+ code="x = {{ count }}",
+ prompt="Open {{ url }}",
+ )
+ ctx = _make_workflow_run_context({"count": "1", "url": "https://example.com"})
+
+ block.format_potential_template_parameters(ctx)
+
+ assert block.code == "x = 1"
+ assert block.prompt == "Open https://example.com"
diff --git a/tests/unit/workflow/test_workflow_run_timeline_tree.py b/tests/unit/workflow/test_workflow_run_timeline_tree.py
index 9c8631aca..10f091072 100644
--- a/tests/unit/workflow/test_workflow_run_timeline_tree.py
+++ b/tests/unit/workflow/test_workflow_run_timeline_tree.py
@@ -128,6 +128,27 @@ async def test_timeline_attaches_actions_to_conditional_blocks(mock_db: AsyncMoc
assert timeline[0].block.actions == [action]
+async def test_timeline_attaches_actions_to_code_blocks(mock_db: AsyncMock) -> None:
+ """A code block's task actions render through the standard task_id path, like any task block."""
+ base = datetime(2026, 6, 10, 12, 0, tzinfo=timezone.utc)
+ code_block = _block(
+ "wrb_code",
+ created_at=base,
+ block_type=BlockType.CODE,
+ task_id="tsk_code",
+ )
+ action = ExtractAction(action_id="act_code", task_id="tsk_code")
+ mock_db.return_value = [code_block]
+ app.DATABASE.tasks.get_tasks_actions.return_value = [action]
+
+ service = WorkflowService()
+ timeline = await service.get_workflow_run_timeline(workflow_run_id="wr_test", organization_id="o_test")
+
+ assert timeline[0].block is not None
+ assert timeline[0].block.actions == [action]
+
+
@pytest.mark.asyncio
async def test_timeline_empty_run_returns_empty_list(mock_db: AsyncMock) -> None:
"""No blocks → empty timeline."""