Skip to content

Automate browser based workflows with AI

Render-validate copilot code blocks at the persist seam (SKY-12591) (#7512)

Andrew Neilson58b0397cc80d
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.py
index ffe49946c..de2852bd7 100644
--- a/skyvern/forge/sdk/copilot/build_test_outcome.py
+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py
@@ -45,6 +45,7 @@ BuildTestOutcomeReasonCode = Literal[
"sandbox_unresolved_name",
"synthesized_parameter_binding_ambiguous",
"code_safety_reject",
+ "code_block_unrenderable",
"credential_scout_reject",
"schema_incompatibility",
"verified_success",
diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.py
index 36310aeb7..3c386a8d5 100644
--- a/skyvern/forge/sdk/copilot/code_block_preflight.py
+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py
@@ -12,12 +12,20 @@ from dataclasses import dataclass
from functools import cache
from pathlib import Path
from types import SimpleNamespace
-from typing import Iterable, Mapping
+from typing import Iterable, Iterator, Mapping
+
+from jinja2 import StrictUndefined, TemplateSyntaxError, UndefinedError
+from jinja2.exceptions import SecurityError
+from jinja2.sandbox import SandboxedEnvironment
from skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError, author_time_code_security_errors
+from skyvern.forge.sdk.workflow.models._jinja import _json_finalize, _json_type_filter
from skyvern.forge.sdk.workflow.models.block import CodeBlock
+from skyvern.utils.templating import get_missing_variables
SANDBOX_UNRESOLVED_NAME_REASON_CODE = "SANDBOX_UNRESOLVED_NAME"
+RENDER_TEMPLATE_SYNTAX_REASON_CODE = "RENDER_TEMPLATE_SYNTAX"
+RENDER_UNDEFINED_NAME_REASON_CODE = "RENDER_UNDEFINED_NAME"
@dataclass(frozen=True)
@@ -26,6 +34,13 @@ class CodeBlockPreflightDiagnostic:
message: str
+@dataclass(frozen=True)
+class CodeBlockRenderDiagnostic:
+ code: str
+ message: str
+ failing_expression: str
+
+
@dataclass(frozen=True)
class CodeBlockSandboxNameDiagnostic:
code: str
@@ -37,6 +52,139 @@ class CodeBlockSandboxNameDiagnostic:
allowed_helper_surface: dict[str, tuple[str, ...]]
+# Mirrors the runtime's strict-mode template formatter (jinja_json_finalize_strict_env)
+# regardless of the WORKFLOW_TEMPLATING_STRICTNESS deployment setting.
+_render_check_env = SandboxedEnvironment(undefined=StrictUndefined, finalize=_json_finalize)
+_render_check_env.filters["json"] = _json_type_filter
+
+_RENDER_SYSTEM_BINDING_NAMES = (
+ "workflow_title",
+ "workflow_id",
+ "workflow_permanent_id",
+ "workflow_run_id",
+ "current_date",
+ "browser_session_id",
+ "workflow_run_outputs",
+ "workflow_run_summary",
+)
+
+# The runtime injects these only inside a for-loop iteration, so bind them only
+# when the source actually opens a loop. ponytail: loop-presence heuristic, not
+# true per-scope tracking — a reference after the loop closes still passes.
+_RENDER_LOOP_BINDING_NAMES = (
+ "current_index",
+ "current_item",
+ "current_value",
+)
+
+_JINJA_EXPRESSION_RE = re.compile(r"\{\{.*?\}\}", re.DOTALL)
+_JINJA_STATEMENT_RE = re.compile(r"\{%.*?%\}", re.DOTALL)
+_JINJA_FOR_STATEMENT_RE = re.compile(r"\{%-?\s*for\s", re.DOTALL)
+
+
+class _PermissiveRenderBinding:
+ def __getattr__(self, name: str) -> _PermissiveRenderBinding:
+ return self
+
+ def __getitem__(self, key: object) -> _PermissiveRenderBinding:
+ return self
+
+ # Without __iter__, __getitem__ triggers Python's legacy iteration protocol,
+ # which never raises IndexError here and spins {% for %} loops forever.
+ def __iter__(self) -> Iterator[_PermissiveRenderBinding]:
+ return iter((self,))
+
+ def __str__(self) -> str:
+ return "value"
+
+
+def _first_template_expression_for(code: str, root: str) -> str:
+ root_re = re.compile(rf"(?<![A-Za-z0-9_]){re.escape(root)}(?![A-Za-z0-9_])")
+ for pattern in (_JINJA_EXPRESSION_RE, _JINJA_STATEMENT_RE):
+ for match in pattern.finditer(code):
+ if root_re.search(match.group(0)):
+ return match.group(0)
+ return f"{{{{ {root} }}}}"
+
+
+def _top_level_form_suggestion(expression: str, root: str) -> str:
+ inner = expression.strip().strip("{}%").strip()
+ if not inner.startswith(f"{root}."):
+ return ""
+ remainder = inner[len(root) + 1 :]
+ match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", remainder)
+ if match is None:
+ return ""
+ return f"{{{{ {match.group(0)} }}}}"
+
+
+def code_block_render_diagnostic(code: str, bound_names: Iterable[str]) -> CodeBlockRenderDiagnostic | None:
+ """Dry-render the code block through the runtime's strict Jinja semantics with every
+ runtime-provided name bound to a permissive sentinel; only genuinely unrenderable
+ templates (undefined names, syntax errors, sandbox violations) produce a diagnostic."""
+ if "{{" not in code and "{%" not in code:
+ return None
+ try:
+ template = _render_check_env.from_string(code)
+ except TemplateSyntaxError as exc:
+ source_lines = code.splitlines()
+ line = source_lines[exc.lineno - 1].strip() if exc.lineno and exc.lineno <= len(source_lines) else ""
+ detail = f" Offending line: `{line}`." if line else ""
+ return CodeBlockRenderDiagnostic(
+ code=RENDER_TEMPLATE_SYNTAX_REASON_CODE,
+ message=f"Jinja template syntax error on line {exc.lineno}: {exc.message}.{detail}",
+ failing_expression=line,
+ )
+ bindings: dict[str, object] = {name: _PermissiveRenderBinding() for name in bound_names}
+ system_names: tuple[str, ...] = _RENDER_SYSTEM_BINDING_NAMES
+ if _JINJA_FOR_STATEMENT_RE.search(code):
+ system_names = system_names + _RENDER_LOOP_BINDING_NAMES
+ for name in system_names:
+ bindings.setdefault(name, _PermissiveRenderBinding())
+ missing: set[str] = set()
+ try:
+ missing = get_missing_variables(code, bindings)
+ except (UndefinedError, SecurityError) as exc:
+ return CodeBlockRenderDiagnostic(
+ code=RENDER_UNDEFINED_NAME_REASON_CODE,
+ message=f"A Jinja expression in this code block cannot render at runtime: {exc}.",
+ failing_expression="",
+ )
+ except Exception:
+ missing = set()
+ if missing:
+ root = sorted(missing)[0].split("[")[0].split(".")[0]
+ expression = _first_template_expression_for(code, root)
+ suggestion = _top_level_form_suggestion(expression, root)
+ guidance = (
+ f" Declared inputs are injected as top-level names; write `{suggestion}` instead."
+ if suggestion
+ else (
+ " Only declared parameter keys, block labels, `<label>_output` values, and workflow "
+ "system names (e.g. `current_date`) are available as top-level template names."
+ )
+ )
+ return CodeBlockRenderDiagnostic(
+ code=RENDER_UNDEFINED_NAME_REASON_CODE,
+ message=(
+ f"The expression `{expression}` cannot render at runtime: "
+ f"`{root}` is not a defined template name.{guidance}"
+ ),
+ failing_expression=expression,
+ )
+ try:
+ template.render(bindings)
+ except (UndefinedError, SecurityError) as exc:
+ return CodeBlockRenderDiagnostic(
+ code=RENDER_UNDEFINED_NAME_REASON_CODE,
+ message=f"A Jinja expression in this code block cannot render at runtime: {exc}.",
+ failing_expression="",
+ )
+ except Exception:
+ return None
+ return None
+
+
_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m|\x1b\([AB]")
_MYPY_ERROR_RE = re.compile(r"^(?P<path>.*?):(?P<line>\d+): (?P<severity>error): (?P<message>.*)$")
_LOCATOR_NOT_CALLABLE_RE = re.compile(r'"Locator" not callable\s+\[operator\]')
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index 056b66415..175887b59 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -50,7 +50,9 @@ from skyvern.forge.sdk.copilot.build_test_outcome import (
)
from skyvern.forge.sdk.copilot.code_block_preflight import (
SANDBOX_UNRESOLVED_NAME_REASON_CODE,
+ CodeBlockRenderDiagnostic,
author_time_code_block_diagnostics,
+ code_block_render_diagnostic,
sandbox_unresolved_name_diagnostics,
sandbox_unresolved_name_repair_diagnostic,
strip_redundant_sandbox_imports,
@@ -141,6 +143,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (
classify_output_contract_bail_family,
code_block_available_binding_keys_by_label,
declared_string_workflow_parameter_keys,
+ declared_workflow_parameter_keys,
resolve_output_contract_actuation,
)
from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate
@@ -200,6 +203,7 @@ from skyvern.forge.sdk.copilot.turn_ownership import (
)
from skyvern.forge.sdk.copilot.workflow_credential_utils import (
credential_param_ids,
+ credential_params,
parse_workflow_yaml,
workflow_blocks,
)
@@ -1065,6 +1069,42 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)
return errors
+def _code_block_render_bound_names(workflow_yaml: str) -> set[str]:
+ parsed = parse_workflow_yaml(workflow_yaml)
+ if not isinstance(parsed, dict):
+ return set()
+ names = set(declared_workflow_parameter_keys(parsed))
+ workflow_definition = parsed.get("workflow_definition")
+ if isinstance(workflow_definition, dict):
+ for key in credential_params(workflow_definition.get("parameters")):
+ names.add(f"{key}_real_username")
+ names.add(f"{key}_real_password")
+ for block in workflow_blocks(parsed):
+ label = str(block.get("label") or "").strip()
+ if label:
+ names.add(label)
+ names.add(f"{label}_output")
+ return names
+
+
+def _code_block_render_reject(
+ workflow_yaml: str, prior_yaml: str | None
+) -> tuple[str, CodeBlockRenderDiagnostic] | None:
+ prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)
+ bound_names = _code_block_render_bound_names(workflow_yaml)
+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():
+ code = str(block.get("code") or "")
+ if not code.strip():
+ continue
+ prior_block = prior_blocks.get(label)
+ if prior_block is not None and str(prior_block.get("code") or "") == code:
+ continue
+ diagnostic = code_block_render_diagnostic(code, bound_names)
+ if diagnostic is not None:
+ return label, diagnostic
+ return None
+
+
def _human_facing_code_safety_errors(errors: list[str | CodeBlockSecurityError]) -> list[str | CodeBlockSecurityError]:
preflight_reason_codes = {
reason_code for error in errors if (reason_code := _generated_code_preflight_reason_code(error)) is not None
@@ -11321,6 +11361,41 @@ async def _update_workflow(
"unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),
},
)
+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:
+ unrenderable = _code_block_render_reject(workflow_yaml, getattr(ctx, "workflow_yaml", None))
+ if unrenderable is not None:
+ unrenderable_label, render_diagnostic = unrenderable
+ render_error = (
+ f"Code block `{unrenderable_label}` contains a Jinja expression the workflow runtime "
+ f"cannot render. {render_diagnostic.message}"
+ )
+ LOG.info(
+ "copilot code block render preflight reject",
+ reason_code="code_block_unrenderable",
+ block_label=unrenderable_label,
+ failing_expression=render_diagnostic.failing_expression,
+ )
+ _record_author_time_reject_outcome(
+ ctx,
+ reason_code="code_block_unrenderable",
+ summary=render_error,
+ structural_payload={
+ "reason_code": "code_block_unrenderable",
+ "block_label": unrenderable_label,
+ "failing_expression": render_diagnostic.failing_expression,
+ },
+ block_labels=[unrenderable_label],
+ )
+ _record_code_authoring_guardrail_reject(ctx)
+ return reject(
+ error=render_error,
+ user_facing_summary=_compiled_authoring_user_summary(),
+ data={
+ "reason_code": "code_block_unrenderable",
+ "block_label": unrenderable_label,
+ "failing_expression": render_diagnostic.failing_expression,
+ },
+ )
params["workflow_yaml"] = workflow_yaml
metadata_scrubbed_by_imposition = False
if (
diff --git a/skyvern/utils/templating.py b/skyvern/utils/templating.py
index 102fe8d99..a76d9c0e7 100644
--- a/skyvern/utils/templating.py
+++ b/skyvern/utils/templating.py
@@ -1,6 +1,7 @@
import re
-from jinja2 import Environment, StrictUndefined, UndefinedError, meta
+from jinja2 import StrictUndefined, UndefinedError, meta
+from jinja2.sandbox import SandboxedEnvironment
class Constants:
@@ -30,8 +31,10 @@ def replace_jinja_reference(text: str, old_key: str, new_key: str) -> str:
def get_missing_variables(template_source: str, template_data: dict) -> set[str]:
- # quick check - catch top-level undefineds
- env = Environment(undefined=StrictUndefined)
+ # quick check - catch top-level undefineds. Sandboxed so that rendering
+ # untrusted source below cannot reach attribute-access SSTI gadgets
+ # (e.g. {{ ''.__class__.__mro__ }}) — SandboxedEnvironment raises SecurityError.
+ env = SandboxedEnvironment(undefined=StrictUndefined)
ast = env.parse(template_source)
undeclared_vars = meta.find_undeclared_variables(ast)
missing_vars = undeclared_vars - set(template_data.keys())
diff --git a/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.json b/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.json
new file mode 100644
index 000000000..b95ec3923
--- /dev/null
+++ b/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.json
@@ -0,0 +1,385 @@
+{
+ "_copilot_code_artifact_metadata": {
+ "submit_commercial_water_request": {
+ "artifact_id": "code_artifact:submit_commercial_water_request",
+ "block_label": "submit_commercial_water_request",
+ "claimed_outcomes": [
+ {
+ "covered_criteria": [
+ "safe_result_terminal"
+ ],
+ "criteria_ids": [
+ "safe_result_terminal"
+ ],
+ "depends_on": [
+ "citrus_request_page"
+ ],
+ "entities": [
+ "Citrus County Utilities"
+ ],
+ "evidence_refs": [],
+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}",
+ "extraction_schema_provenance": "self_authored",
+ "goal_value_paths": [
+ "output.confirmation_number",
+ "output.account_number",
+ "output.selected_start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ "output.blocker",
+ "output.evidence_text"
+ ],
+ "id": "safe_request_result",
+ "observation_refs": [
+ "obs_form"
+ ],
+ "required_tokens": [
+ "Submit Request"
+ ],
+ "scope": "block",
+ "status": "observed_not_verified",
+ "text": "Returns confirmation number, account number, selected start date, deposit amount, next owner, and a nullable blocker from the visible terminal page."
+ }
+ ],
+ "completion_criteria": [
+ {
+ "id": "safe_result_terminal",
+ "level": "terminal",
+ "outcome": "safe_request_result",
+ "output_path": "output.blocker",
+ "requested_output_evidence_source": "runtime_output",
+ "terminal": true,
+ "text": "Return the safe structured result from the terminal confirmation page or the manual-service blocker."
+ }
+ ],
+ "declared_goal": "Submit the observed commercial water service form and safely return confirmation details or an explicit blocker.",
+ "evidence_refs": [],
+ "exploration_observations": [
+ {
+ "current_url": "http://localhost:8900/utility_services/citrus_water/",
+ "id": "observed_commercial_form",
+ "observation_ref": "obs_form",
+ "observation_step": 3,
+ "source_label": "commercial_request_form",
+ "source_tool": "inspect_page_for_composition",
+ "status": "observed_not_verified",
+ "text": "Observed the request form and enabled Submit Request button."
+ }
+ ],
+ "observation_refs": [
+ {
+ "claim_id": "safe_request_result",
+ "criterion_id": "safe_result_terminal",
+ "current_url": "http://localhost:8900/utility_services/citrus_water/",
+ "dependency_id": "citrus_request_page",
+ "observation_ref": "obs_form",
+ "observation_step": 3,
+ "source_label": "commercial_request_form",
+ "source_tool": "inspect_page_for_composition",
+ "status": "observed_not_verified"
+ }
+ ],
+ "page_dependencies": [
+ {
+ "evidence_refs": [],
+ "id": "citrus_request_page",
+ "observation_refs": [
+ "obs_form"
+ ],
+ "page_state_hint": "Commercial Water Service Request form.",
+ "required_affordances": [
+ "button[data-action=\"citrusForm\"]",
+ "#company",
+ "#email",
+ "#serviceAddress",
+ "#startDate",
+ "Submit Request button"
+ ],
+ "required_outcomes": [
+ "Form is visible and fillable"
+ ],
+ "scope": "page",
+ "status": "observed_not_verified",
+ "url_hint": "http://localhost:8900/utility_services/citrus_water/"
+ }
+ ],
+ "terminal_verifier_expectations": [
+ {
+ "claimed_outcome_ids": [
+ "safe_request_result"
+ ],
+ "criteria_ids": [
+ "safe_result_terminal"
+ ],
+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}",
+ "extraction_schema_provenance": "self_authored",
+ "goal_value_paths": [
+ "output.confirmation_number",
+ "output.account_number",
+ "output.selected_start_date",
+ "output.deposit_amount",
+ "output.next_owner",
+ "output.blocker",
+ "output.evidence_text"
+ ],
+ "id": "verify_safe_result",
+ "text": "The runtime output contains every requested field, with blocker null only when visible confirmation data is returned."
+ }
+ ]
+ }
+ },
+ "_copilot_unvalidated": true,
+ "_copilot_yaml": "title: Start Citrus County Commercial Water Service\nsummary: Submit a reusable commercial water service request and return its confirmation\n details.\ndescription: Uses configurable business, contact, service address, and start date\n inputs.\nproxy_location: NONE\nwebhook_callback_url: ''\npersist_browser_session: false\npin_saved_session_ip: false\nbrowser_profile_id: null\nbrowser_profile_key: null\nmodel: null\nmax_screenshot_scrolls: null\nmax_elapsed_time_minutes: null\ntotp_verification_url: null\nextra_http_headers: {}\nrun_with: agent\ncache_key: default\nai_fallback: true\nenable_self_healing: false\nworkflow_definition:\n version: 2\n parameters:\n - parameter_type: workflow\n key: business_name\n workflow_parameter_type: string\n description: Business or company name\n - parameter_type: workflow\n key: contact_email\n workflow_parameter_type: string\n description: Contact email for the request\n - parameter_type: workflow\n key: service_address\n workflow_parameter_type: string\n description: Full commercial service address\n - parameter_type: workflow\n key: desired_start_date\n workflow_parameter_type: string\n description: Requested service start date in YYYY-MM-DD format\n blocks:\n - block_type: code\n label: submit_commercial_water_request\n next_block_label: null\n code: \"# Workflow input bindings: {{ parameters.business_name }}, {{ parameters.contact_email\\\n \\ }}, {{ parameters.service_address }}, {{ parameters.desired_start_date }}\\n\\\n await page.goto(\\\"http://localhost:8900/utility_services/citrus_water/\\\")\\n\\\n await page.wait_for_load_state(\\\"domcontentloaded\\\")\\nawait page.locator(\\\"\\\n button[data-action=\\\\\\\"citrusForm\\\\\\\"]\\\").click()\\nawait page.wait_for_load_state(\\\"\\\n domcontentloaded\\\")\\nawait page.locator(\\\"#company\\\").wait_for(state=\\\"visible\\\"\\\n )\\nawait page.locator(\\\"#company\\\").fill(str(business_name).strip())\\nawait\\\n \\ page.locator(\\\"#email\\\").fill(str(contact_email).strip())\\nawait page.locator(\\\"\\\n #serviceAddress\\\").fill(str(service_address).strip())\\nawait page.locator(\\\"\\\n #startDate\\\").fill(str(desired_start_date).strip())\\nsubmit_button = page.get_by_role(\\\"\\\n button\\\", name=\\\"Submit Request\\\", exact=True)\\nawait submit_button.wait_for(state=\\\"\\\n visible\\\")\\nif not await submit_button.is_enabled():\\n body_text = await\\\n \\ page.locator(\\\"body\\\").inner_text()\\n return {\\\"output\\\": {\\\"confirmation_number\\\"\\\n : None, \\\"account_number\\\": None, \\\"selected_start_date\\\": None, \\\"deposit_amount\\\"\\\n : None, \\\"next_owner\\\": None, \\\"blocker\\\": \\\"The online request form is not\\\n \\ reviewable or submit-enabled.\\\", \\\"evidence_text\\\": body_text[:4000]}}\\nawait\\\n \\ submit_button.click()\\nawait page.wait_for_timeout(700)\\nbody_text = await\\\n \\ page.locator(\\\"body\\\").inner_text()\\nlines = [line.strip() for line in body_text.splitlines()\\\n \\ if line.strip()]\\ndef value_after(labels):\\n for index, line in enumerate(lines):\\n\\\n \\ low = line.lower()\\n for label in labels:\\n if label\\\n \\ in low:\\n if \\\":\\\" in line:\\n value = line.split(\\\"\\\n :\\\", 1)[1].strip()\\n if value:\\n return\\\n \\ value\\n if index + 1 < len(lines):\\n return\\\n \\ lines[index + 1]\\n return None\\nconfirmation_number = value_after([\\\"confirmation\\\n \\ number\\\", \\\"confirmation #\\\"])\\naccount_number = value_after([\\\"account number\\\"\\\n , \\\"account #\\\"])\\nselected_start_date = value_after([\\\"selected start date\\\"\\\n , \\\"requested start date\\\", \\\"start date\\\"])\\ndeposit_amount = value_after([\\\"\\\n deposit amount\\\", \\\"deposit\\\"])\\nnext_owner = value_after([\\\"next owner\\\", \\\"\\\n assigned to\\\", \\\"owner\\\"])\\nlower_text = body_text.lower()\\nmanual_only = (\\\"\\\n manual\\\" in lower_text or \\\"email us\\\" in lower_text) and confirmation_number\\\n \\ is None\\nmissing_confirmation = confirmation_number is None and not manual_only\\n\\\n blocker = \\\"The portal exposes an email/manual-service path instead of an online\\\n \\ confirmation.\\\" if manual_only else (\\\"The submitted page did not expose a\\\n \\ confirmation number.\\\" if missing_confirmation else None)\\nreturn {\\\"output\\\"\\\n : {\\\"confirmation_number\\\": confirmation_number, \\\"account_number\\\": account_number,\\\n \\ \\\"selected_start_date\\\": selected_start_date, \\\"deposit_amount\\\": deposit_amount,\\\n \\ \\\"next_owner\\\": next_owner, \\\"blocker\\\": blocker, \\\"evidence_text\\\": body_text[:6000]}}\\n\"\n parameter_keys:\n - business_name\n - contact_email\n - desired_start_date\n - service_address\n extraction_schema: '{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}'\n prompt: Submit the observed commercial water service form and safely return confirmation\n details or an explicit blocker.\nis_saved_task: false\nstatus: published\nrun_sequentially: false\nsequential_key: null\n",
+ "adaptive_caching": false,
+ "agent_id": "wpid_552079557434126732",
+ "ai_fallback": true,
+ "browser_profile_id": null,
+ "browser_profile_key": null,
+ "cache_key": "default",
+ "cdp_connect_headers": null,
+ "code_version": null,
+ "copilot_authored": false,
+ "created_at": "2026-07-16T17:53:26.232218Z",
+ "created_by": null,
+ "deleted_at": null,
+ "description": "Uses configurable business, contact, service address, and start date inputs.",
+ "edited_by": null,
+ "enable_self_healing": false,
+ "extra_http_headers": {},
+ "folder_id": null,
+ "generate_script_on_terminal": false,
+ "import_error": null,
+ "is_saved_task": false,
+ "is_template": false,
+ "max_elapsed_time_minutes": null,
+ "max_screenshot_scrolls": null,
+ "model": null,
+ "modified_at": "2026-07-16T17:53:26.232218Z",
+ "organization_id": "o_551846329558231414",
+ "persist_browser_session": false,
+ "pin_saved_session_ip": false,
+ "proxy_location": "NONE",
+ "run_sequentially": false,
+ "run_with": "agent",
+ "sequential_key": null,
+ "status": "published",
+ "title": "Start Citrus County Commercial Water Service",
+ "totp_identifier": null,
+ "totp_verification_url": null,
+ "version": 1,
+ "webhook_callback_url": "",
+ "workflow_definition": {
+ "blocks": [
+ {
+ "block_type": "code",
+ "code": "# Workflow input bindings: {{ parameters.business_name }}, {{ parameters.contact_email }}, {{ parameters.service_address }}, {{ parameters.desired_start_date }}\nawait page.goto(\"http://localhost:8900/utility_services/citrus_water/\")\nawait page.wait_for_load_state(\"domcontentloaded\")\nawait page.locator(\"button[data-action=\\\"citrusForm\\\"]\").click()\nawait page.wait_for_load_state(\"domcontentloaded\")\nawait page.locator(\"#company\").wait_for(state=\"visible\")\nawait page.locator(\"#company\").fill(str(business_name).strip())\nawait page.locator(\"#email\").fill(str(contact_email).strip())\nawait page.locator(\"#serviceAddress\").fill(str(service_address).strip())\nawait page.locator(\"#startDate\").fill(str(desired_start_date).strip())\nsubmit_button = page.get_by_role(\"button\", name=\"Submit Request\", exact=True)\nawait submit_button.wait_for(state=\"visible\")\nif not await submit_button.is_enabled():\n body_text = await page.locator(\"body\").inner_text()\n return {\"output\": {\"confirmation_number\": None, \"account_number\": None, \"selected_start_date\": None, \"deposit_amount\": None, \"next_owner\": None, \"blocker\": \"The online request form is not reviewable or submit-enabled.\", \"evidence_text\": body_text[:4000]}}\nawait submit_button.click()\nawait page.wait_for_timeout(700)\nbody_text = await page.locator(\"body\").inner_text()\nlines = [line.strip() for line in body_text.splitlines() if line.strip()]\ndef value_after(labels):\n for index, line in enumerate(lines):\n low = line.lower()\n for label in labels:\n if label in low:\n if \":\" in line:\n value = line.split(\":\", 1)[1].strip()\n if value:\n return value\n if index + 1 < len(lines):\n return lines[index + 1]\n return None\nconfirmation_number = value_after([\"confirmation number\", \"confirmation #\"])\naccount_number = value_after([\"account number\", \"account #\"])\nselected_start_date = value_after([\"selected start date\", \"requested start date\", \"start date\"])\ndeposit_amount = value_after([\"deposit amount\", \"deposit\"])\nnext_owner = value_after([\"next owner\", \"assigned to\", \"owner\"])\nlower_text = body_text.lower()\nmanual_only = (\"manual\" in lower_text or \"email us\" in lower_text) and confirmation_number is None\nmissing_confirmation = confirmation_number is None and not manual_only\nblocker = \"The portal exposes an email/manual-service path instead of an online confirmation.\" if manual_only else (\"The submitted page did not expose a confirmation number.\" if missing_confirmation else None)\nreturn {\"output\": {\"confirmation_number\": confirmation_number, \"account_number\": account_number, \"selected_start_date\": selected_start_date, \"deposit_amount\": deposit_amount, \"next_owner\": next_owner, \"blocker\": blocker, \"evidence_text\": body_text[:6000]}}\n",
+ "continue_on_failure": false,
+ "disable_cache": false,
+ "ignore_workflow_system_prompt": false,
+ "label": "submit_commercial_water_request",
+ "model": null,
+ "next_block_label": null,
+ "next_loop_on_failure": false,
+ "output_parameter": {
+ "created_at": "2026-07-16T17:53:26.227963",
+ "deleted_at": null,
+ "description": "Output parameter for block submit_commercial_water_request",
+ "key": "submit_commercial_water_request_output",
+ "modified_at": "2026-07-16T17:53:26.227968",
+ "output_parameter_id": "op_552080278988632478",
+ "parameter_type": "output",
+ "workflow_id": "w_552079557434126730"
+ },
+ "parameters": [
+ {
+ "created_at": "2026-07-16T17:53:26.227993Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Business or company name",
+ "key": "business_name",
+ "modified_at": "2026-07-16T17:53:26.227993Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632480",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228010Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Contact email for the request",
+ "key": "contact_email",
+ "modified_at": "2026-07-16T17:53:26.228010Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632482",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228016Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Requested service start date in YYYY-MM-DD format",
+ "key": "desired_start_date",
+ "modified_at": "2026-07-16T17:53:26.228016Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632486",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228014Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Full commercial service address",
+ "key": "service_address",
+ "modified_at": "2026-07-16T17:53:26.228014Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632484",
+ "workflow_parameter_type": "string"
+ }
+ ],
+ "prompt": "Submit the observed commercial water service form and safely return confirmation details or an explicit blocker.",
+ "steps": [
+ {
+ "action_type": "goto_url",
+ "description": "Open http://localhost:8900/utility_services/citrus_water/",
+ "line_end": 2,
+ "line_start": 2,
+ "title": null
+ },
+ {
+ "action_type": "click",
+ "description": "Click the element",
+ "line_end": 4,
+ "line_start": 4,
+ "title": null
+ },
+ {
+ "action_type": "input_text",
+ "description": "Type into the element",
+ "line_end": 7,
+ "line_start": 7,
+ "title": null
+ },
+ {
+ "action_type": "input_text",
+ "description": "Type into the element",
+ "line_end": 8,
+ "line_start": 8,
+ "title": null
+ },
+ {
+ "action_type": "input_text",
+ "description": "Type into the element",
+ "line_end": 9,
+ "line_start": 9,
+ "title": null
+ },
+ {
+ "action_type": "input_text",
+ "description": "Type into the element",
+ "line_end": 10,
+ "line_start": 10,
+ "title": null
+ },
+ {
+ "action_type": "extract",
+ "description": "Extract information from the page",
+ "line_end": 14,
+ "line_start": 14,
+ "title": null
+ },
+ {
+ "action_type": "click",
+ "description": "Click the element",
+ "line_end": 16,
+ "line_start": 16,
+ "title": null
+ },
+ {
+ "action_type": "wait",
+ "description": "Wait",
+ "line_end": 17,
+ "line_start": 17,
+ "title": null
+ },
+ {
+ "action_type": "extract",
+ "description": "Extract information from the page",
+ "line_end": 18,
+ "line_start": 18,
+ "title": null
+ }
+ ]
+ }
+ ],
+ "error_code_mapping": null,
+ "finally_block_label": null,
+ "parameters": [
+ {
+ "created_at": "2026-07-16T17:53:26.227963",
+ "deleted_at": null,
+ "description": "Output parameter for block submit_commercial_water_request",
+ "key": "submit_commercial_water_request_output",
+ "modified_at": "2026-07-16T17:53:26.227968",
+ "output_parameter_id": "op_552080278988632478",
+ "parameter_type": "output",
+ "workflow_id": "w_552079557434126730"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.227993Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Business or company name",
+ "key": "business_name",
+ "modified_at": "2026-07-16T17:53:26.227993Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632480",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228010Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Contact email for the request",
+ "key": "contact_email",
+ "modified_at": "2026-07-16T17:53:26.228010Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632482",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228014Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Full commercial service address",
+ "key": "service_address",
+ "modified_at": "2026-07-16T17:53:26.228014Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632484",
+ "workflow_parameter_type": "string"
+ },
+ {
+ "created_at": "2026-07-16T17:53:26.228016Z",
+ "default_value": null,
+ "deleted_at": null,
+ "description": "Requested service start date in YYYY-MM-DD format",
+ "key": "desired_start_date",
+ "modified_at": "2026-07-16T17:53:26.228016Z",
+ "parameter_type": "workflow",
+ "workflow_id": "w_552079557434126730",
+ "workflow_parameter_id": "wp_552080278988632486",
+ "workflow_parameter_type": "string"
+ }
+ ],
+ "version": 2,
+ "workflow_system_prompt": null
+ },
+ "workflow_id": "w_552079557434126730",
+ "workflow_permanent_id": "wpid_552079557434126732"
+}
diff --git a/tests/unit/test_code_block_preflight.py b/tests/unit/test_code_block_preflight.py
new file mode 100644
index 000000000..3a51cb22c
--- /dev/null
+++ b/tests/unit/test_code_block_preflight.py
@@ -0,0 +1,104 @@
+"""Tests for author-time render validation of copilot code blocks.
+
+OSS-synced: only example.* placeholder targets and synthetic labels.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.forge.sdk.copilot.code_block_preflight import (
+ RENDER_TEMPLATE_SYNTAX_REASON_CODE,
+ RENDER_UNDEFINED_NAME_REASON_CODE,
+ CodeBlockRenderDiagnostic,
+ code_block_render_diagnostic,
+)
+
+_BOUND_NAMES = frozenset({"business_name", "contact_email", "submit_request", "submit_request_output"})
+
+
+class TestCodeBlockRenderDiagnosticRejects:
+ def test_parameters_namespace_reference_is_unrenderable(self) -> None:
+ code = (
+ "# Workflow input bindings: {{ parameters.business_name }}\n"
+ 'await page.goto("https://example.com/request")\n'
+ 'await page.locator("#company").fill(str(business_name).strip())\n'
+ )
+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)
+ assert diagnostic is not None
+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE
+ assert diagnostic.failing_expression == "{{ parameters.business_name }}"
+ assert "{{ business_name }}" in diagnostic.message
+
+ def test_undeclared_root_is_unrenderable(self) -> None:
+ diagnostic = code_block_render_diagnostic("value = str({{ frobnicator }})", _BOUND_NAMES)
+ assert diagnostic is not None
+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE
+ assert diagnostic.failing_expression == "{{ frobnicator }}"
+ assert "frobnicator" in diagnostic.message
+
+ def test_template_syntax_error_is_unrenderable(self) -> None:
+ diagnostic = code_block_render_diagnostic("value = {{ business_name\nother = 1", _BOUND_NAMES)
+ assert diagnostic is not None
+ assert diagnostic.code == RENDER_TEMPLATE_SYNTAX_REASON_CODE
+ assert diagnostic.message
+
+ def test_statement_only_undeclared_root_is_attributed(self) -> None:
+ code = "{% if unknown_flag %}\nvalue = 1\n{% endif %}"
+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)
+ assert diagnostic is not None
+ assert "unknown_flag" in diagnostic.failing_expression
+
+ def test_unattributable_undefined_yields_diagnostic_not_exception(self) -> None:
+ code = "{% macro helper() %}{{ caller() }}{% endmacro %}\nvalue = {{ helper() }}"
+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)
+ assert isinstance(diagnostic, CodeBlockRenderDiagnostic)
+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE
+
+ @pytest.mark.parametrize(
+ "gadget",
+ [
+ "value = {{ ''.__class__.__mro__[1].__subclasses__() }}",
+ "value = {{ business_name.__class__.__init__.__globals__ }}",
+ ],
+ )
+ def test_ssti_gadget_is_rejected_without_executing(self, gadget: str) -> None:
+ diagnostic = code_block_render_diagnostic(gadget, _BOUND_NAMES)
+ assert diagnostic is not None
+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE
+
+ def test_loop_names_outside_loop_scope_are_unrenderable(self) -> None:
+ diagnostic = code_block_render_diagnostic("value = {{ current_item }}", _BOUND_NAMES)
+ assert diagnostic is not None
+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE
+ assert "current_item" in diagnostic.message
+
+
+class TestCodeBlockRenderDiagnosticPasses:
+ @pytest.mark.parametrize(
+ "code",
+ [
+ 'await page.locator("#company").fill("{{ business_name }}")',
+ "value = {{ submit_request_output.field }}",
+ "value = {{ submit_request_output['nested'][0] }}",
+ "today = {{ current_date }}",
+ "payload = {{ business_name | json }}",
+ "{% for item in workflow_run_outputs %}{{ item }}{% endfor %}",
+ ],
+ )
+ def test_renderable_templates_pass(self, code: str) -> None:
+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is None
+
+ @pytest.mark.parametrize(
+ "code",
+ [
+ "{% for item in workflow_run_outputs %}{{ current_index }}{% endfor %}",
+ "{% for item in workflow_run_outputs %}\nvalue = {{ current_item }}\n{% endfor %}",
+ ],
+ )
+ def test_loop_names_pass_inside_loop(self, code: str) -> None:
+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is None
+
+ def test_jinja_free_code_passes(self) -> None:
+ code = 'await page.goto("https://example.com")\nreturn {"output": {"a": 1}}'
+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is None
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index 5eddfe89e..290963080 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -10,6 +10,7 @@ import hashlib
import inspect
import json
import textwrap
+from pathlib import Path
from types import EllipsisType, SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -98,6 +99,7 @@ from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modul
from skyvern.forge.sdk.copilot.tools import scouting as scouting_module
from skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_module
from skyvern.forge.sdk.copilot.tools.workflow_update import (
+ _code_block_render_reject,
_code_safety_reject_payload,
_OutputContractEvaluation,
_strip_redundant_sandbox_imports_in_yaml,
@@ -647,6 +649,27 @@ def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") ->
)
+class TestCredentialRealValueBindings:
+ def test_real_username_reference_is_not_falsely_rejected(self) -> None:
+ yaml_text = _credential_code_yaml(
+ code='await page.locator("#email").fill("{{ login_credential_real_username }}")',
+ )
+ assert _code_block_render_reject(yaml_text, None) is None
+
+ def test_real_password_reference_is_not_falsely_rejected(self) -> None:
+ yaml_text = _credential_code_yaml(
+ code='await page.locator("input[type=\'password\']").fill("{{ login_credential_real_password }}")',
+ )
+ assert _code_block_render_reject(yaml_text, None) is None
+
+ def test_unbound_credential_derived_name_still_rejected(self) -> None:
+ yaml_text = _credential_code_yaml(
+ code='await page.locator("#email").fill("{{ nonexistent_credential_real_username }}")',
+ )
+ result = _code_block_render_reject(yaml_text, None)
+ assert result is not None
+
+
def _directory_blocks_yaml(blocks: str) -> str:
indented_blocks = textwrap.indent(textwrap.dedent(blocks).strip(), " ")
return f"title: Directory lookup\nworkflow_definition:\n blocks:\n{indented_blocks}\n"
@@ -19034,3 +19057,180 @@ class TestFreehandSurfacePersistSeam:
code = 'await page.get_by_label("State").select_option("CA")'
result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)
assert result is None
+
+
+_RENDER_PREFLIGHT_FIXTURE_PATH = (
+ Path(__file__).parent / "fixtures" / "copilot" / "wr_jinja_unrenderable_proposed_workflow.json"
+)
+
+_PARAMETERS_NAMESPACE_CODE_YAML = _yaml(
+ """
+ title: Utility service request
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}
+ blocks:
+ - block_type: code
+ label: submit_request
+ parameter_keys: [business_name]
+ code: |
+ # Workflow input bindings: {{ parameters.business_name }}
+ await page.goto("https://example.com/request")
+ await page.locator("#company").fill(str(business_name).strip())
+ """
+)
+
+_UNDECLARED_ROOT_CODE_YAML = _yaml(
+ """
+ title: Utility service request
+ workflow_definition:
+ blocks:
+ - block_type: code
+ label: submit_request
+ code: |
+ await page.goto("https://example.com/request")
+ await page.locator("#company").fill("{{ frobnicator }}")
+ """
+)
+
+_TEMPLATE_SYNTAX_ERROR_CODE_YAML = _yaml(
+ """
+ title: Utility service request
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}
+ blocks:
+ - block_type: code
+ label: submit_request
+ parameter_keys: [business_name]
+ code: |
+ await page.goto("https://example.com/request")
+ await page.locator("#company").fill("{{ business_name")
+ """
+)
+
+_RENDERABLE_JINJA_CODE_YAML = _yaml(
+ """
+ title: Utility service request
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}
+ blocks:
+ - block_type: code
+ label: submit_request
+ parameter_keys: [business_name]
+ code: |
+ await page.goto("https://example.com/request")
+ await page.locator("#company").fill("{{ business_name }}")
+ """
+)
+
+
+class TestCodeBlockRenderPreflightSeam:
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_rejects_parameters_namespace_at_persist_seam(self) -> None:
+ ctx = _code_only_ctx()
+ with capture_logs() as logs:
+ result = await _update_workflow({"workflow_yaml": _PARAMETERS_NAMESPACE_CODE_YAML}, ctx)
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "code_block_unrenderable"
+ assert result["data"]["block_label"] == "submit_request"
+ assert result["data"]["failing_expression"] == "{{ parameters.business_name }}"
+ assert "{{ business_name }}" in result["error"]
+ outcome = ctx.latest_recorded_build_test_outcome
+ assert isinstance(outcome, RecordedBuildTestOutcome)
+ assert outcome.reason_code == "code_block_unrenderable"
+ assert any(
+ log.get("reason_code") == "code_block_unrenderable"
+ and log.get("failing_expression") == "{{ parameters.business_name }}"
+ for log in logs
+ )
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_rejects_undeclared_root_at_persist_seam(self) -> None:
+ result = await _update_workflow({"workflow_yaml": _UNDECLARED_ROOT_CODE_YAML}, _code_only_ctx())
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "code_block_unrenderable"
+ assert result["data"]["failing_expression"] == "{{ frobnicator }}"
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_rejects_template_syntax_error_at_persist_seam(self) -> None:
+ result = await _update_workflow({"workflow_yaml": _TEMPLATE_SYNTAX_ERROR_CODE_YAML}, _code_only_ctx())
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "code_block_unrenderable"
+ assert "syntax error" in result["error"]
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_renderable_draft_persists(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ result = await _update_workflow({"workflow_yaml": _RENDERABLE_JINJA_CODE_YAML}, ctx)
+ assert result["ok"] is True
+ assert "{{ business_name }}" in ctx.workflow_yaml
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_renderable_draft_with_envelope_imposition_persists(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.request_policy = RequestPolicy(
+ completion_criteria=[
+ _typed_completion_criterion(
+ id="requested_value",
+ output_path="output.record_id",
+ level="run",
+ method_mandated=False,
+ kind="outcome",
+ )
+ ]
+ )
+ workflow_yaml = _yaml(
+ """
+ title: Entry lookup
+ workflow_definition:
+ parameters:
+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}
+ blocks:
+ - block_type: code
+ label: extract_entry_output
+ parameter_keys: [business_name]
+ code: |
+ record_id = "{{ business_name }}"
+ """
+ )
+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)
+ assert result["ok"] is True
+ parsed = parse_workflow_yaml(ctx.workflow_yaml)
+ assert isinstance(parsed, dict)
+ code = str(_single_code_block(parsed)["code"])
+ assert "{{ business_name }}" in code
+ assert 'return {"output": {"record_id": record_id}}' in code
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_unchanged_legacy_block_is_not_rerejected(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ _stub_successful_update(monkeypatch)
+ ctx = _code_only_ctx()
+ ctx.workflow_yaml = _PARAMETERS_NAMESPACE_CODE_YAML
+ result = await _update_workflow({"workflow_yaml": _PARAMETERS_NAMESPACE_CODE_YAML}, ctx)
+ assert (result.get("data") or {}).get("reason_code") != "code_block_unrenderable"
+ assert result["ok"] is True
+
+ @pytest.mark.asyncio
+ async def test_code_block_preflight_rejects_recorded_unrenderable_draft_replay(self) -> None:
+ artifact = json.loads(_RENDER_PREFLIGHT_FIXTURE_PATH.read_text())
+ ctx = _code_only_ctx()
+ with capture_logs() as logs:
+ result = await _update_workflow({"workflow_yaml": artifact["_copilot_yaml"]}, ctx)
+ assert result["ok"] is False
+ assert result["data"]["reason_code"] == "code_block_unrenderable"
+ assert result["data"]["block_label"] == "submit_commercial_water_request"
+ assert "parameters.business_name" in result["data"]["failing_expression"]
+ assert "{{ business_name }}" in result["error"]
+ assert any(
+ log.get("reason_code") == "code_block_unrenderable"
+ and "parameters.business_name" in str(log.get("failing_expression"))
+ for log in logs
+ )
diff --git a/tests/unit/test_utils_templating.py b/tests/unit/test_utils_templating.py
index 33d81d81d..49e7de5a3 100644
--- a/tests/unit/test_utils_templating.py
+++ b/tests/unit/test_utils_templating.py
@@ -1,6 +1,7 @@
import re
import pytest
+from jinja2.exceptions import SecurityError
from skyvern.utils.templating import Constants, get_missing_variables
@@ -46,3 +47,16 @@ def test_get_missing_variables(template, data, expected):
def test_regex_missing_variable_pattern(template, expected):
matches = set(re.findall(Constants.MissingVariablePattern, template))
assert matches == expected
+
+
+ "gadget",
+ [
+ "{{ ''.__class__.__mro__[1].__subclasses__() }}",
+ "{{ [].__class__.__base__ }}",
+ "{% for cls in ().__class__.__bases__ %}{{ cls }}{% endfor %}",
+ ],
+)
+def test_get_missing_variables_never_renders_gadgets_unsandboxed(gadget):
+ with pytest.raises(SecurityError):
+ get_missing_variables(gadget, {})