Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Render-validate copilot code blocks at the persist seam (SKY-12591) (#7512)
Andrew Neilson58b0397cc80d
diff --git a/skyvern/forge/sdk/copilot/build_test_outcome.py b/skyvern/forge/sdk/copilot/build_test_outcome.pyindex ffe49946c..de2852bd7 100644--- a/skyvern/forge/sdk/copilot/build_test_outcome.py+++ b/skyvern/forge/sdk/copilot/build_test_outcome.py@@ -45,6 +45,7 @@ BuildTestOutcomeReasonCode = Literal["sandbox_unresolved_name","synthesized_parameter_binding_ambiguous","code_safety_reject",+ "code_block_unrenderable","credential_scout_reject","schema_incompatibility","verified_success",diff --git a/skyvern/forge/sdk/copilot/code_block_preflight.py b/skyvern/forge/sdk/copilot/code_block_preflight.pyindex 36310aeb7..3c386a8d5 100644--- a/skyvern/forge/sdk/copilot/code_block_preflight.py+++ b/skyvern/forge/sdk/copilot/code_block_preflight.py@@ -12,12 +12,20 @@ from dataclasses import dataclassfrom functools import cachefrom pathlib import Pathfrom types import SimpleNamespace-from typing import Iterable, Mapping+from typing import Iterable, Iterator, Mapping++from jinja2 import StrictUndefined, TemplateSyntaxError, UndefinedError+from jinja2.exceptions import SecurityError+from jinja2.sandbox import SandboxedEnvironmentfrom skyvern.forge.sdk.copilot.code_block_security import CodeBlockSecurityError, author_time_code_security_errors+from skyvern.forge.sdk.workflow.models._jinja import _json_finalize, _json_type_filterfrom skyvern.forge.sdk.workflow.models.block import CodeBlock+from skyvern.utils.templating import get_missing_variablesSANDBOX_UNRESOLVED_NAME_REASON_CODE = "SANDBOX_UNRESOLVED_NAME"+RENDER_TEMPLATE_SYNTAX_REASON_CODE = "RENDER_TEMPLATE_SYNTAX"+RENDER_UNDEFINED_NAME_REASON_CODE = "RENDER_UNDEFINED_NAME"@dataclass(frozen=True)@@ -26,6 +34,13 @@ class CodeBlockPreflightDiagnostic:message: str+@dataclass(frozen=True)+class CodeBlockRenderDiagnostic:+ code: str+ message: str+ failing_expression: str++@dataclass(frozen=True)class CodeBlockSandboxNameDiagnostic:code: str@@ -37,6 +52,139 @@ class CodeBlockSandboxNameDiagnostic:allowed_helper_surface: dict[str, tuple[str, ...]]+# Mirrors the runtime's strict-mode template formatter (jinja_json_finalize_strict_env)+# regardless of the WORKFLOW_TEMPLATING_STRICTNESS deployment setting.+_render_check_env = SandboxedEnvironment(undefined=StrictUndefined, finalize=_json_finalize)+_render_check_env.filters["json"] = _json_type_filter++_RENDER_SYSTEM_BINDING_NAMES = (+ "workflow_title",+ "workflow_id",+ "workflow_permanent_id",+ "workflow_run_id",+ "current_date",+ "browser_session_id",+ "workflow_run_outputs",+ "workflow_run_summary",+)++# The runtime injects these only inside a for-loop iteration, so bind them only+# when the source actually opens a loop. ponytail: loop-presence heuristic, not+# true per-scope tracking — a reference after the loop closes still passes.+_RENDER_LOOP_BINDING_NAMES = (+ "current_index",+ "current_item",+ "current_value",+)++_JINJA_EXPRESSION_RE = re.compile(r"\{\{.*?\}\}", re.DOTALL)+_JINJA_STATEMENT_RE = re.compile(r"\{%.*?%\}", re.DOTALL)+_JINJA_FOR_STATEMENT_RE = re.compile(r"\{%-?\s*for\s", re.DOTALL)+++class _PermissiveRenderBinding:+ def __getattr__(self, name: str) -> _PermissiveRenderBinding:+ return self++ def __getitem__(self, key: object) -> _PermissiveRenderBinding:+ return self++ # Without __iter__, __getitem__ triggers Python's legacy iteration protocol,+ # which never raises IndexError here and spins {% for %} loops forever.+ def __iter__(self) -> Iterator[_PermissiveRenderBinding]:+ return iter((self,))++ def __str__(self) -> str:+ return "value"+++def _first_template_expression_for(code: str, root: str) -> str:+ root_re = re.compile(rf"(?<![A-Za-z0-9_]){re.escape(root)}(?![A-Za-z0-9_])")+ for pattern in (_JINJA_EXPRESSION_RE, _JINJA_STATEMENT_RE):+ for match in pattern.finditer(code):+ if root_re.search(match.group(0)):+ return match.group(0)+ return f"{{{{ {root} }}}}"+++def _top_level_form_suggestion(expression: str, root: str) -> str:+ inner = expression.strip().strip("{}%").strip()+ if not inner.startswith(f"{root}."):+ return ""+ remainder = inner[len(root) + 1 :]+ match = re.match(r"[A-Za-z_][A-Za-z0-9_]*", remainder)+ if match is None:+ return ""+ return f"{{{{ {match.group(0)} }}}}"+++def code_block_render_diagnostic(code: str, bound_names: Iterable[str]) -> CodeBlockRenderDiagnostic | None:+ """Dry-render the code block through the runtime's strict Jinja semantics with every+ runtime-provided name bound to a permissive sentinel; only genuinely unrenderable+ templates (undefined names, syntax errors, sandbox violations) produce a diagnostic."""+ if "{{" not in code and "{%" not in code:+ return None+ try:+ template = _render_check_env.from_string(code)+ except TemplateSyntaxError as exc:+ source_lines = code.splitlines()+ line = source_lines[exc.lineno - 1].strip() if exc.lineno and exc.lineno <= len(source_lines) else ""+ detail = f" Offending line: `{line}`." if line else ""+ return CodeBlockRenderDiagnostic(+ code=RENDER_TEMPLATE_SYNTAX_REASON_CODE,+ message=f"Jinja template syntax error on line {exc.lineno}: {exc.message}.{detail}",+ failing_expression=line,+ )+ bindings: dict[str, object] = {name: _PermissiveRenderBinding() for name in bound_names}+ system_names: tuple[str, ...] = _RENDER_SYSTEM_BINDING_NAMES+ if _JINJA_FOR_STATEMENT_RE.search(code):+ system_names = system_names + _RENDER_LOOP_BINDING_NAMES+ for name in system_names:+ bindings.setdefault(name, _PermissiveRenderBinding())+ missing: set[str] = set()+ try:+ missing = get_missing_variables(code, bindings)+ except (UndefinedError, SecurityError) as exc:+ return CodeBlockRenderDiagnostic(+ code=RENDER_UNDEFINED_NAME_REASON_CODE,+ message=f"A Jinja expression in this code block cannot render at runtime: {exc}.",+ failing_expression="",+ )+ except Exception:+ missing = set()+ if missing:+ root = sorted(missing)[0].split("[")[0].split(".")[0]+ expression = _first_template_expression_for(code, root)+ suggestion = _top_level_form_suggestion(expression, root)+ guidance = (+ f" Declared inputs are injected as top-level names; write `{suggestion}` instead."+ if suggestion+ else (+ " Only declared parameter keys, block labels, `<label>_output` values, and workflow "+ "system names (e.g. `current_date`) are available as top-level template names."+ )+ )+ return CodeBlockRenderDiagnostic(+ code=RENDER_UNDEFINED_NAME_REASON_CODE,+ message=(+ f"The expression `{expression}` cannot render at runtime: "+ f"`{root}` is not a defined template name.{guidance}"+ ),+ failing_expression=expression,+ )+ try:+ template.render(bindings)+ except (UndefinedError, SecurityError) as exc:+ return CodeBlockRenderDiagnostic(+ code=RENDER_UNDEFINED_NAME_REASON_CODE,+ message=f"A Jinja expression in this code block cannot render at runtime: {exc}.",+ failing_expression="",+ )+ except Exception:+ return None+ return None++_ANSI_ESCAPE_RE = re.compile(r"\x1b\[[0-9;]*m|\x1b\([AB]")_MYPY_ERROR_RE = re.compile(r"^(?P<path>.*?):(?P<line>\d+): (?P<severity>error): (?P<message>.*)$")_LOCATOR_NOT_CALLABLE_RE = re.compile(r'"Locator" not callable\s+\[operator\]')diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex 056b66415..175887b59 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -50,7 +50,9 @@ from skyvern.forge.sdk.copilot.build_test_outcome import ()from skyvern.forge.sdk.copilot.code_block_preflight import (SANDBOX_UNRESOLVED_NAME_REASON_CODE,+ CodeBlockRenderDiagnostic,author_time_code_block_diagnostics,+ code_block_render_diagnostic,sandbox_unresolved_name_diagnostics,sandbox_unresolved_name_repair_diagnostic,strip_redundant_sandbox_imports,@@ -141,6 +143,7 @@ from skyvern.forge.sdk.copilot.output_contracts import (classify_output_contract_bail_family,code_block_available_binding_keys_by_label,declared_string_workflow_parameter_keys,+ declared_workflow_parameter_keys,resolve_output_contract_actuation,)from skyvern.forge.sdk.copilot.output_extraction_plan import FrozenRequestedOutputExtractionCandidate@@ -200,6 +203,7 @@ from skyvern.forge.sdk.copilot.turn_ownership import ()from skyvern.forge.sdk.copilot.workflow_credential_utils import (credential_param_ids,+ credential_params,parse_workflow_yaml,workflow_blocks,)@@ -1065,6 +1069,42 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)return errors+def _code_block_render_bound_names(workflow_yaml: str) -> set[str]:+ parsed = parse_workflow_yaml(workflow_yaml)+ if not isinstance(parsed, dict):+ return set()+ names = set(declared_workflow_parameter_keys(parsed))+ workflow_definition = parsed.get("workflow_definition")+ if isinstance(workflow_definition, dict):+ for key in credential_params(workflow_definition.get("parameters")):+ names.add(f"{key}_real_username")+ names.add(f"{key}_real_password")+ for block in workflow_blocks(parsed):+ label = str(block.get("label") or "").strip()+ if label:+ names.add(label)+ names.add(f"{label}_output")+ return names+++def _code_block_render_reject(+ workflow_yaml: str, prior_yaml: str | None+) -> tuple[str, CodeBlockRenderDiagnostic] | None:+ prior_blocks = _workflow_yaml_code_blocks_by_label(prior_yaml)+ bound_names = _code_block_render_bound_names(workflow_yaml)+ for label, block in _workflow_yaml_code_blocks_by_label(workflow_yaml).items():+ code = str(block.get("code") or "")+ if not code.strip():+ continue+ prior_block = prior_blocks.get(label)+ if prior_block is not None and str(prior_block.get("code") or "") == code:+ continue+ diagnostic = code_block_render_diagnostic(code, bound_names)+ if diagnostic is not None:+ return label, diagnostic+ return None++def _human_facing_code_safety_errors(errors: list[str | CodeBlockSecurityError]) -> list[str | CodeBlockSecurityError]:preflight_reason_codes = {reason_code for error in errors if (reason_code := _generated_code_preflight_reason_code(error)) is not None@@ -11321,6 +11361,41 @@ async def _update_workflow("unreferenced_parameter_keys": list(definition_reject.unreferenced_parameter_keys),},)+ if _copilot_block_authoring_policy(ctx) == BlockAuthoringPolicy.CODE_ONLY_BROWSER:+ unrenderable = _code_block_render_reject(workflow_yaml, getattr(ctx, "workflow_yaml", None))+ if unrenderable is not None:+ unrenderable_label, render_diagnostic = unrenderable+ render_error = (+ f"Code block `{unrenderable_label}` contains a Jinja expression the workflow runtime "+ f"cannot render. {render_diagnostic.message}"+ )+ LOG.info(+ "copilot code block render preflight reject",+ reason_code="code_block_unrenderable",+ block_label=unrenderable_label,+ failing_expression=render_diagnostic.failing_expression,+ )+ _record_author_time_reject_outcome(+ ctx,+ reason_code="code_block_unrenderable",+ summary=render_error,+ structural_payload={+ "reason_code": "code_block_unrenderable",+ "block_label": unrenderable_label,+ "failing_expression": render_diagnostic.failing_expression,+ },+ block_labels=[unrenderable_label],+ )+ _record_code_authoring_guardrail_reject(ctx)+ return reject(+ error=render_error,+ user_facing_summary=_compiled_authoring_user_summary(),+ data={+ "reason_code": "code_block_unrenderable",+ "block_label": unrenderable_label,+ "failing_expression": render_diagnostic.failing_expression,+ },+ )params["workflow_yaml"] = workflow_yamlmetadata_scrubbed_by_imposition = Falseif (diff --git a/skyvern/utils/templating.py b/skyvern/utils/templating.pyindex 102fe8d99..a76d9c0e7 100644--- a/skyvern/utils/templating.py+++ b/skyvern/utils/templating.py@@ -1,6 +1,7 @@import re-from jinja2 import Environment, StrictUndefined, UndefinedError, meta+from jinja2 import StrictUndefined, UndefinedError, meta+from jinja2.sandbox import SandboxedEnvironmentclass Constants:@@ -30,8 +31,10 @@ def replace_jinja_reference(text: str, old_key: str, new_key: str) -> str:def get_missing_variables(template_source: str, template_data: dict) -> set[str]:- # quick check - catch top-level undefineds- env = Environment(undefined=StrictUndefined)+ # quick check - catch top-level undefineds. Sandboxed so that rendering+ # untrusted source below cannot reach attribute-access SSTI gadgets+ # (e.g. {{ ''.__class__.__mro__ }}) — SandboxedEnvironment raises SecurityError.+ env = SandboxedEnvironment(undefined=StrictUndefined)ast = env.parse(template_source)undeclared_vars = meta.find_undeclared_variables(ast)missing_vars = undeclared_vars - set(template_data.keys())diff --git a/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.json b/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.jsonnew file mode 100644index 000000000..b95ec3923--- /dev/null+++ b/tests/unit/fixtures/copilot/wr_jinja_unrenderable_proposed_workflow.json@@ -0,0 +1,385 @@+{+ "_copilot_code_artifact_metadata": {+ "submit_commercial_water_request": {+ "artifact_id": "code_artifact:submit_commercial_water_request",+ "block_label": "submit_commercial_water_request",+ "claimed_outcomes": [+ {+ "covered_criteria": [+ "safe_result_terminal"+ ],+ "criteria_ids": [+ "safe_result_terminal"+ ],+ "depends_on": [+ "citrus_request_page"+ ],+ "entities": [+ "Citrus County Utilities"+ ],+ "evidence_refs": [],+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}",+ "extraction_schema_provenance": "self_authored",+ "goal_value_paths": [+ "output.confirmation_number",+ "output.account_number",+ "output.selected_start_date",+ "output.deposit_amount",+ "output.next_owner",+ "output.blocker",+ "output.evidence_text"+ ],+ "id": "safe_request_result",+ "observation_refs": [+ "obs_form"+ ],+ "required_tokens": [+ "Submit Request"+ ],+ "scope": "block",+ "status": "observed_not_verified",+ "text": "Returns confirmation number, account number, selected start date, deposit amount, next owner, and a nullable blocker from the visible terminal page."+ }+ ],+ "completion_criteria": [+ {+ "id": "safe_result_terminal",+ "level": "terminal",+ "outcome": "safe_request_result",+ "output_path": "output.blocker",+ "requested_output_evidence_source": "runtime_output",+ "terminal": true,+ "text": "Return the safe structured result from the terminal confirmation page or the manual-service blocker."+ }+ ],+ "declared_goal": "Submit the observed commercial water service form and safely return confirmation details or an explicit blocker.",+ "evidence_refs": [],+ "exploration_observations": [+ {+ "current_url": "http://localhost:8900/utility_services/citrus_water/",+ "id": "observed_commercial_form",+ "observation_ref": "obs_form",+ "observation_step": 3,+ "source_label": "commercial_request_form",+ "source_tool": "inspect_page_for_composition",+ "status": "observed_not_verified",+ "text": "Observed the request form and enabled Submit Request button."+ }+ ],+ "observation_refs": [+ {+ "claim_id": "safe_request_result",+ "criterion_id": "safe_result_terminal",+ "current_url": "http://localhost:8900/utility_services/citrus_water/",+ "dependency_id": "citrus_request_page",+ "observation_ref": "obs_form",+ "observation_step": 3,+ "source_label": "commercial_request_form",+ "source_tool": "inspect_page_for_composition",+ "status": "observed_not_verified"+ }+ ],+ "page_dependencies": [+ {+ "evidence_refs": [],+ "id": "citrus_request_page",+ "observation_refs": [+ "obs_form"+ ],+ "page_state_hint": "Commercial Water Service Request form.",+ "required_affordances": [+ "button[data-action=\"citrusForm\"]",+ "#company",+ "#email",+ "#serviceAddress",+ "#startDate",+ "Submit Request button"+ ],+ "required_outcomes": [+ "Form is visible and fillable"+ ],+ "scope": "page",+ "status": "observed_not_verified",+ "url_hint": "http://localhost:8900/utility_services/citrus_water/"+ }+ ],+ "terminal_verifier_expectations": [+ {+ "claimed_outcome_ids": [+ "safe_request_result"+ ],+ "criteria_ids": [+ "safe_result_terminal"+ ],+ "extraction_schema": "{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}",+ "extraction_schema_provenance": "self_authored",+ "goal_value_paths": [+ "output.confirmation_number",+ "output.account_number",+ "output.selected_start_date",+ "output.deposit_amount",+ "output.next_owner",+ "output.blocker",+ "output.evidence_text"+ ],+ "id": "verify_safe_result",+ "text": "The runtime output contains every requested field, with blocker null only when visible confirmation data is returned."+ }+ ]+ }+ },+ "_copilot_unvalidated": true,+ "_copilot_yaml": "title: Start Citrus County Commercial Water Service\nsummary: Submit a reusable commercial water service request and return its confirmation\n details.\ndescription: Uses configurable business, contact, service address, and start date\n inputs.\nproxy_location: NONE\nwebhook_callback_url: ''\npersist_browser_session: false\npin_saved_session_ip: false\nbrowser_profile_id: null\nbrowser_profile_key: null\nmodel: null\nmax_screenshot_scrolls: null\nmax_elapsed_time_minutes: null\ntotp_verification_url: null\nextra_http_headers: {}\nrun_with: agent\ncache_key: default\nai_fallback: true\nenable_self_healing: false\nworkflow_definition:\n version: 2\n parameters:\n - parameter_type: workflow\n key: business_name\n workflow_parameter_type: string\n description: Business or company name\n - parameter_type: workflow\n key: contact_email\n workflow_parameter_type: string\n description: Contact email for the request\n - parameter_type: workflow\n key: service_address\n workflow_parameter_type: string\n description: Full commercial service address\n - parameter_type: workflow\n key: desired_start_date\n workflow_parameter_type: string\n description: Requested service start date in YYYY-MM-DD format\n blocks:\n - block_type: code\n label: submit_commercial_water_request\n next_block_label: null\n code: \"# Workflow input bindings: {{ parameters.business_name }}, {{ parameters.contact_email\\\n \\ }}, {{ parameters.service_address }}, {{ parameters.desired_start_date }}\\n\\\n await page.goto(\\\"http://localhost:8900/utility_services/citrus_water/\\\")\\n\\\n await page.wait_for_load_state(\\\"domcontentloaded\\\")\\nawait page.locator(\\\"\\\n button[data-action=\\\\\\\"citrusForm\\\\\\\"]\\\").click()\\nawait page.wait_for_load_state(\\\"\\\n domcontentloaded\\\")\\nawait page.locator(\\\"#company\\\").wait_for(state=\\\"visible\\\"\\\n )\\nawait page.locator(\\\"#company\\\").fill(str(business_name).strip())\\nawait\\\n \\ page.locator(\\\"#email\\\").fill(str(contact_email).strip())\\nawait page.locator(\\\"\\\n #serviceAddress\\\").fill(str(service_address).strip())\\nawait page.locator(\\\"\\\n #startDate\\\").fill(str(desired_start_date).strip())\\nsubmit_button = page.get_by_role(\\\"\\\n button\\\", name=\\\"Submit Request\\\", exact=True)\\nawait submit_button.wait_for(state=\\\"\\\n visible\\\")\\nif not await submit_button.is_enabled():\\n body_text = await\\\n \\ page.locator(\\\"body\\\").inner_text()\\n return {\\\"output\\\": {\\\"confirmation_number\\\"\\\n : None, \\\"account_number\\\": None, \\\"selected_start_date\\\": None, \\\"deposit_amount\\\"\\\n : None, \\\"next_owner\\\": None, \\\"blocker\\\": \\\"The online request form is not\\\n \\ reviewable or submit-enabled.\\\", \\\"evidence_text\\\": body_text[:4000]}}\\nawait\\\n \\ submit_button.click()\\nawait page.wait_for_timeout(700)\\nbody_text = await\\\n \\ page.locator(\\\"body\\\").inner_text()\\nlines = [line.strip() for line in body_text.splitlines()\\\n \\ if line.strip()]\\ndef value_after(labels):\\n for index, line in enumerate(lines):\\n\\\n \\ low = line.lower()\\n for label in labels:\\n if label\\\n \\ in low:\\n if \\\":\\\" in line:\\n value = line.split(\\\"\\\n :\\\", 1)[1].strip()\\n if value:\\n return\\\n \\ value\\n if index + 1 < len(lines):\\n return\\\n \\ lines[index + 1]\\n return None\\nconfirmation_number = value_after([\\\"confirmation\\\n \\ number\\\", \\\"confirmation #\\\"])\\naccount_number = value_after([\\\"account number\\\"\\\n , \\\"account #\\\"])\\nselected_start_date = value_after([\\\"selected start date\\\"\\\n , \\\"requested start date\\\", \\\"start date\\\"])\\ndeposit_amount = value_after([\\\"\\\n deposit amount\\\", \\\"deposit\\\"])\\nnext_owner = value_after([\\\"next owner\\\", \\\"\\\n assigned to\\\", \\\"owner\\\"])\\nlower_text = body_text.lower()\\nmanual_only = (\\\"\\\n manual\\\" in lower_text or \\\"email us\\\" in lower_text) and confirmation_number\\\n \\ is None\\nmissing_confirmation = confirmation_number is None and not manual_only\\n\\\n blocker = \\\"The portal exposes an email/manual-service path instead of an online\\\n \\ confirmation.\\\" if manual_only else (\\\"The submitted page did not expose a\\\n \\ confirmation number.\\\" if missing_confirmation else None)\\nreturn {\\\"output\\\"\\\n : {\\\"confirmation_number\\\": confirmation_number, \\\"account_number\\\": account_number,\\\n \\ \\\"selected_start_date\\\": selected_start_date, \\\"deposit_amount\\\": deposit_amount,\\\n \\ \\\"next_owner\\\": next_owner, \\\"blocker\\\": blocker, \\\"evidence_text\\\": body_text[:6000]}}\\n\"\n parameter_keys:\n - business_name\n - contact_email\n - desired_start_date\n - service_address\n extraction_schema: '{\"type\":\"object\",\"properties\":{\"output\":{\"type\":\"object\",\"properties\":{\"confirmation_number\":{\"type\":[\"string\",\"null\"]},\"account_number\":{\"type\":[\"string\",\"null\"]},\"selected_start_date\":{\"type\":[\"string\",\"null\"]},\"deposit_amount\":{\"type\":[\"string\",\"null\"]},\"next_owner\":{\"type\":[\"string\",\"null\"]},\"blocker\":{\"type\":[\"string\",\"null\"]},\"evidence_text\":{\"type\":\"string\"}},\"required\":[\"confirmation_number\",\"account_number\",\"selected_start_date\",\"deposit_amount\",\"next_owner\",\"blocker\",\"evidence_text\"]}},\"required\":[\"output\"]}'\n prompt: Submit the observed commercial water service form and safely return confirmation\n details or an explicit blocker.\nis_saved_task: false\nstatus: published\nrun_sequentially: false\nsequential_key: null\n",+ "adaptive_caching": false,+ "agent_id": "wpid_552079557434126732",+ "ai_fallback": true,+ "browser_profile_id": null,+ "browser_profile_key": null,+ "cache_key": "default",+ "cdp_connect_headers": null,+ "code_version": null,+ "copilot_authored": false,+ "created_at": "2026-07-16T17:53:26.232218Z",+ "created_by": null,+ "deleted_at": null,+ "description": "Uses configurable business, contact, service address, and start date inputs.",+ "edited_by": null,+ "enable_self_healing": false,+ "extra_http_headers": {},+ "folder_id": null,+ "generate_script_on_terminal": false,+ "import_error": null,+ "is_saved_task": false,+ "is_template": false,+ "max_elapsed_time_minutes": null,+ "max_screenshot_scrolls": null,+ "model": null,+ "modified_at": "2026-07-16T17:53:26.232218Z",+ "organization_id": "o_551846329558231414",+ "persist_browser_session": false,+ "pin_saved_session_ip": false,+ "proxy_location": "NONE",+ "run_sequentially": false,+ "run_with": "agent",+ "sequential_key": null,+ "status": "published",+ "title": "Start Citrus County Commercial Water Service",+ "totp_identifier": null,+ "totp_verification_url": null,+ "version": 1,+ "webhook_callback_url": "",+ "workflow_definition": {+ "blocks": [+ {+ "block_type": "code",+ "code": "# Workflow input bindings: {{ parameters.business_name }}, {{ parameters.contact_email }}, {{ parameters.service_address }}, {{ parameters.desired_start_date }}\nawait page.goto(\"http://localhost:8900/utility_services/citrus_water/\")\nawait page.wait_for_load_state(\"domcontentloaded\")\nawait page.locator(\"button[data-action=\\\"citrusForm\\\"]\").click()\nawait page.wait_for_load_state(\"domcontentloaded\")\nawait page.locator(\"#company\").wait_for(state=\"visible\")\nawait page.locator(\"#company\").fill(str(business_name).strip())\nawait page.locator(\"#email\").fill(str(contact_email).strip())\nawait page.locator(\"#serviceAddress\").fill(str(service_address).strip())\nawait page.locator(\"#startDate\").fill(str(desired_start_date).strip())\nsubmit_button = page.get_by_role(\"button\", name=\"Submit Request\", exact=True)\nawait submit_button.wait_for(state=\"visible\")\nif not await submit_button.is_enabled():\n body_text = await page.locator(\"body\").inner_text()\n return {\"output\": {\"confirmation_number\": None, \"account_number\": None, \"selected_start_date\": None, \"deposit_amount\": None, \"next_owner\": None, \"blocker\": \"The online request form is not reviewable or submit-enabled.\", \"evidence_text\": body_text[:4000]}}\nawait submit_button.click()\nawait page.wait_for_timeout(700)\nbody_text = await page.locator(\"body\").inner_text()\nlines = [line.strip() for line in body_text.splitlines() if line.strip()]\ndef value_after(labels):\n for index, line in enumerate(lines):\n low = line.lower()\n for label in labels:\n if label in low:\n if \":\" in line:\n value = line.split(\":\", 1)[1].strip()\n if value:\n return value\n if index + 1 < len(lines):\n return lines[index + 1]\n return None\nconfirmation_number = value_after([\"confirmation number\", \"confirmation #\"])\naccount_number = value_after([\"account number\", \"account #\"])\nselected_start_date = value_after([\"selected start date\", \"requested start date\", \"start date\"])\ndeposit_amount = value_after([\"deposit amount\", \"deposit\"])\nnext_owner = value_after([\"next owner\", \"assigned to\", \"owner\"])\nlower_text = body_text.lower()\nmanual_only = (\"manual\" in lower_text or \"email us\" in lower_text) and confirmation_number is None\nmissing_confirmation = confirmation_number is None and not manual_only\nblocker = \"The portal exposes an email/manual-service path instead of an online confirmation.\" if manual_only else (\"The submitted page did not expose a confirmation number.\" if missing_confirmation else None)\nreturn {\"output\": {\"confirmation_number\": confirmation_number, \"account_number\": account_number, \"selected_start_date\": selected_start_date, \"deposit_amount\": deposit_amount, \"next_owner\": next_owner, \"blocker\": blocker, \"evidence_text\": body_text[:6000]}}\n",+ "continue_on_failure": false,+ "disable_cache": false,+ "ignore_workflow_system_prompt": false,+ "label": "submit_commercial_water_request",+ "model": null,+ "next_block_label": null,+ "next_loop_on_failure": false,+ "output_parameter": {+ "created_at": "2026-07-16T17:53:26.227963",+ "deleted_at": null,+ "description": "Output parameter for block submit_commercial_water_request",+ "key": "submit_commercial_water_request_output",+ "modified_at": "2026-07-16T17:53:26.227968",+ "output_parameter_id": "op_552080278988632478",+ "parameter_type": "output",+ "workflow_id": "w_552079557434126730"+ },+ "parameters": [+ {+ "created_at": "2026-07-16T17:53:26.227993Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Business or company name",+ "key": "business_name",+ "modified_at": "2026-07-16T17:53:26.227993Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632480",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228010Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Contact email for the request",+ "key": "contact_email",+ "modified_at": "2026-07-16T17:53:26.228010Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632482",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228016Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Requested service start date in YYYY-MM-DD format",+ "key": "desired_start_date",+ "modified_at": "2026-07-16T17:53:26.228016Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632486",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228014Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Full commercial service address",+ "key": "service_address",+ "modified_at": "2026-07-16T17:53:26.228014Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632484",+ "workflow_parameter_type": "string"+ }+ ],+ "prompt": "Submit the observed commercial water service form and safely return confirmation details or an explicit blocker.",+ "steps": [+ {+ "action_type": "goto_url",+ "description": "Open http://localhost:8900/utility_services/citrus_water/",+ "line_end": 2,+ "line_start": 2,+ "title": null+ },+ {+ "action_type": "click",+ "description": "Click the element",+ "line_end": 4,+ "line_start": 4,+ "title": null+ },+ {+ "action_type": "input_text",+ "description": "Type into the element",+ "line_end": 7,+ "line_start": 7,+ "title": null+ },+ {+ "action_type": "input_text",+ "description": "Type into the element",+ "line_end": 8,+ "line_start": 8,+ "title": null+ },+ {+ "action_type": "input_text",+ "description": "Type into the element",+ "line_end": 9,+ "line_start": 9,+ "title": null+ },+ {+ "action_type": "input_text",+ "description": "Type into the element",+ "line_end": 10,+ "line_start": 10,+ "title": null+ },+ {+ "action_type": "extract",+ "description": "Extract information from the page",+ "line_end": 14,+ "line_start": 14,+ "title": null+ },+ {+ "action_type": "click",+ "description": "Click the element",+ "line_end": 16,+ "line_start": 16,+ "title": null+ },+ {+ "action_type": "wait",+ "description": "Wait",+ "line_end": 17,+ "line_start": 17,+ "title": null+ },+ {+ "action_type": "extract",+ "description": "Extract information from the page",+ "line_end": 18,+ "line_start": 18,+ "title": null+ }+ ]+ }+ ],+ "error_code_mapping": null,+ "finally_block_label": null,+ "parameters": [+ {+ "created_at": "2026-07-16T17:53:26.227963",+ "deleted_at": null,+ "description": "Output parameter for block submit_commercial_water_request",+ "key": "submit_commercial_water_request_output",+ "modified_at": "2026-07-16T17:53:26.227968",+ "output_parameter_id": "op_552080278988632478",+ "parameter_type": "output",+ "workflow_id": "w_552079557434126730"+ },+ {+ "created_at": "2026-07-16T17:53:26.227993Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Business or company name",+ "key": "business_name",+ "modified_at": "2026-07-16T17:53:26.227993Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632480",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228010Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Contact email for the request",+ "key": "contact_email",+ "modified_at": "2026-07-16T17:53:26.228010Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632482",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228014Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Full commercial service address",+ "key": "service_address",+ "modified_at": "2026-07-16T17:53:26.228014Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632484",+ "workflow_parameter_type": "string"+ },+ {+ "created_at": "2026-07-16T17:53:26.228016Z",+ "default_value": null,+ "deleted_at": null,+ "description": "Requested service start date in YYYY-MM-DD format",+ "key": "desired_start_date",+ "modified_at": "2026-07-16T17:53:26.228016Z",+ "parameter_type": "workflow",+ "workflow_id": "w_552079557434126730",+ "workflow_parameter_id": "wp_552080278988632486",+ "workflow_parameter_type": "string"+ }+ ],+ "version": 2,+ "workflow_system_prompt": null+ },+ "workflow_id": "w_552079557434126730",+ "workflow_permanent_id": "wpid_552079557434126732"+}diff --git a/tests/unit/test_code_block_preflight.py b/tests/unit/test_code_block_preflight.pynew file mode 100644index 000000000..3a51cb22c--- /dev/null+++ b/tests/unit/test_code_block_preflight.py@@ -0,0 +1,104 @@+"""Tests for author-time render validation of copilot code blocks.++OSS-synced: only example.* placeholder targets and synthetic labels.+"""++from __future__ import annotations++import pytest++from skyvern.forge.sdk.copilot.code_block_preflight import (+ RENDER_TEMPLATE_SYNTAX_REASON_CODE,+ RENDER_UNDEFINED_NAME_REASON_CODE,+ CodeBlockRenderDiagnostic,+ code_block_render_diagnostic,+)++_BOUND_NAMES = frozenset({"business_name", "contact_email", "submit_request", "submit_request_output"})+++class TestCodeBlockRenderDiagnosticRejects:+ def test_parameters_namespace_reference_is_unrenderable(self) -> None:+ code = (+ "# Workflow input bindings: {{ parameters.business_name }}\n"+ 'await page.goto("https://example.com/request")\n'+ 'await page.locator("#company").fill(str(business_name).strip())\n'+ )+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)+ assert diagnostic is not None+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE+ assert diagnostic.failing_expression == "{{ parameters.business_name }}"+ assert "{{ business_name }}" in diagnostic.message++ def test_undeclared_root_is_unrenderable(self) -> None:+ diagnostic = code_block_render_diagnostic("value = str({{ frobnicator }})", _BOUND_NAMES)+ assert diagnostic is not None+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE+ assert diagnostic.failing_expression == "{{ frobnicator }}"+ assert "frobnicator" in diagnostic.message++ def test_template_syntax_error_is_unrenderable(self) -> None:+ diagnostic = code_block_render_diagnostic("value = {{ business_name\nother = 1", _BOUND_NAMES)+ assert diagnostic is not None+ assert diagnostic.code == RENDER_TEMPLATE_SYNTAX_REASON_CODE+ assert diagnostic.message++ def test_statement_only_undeclared_root_is_attributed(self) -> None:+ code = "{% if unknown_flag %}\nvalue = 1\n{% endif %}"+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)+ assert diagnostic is not None+ assert "unknown_flag" in diagnostic.failing_expression++ def test_unattributable_undefined_yields_diagnostic_not_exception(self) -> None:+ code = "{% macro helper() %}{{ caller() }}{% endmacro %}\nvalue = {{ helper() }}"+ diagnostic = code_block_render_diagnostic(code, _BOUND_NAMES)+ assert isinstance(diagnostic, CodeBlockRenderDiagnostic)+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE++ @pytest.mark.parametrize(+ "gadget",+ [+ "value = {{ ''.__class__.__mro__[1].__subclasses__() }}",+ "value = {{ business_name.__class__.__init__.__globals__ }}",+ ],+ )+ def test_ssti_gadget_is_rejected_without_executing(self, gadget: str) -> None:+ diagnostic = code_block_render_diagnostic(gadget, _BOUND_NAMES)+ assert diagnostic is not None+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE++ def test_loop_names_outside_loop_scope_are_unrenderable(self) -> None:+ diagnostic = code_block_render_diagnostic("value = {{ current_item }}", _BOUND_NAMES)+ assert diagnostic is not None+ assert diagnostic.code == RENDER_UNDEFINED_NAME_REASON_CODE+ assert "current_item" in diagnostic.message+++class TestCodeBlockRenderDiagnosticPasses:+ @pytest.mark.parametrize(+ "code",+ [+ 'await page.locator("#company").fill("{{ business_name }}")',+ "value = {{ submit_request_output.field }}",+ "value = {{ submit_request_output['nested'][0] }}",+ "today = {{ current_date }}",+ "payload = {{ business_name | json }}",+ "{% for item in workflow_run_outputs %}{{ item }}{% endfor %}",+ ],+ )+ def test_renderable_templates_pass(self, code: str) -> None:+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is None++ @pytest.mark.parametrize(+ "code",+ [+ "{% for item in workflow_run_outputs %}{{ current_index }}{% endfor %}",+ "{% for item in workflow_run_outputs %}\nvalue = {{ current_item }}\n{% endfor %}",+ ],+ )+ def test_loop_names_pass_inside_loop(self, code: str) -> None:+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is None++ def test_jinja_free_code_passes(self) -> None:+ code = 'await page.goto("https://example.com")\nreturn {"output": {"a": 1}}'+ assert code_block_render_diagnostic(code, _BOUND_NAMES) is Nonediff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex 5eddfe89e..290963080 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -10,6 +10,7 @@ import hashlibimport inspectimport jsonimport textwrap+from pathlib import Pathfrom types import EllipsisType, SimpleNamespacefrom unittest.mock import AsyncMock, Mock@@ -98,6 +99,7 @@ from skyvern.forge.sdk.copilot.tools import run_execution as run_execution_modulfrom skyvern.forge.sdk.copilot.tools import scouting as scouting_modulefrom skyvern.forge.sdk.copilot.tools import workflow_update as workflow_update_modulefrom skyvern.forge.sdk.copilot.tools.workflow_update import (+ _code_block_render_reject,_code_safety_reject_payload,_OutputContractEvaluation,_strip_redundant_sandbox_imports_in_yaml,@@ -647,6 +649,27 @@ def _credential_code_yaml(*, code: str, credential_id: str = "cred_missing") ->)+class TestCredentialRealValueBindings:+ def test_real_username_reference_is_not_falsely_rejected(self) -> None:+ yaml_text = _credential_code_yaml(+ code='await page.locator("#email").fill("{{ login_credential_real_username }}")',+ )+ assert _code_block_render_reject(yaml_text, None) is None++ def test_real_password_reference_is_not_falsely_rejected(self) -> None:+ yaml_text = _credential_code_yaml(+ code='await page.locator("input[type=\'password\']").fill("{{ login_credential_real_password }}")',+ )+ assert _code_block_render_reject(yaml_text, None) is None++ def test_unbound_credential_derived_name_still_rejected(self) -> None:+ yaml_text = _credential_code_yaml(+ code='await page.locator("#email").fill("{{ nonexistent_credential_real_username }}")',+ )+ result = _code_block_render_reject(yaml_text, None)+ assert result is not None++def _directory_blocks_yaml(blocks: str) -> str:indented_blocks = textwrap.indent(textwrap.dedent(blocks).strip(), " ")return f"title: Directory lookup\nworkflow_definition:\n blocks:\n{indented_blocks}\n"@@ -19034,3 +19057,180 @@ class TestFreehandSurfacePersistSeam:code = 'await page.get_by_label("State").select_option("CA")'result = workflow_update_module._persist_seam_freehand_surface_result(_freehand_block_yaml(code), ctx)assert result is None+++_RENDER_PREFLIGHT_FIXTURE_PATH = (+ Path(__file__).parent / "fixtures" / "copilot" / "wr_jinja_unrenderable_proposed_workflow.json"+)++_PARAMETERS_NAMESPACE_CODE_YAML = _yaml(+ """+ title: Utility service request+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}+ blocks:+ - block_type: code+ label: submit_request+ parameter_keys: [business_name]+ code: |+ # Workflow input bindings: {{ parameters.business_name }}+ await page.goto("https://example.com/request")+ await page.locator("#company").fill(str(business_name).strip())+ """+)++_UNDECLARED_ROOT_CODE_YAML = _yaml(+ """+ title: Utility service request+ workflow_definition:+ blocks:+ - block_type: code+ label: submit_request+ code: |+ await page.goto("https://example.com/request")+ await page.locator("#company").fill("{{ frobnicator }}")+ """+)++_TEMPLATE_SYNTAX_ERROR_CODE_YAML = _yaml(+ """+ title: Utility service request+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}+ blocks:+ - block_type: code+ label: submit_request+ parameter_keys: [business_name]+ code: |+ await page.goto("https://example.com/request")+ await page.locator("#company").fill("{{ business_name")+ """+)++_RENDERABLE_JINJA_CODE_YAML = _yaml(+ """+ title: Utility service request+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}+ blocks:+ - block_type: code+ label: submit_request+ parameter_keys: [business_name]+ code: |+ await page.goto("https://example.com/request")+ await page.locator("#company").fill("{{ business_name }}")+ """+)+++class TestCodeBlockRenderPreflightSeam:+ @pytest.mark.asyncio+ async def test_code_block_preflight_rejects_parameters_namespace_at_persist_seam(self) -> None:+ ctx = _code_only_ctx()+ with capture_logs() as logs:+ result = await _update_workflow({"workflow_yaml": _PARAMETERS_NAMESPACE_CODE_YAML}, ctx)+ assert result["ok"] is False+ assert result["data"]["reason_code"] == "code_block_unrenderable"+ assert result["data"]["block_label"] == "submit_request"+ assert result["data"]["failing_expression"] == "{{ parameters.business_name }}"+ assert "{{ business_name }}" in result["error"]+ outcome = ctx.latest_recorded_build_test_outcome+ assert isinstance(outcome, RecordedBuildTestOutcome)+ assert outcome.reason_code == "code_block_unrenderable"+ assert any(+ log.get("reason_code") == "code_block_unrenderable"+ and log.get("failing_expression") == "{{ parameters.business_name }}"+ for log in logs+ )++ @pytest.mark.asyncio+ async def test_code_block_preflight_rejects_undeclared_root_at_persist_seam(self) -> None:+ result = await _update_workflow({"workflow_yaml": _UNDECLARED_ROOT_CODE_YAML}, _code_only_ctx())+ assert result["ok"] is False+ assert result["data"]["reason_code"] == "code_block_unrenderable"+ assert result["data"]["failing_expression"] == "{{ frobnicator }}"++ @pytest.mark.asyncio+ async def test_code_block_preflight_rejects_template_syntax_error_at_persist_seam(self) -> None:+ result = await _update_workflow({"workflow_yaml": _TEMPLATE_SYNTAX_ERROR_CODE_YAML}, _code_only_ctx())+ assert result["ok"] is False+ assert result["data"]["reason_code"] == "code_block_unrenderable"+ assert "syntax error" in result["error"]++ @pytest.mark.asyncio+ async def test_code_block_preflight_renderable_draft_persists(self, monkeypatch: pytest.MonkeyPatch) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ result = await _update_workflow({"workflow_yaml": _RENDERABLE_JINJA_CODE_YAML}, ctx)+ assert result["ok"] is True+ assert "{{ business_name }}" in ctx.workflow_yaml++ @pytest.mark.asyncio+ async def test_code_block_preflight_renderable_draft_with_envelope_imposition_persists(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ ctx.request_policy = RequestPolicy(+ completion_criteria=[+ _typed_completion_criterion(+ id="requested_value",+ output_path="output.record_id",+ level="run",+ method_mandated=False,+ kind="outcome",+ )+ ]+ )+ workflow_yaml = _yaml(+ """+ title: Entry lookup+ workflow_definition:+ parameters:+ - {parameter_type: workflow, workflow_parameter_type: string, key: business_name, default_value: Sample Business}+ blocks:+ - block_type: code+ label: extract_entry_output+ parameter_keys: [business_name]+ code: |+ record_id = "{{ business_name }}"+ """+ )+ result = await _update_workflow({"workflow_yaml": workflow_yaml}, ctx, allow_missing_credentials=True)+ assert result["ok"] is True+ parsed = parse_workflow_yaml(ctx.workflow_yaml)+ assert isinstance(parsed, dict)+ code = str(_single_code_block(parsed)["code"])+ assert "{{ business_name }}" in code+ assert 'return {"output": {"record_id": record_id}}' in code++ @pytest.mark.asyncio+ async def test_code_block_preflight_unchanged_legacy_block_is_not_rerejected(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ _stub_successful_update(monkeypatch)+ ctx = _code_only_ctx()+ ctx.workflow_yaml = _PARAMETERS_NAMESPACE_CODE_YAML+ result = await _update_workflow({"workflow_yaml": _PARAMETERS_NAMESPACE_CODE_YAML}, ctx)+ assert (result.get("data") or {}).get("reason_code") != "code_block_unrenderable"+ assert result["ok"] is True++ @pytest.mark.asyncio+ async def test_code_block_preflight_rejects_recorded_unrenderable_draft_replay(self) -> None:+ artifact = json.loads(_RENDER_PREFLIGHT_FIXTURE_PATH.read_text())+ ctx = _code_only_ctx()+ with capture_logs() as logs:+ result = await _update_workflow({"workflow_yaml": artifact["_copilot_yaml"]}, ctx)+ assert result["ok"] is False+ assert result["data"]["reason_code"] == "code_block_unrenderable"+ assert result["data"]["block_label"] == "submit_commercial_water_request"+ assert "parameters.business_name" in result["data"]["failing_expression"]+ assert "{{ business_name }}" in result["error"]+ assert any(+ log.get("reason_code") == "code_block_unrenderable"+ and "parameters.business_name" in str(log.get("failing_expression"))+ for log in logs+ )diff --git a/tests/unit/test_utils_templating.py b/tests/unit/test_utils_templating.pyindex 33d81d81d..49e7de5a3 100644--- a/tests/unit/test_utils_templating.py+++ b/tests/unit/test_utils_templating.py@@ -1,6 +1,7 @@import reimport pytest+from jinja2.exceptions import SecurityErrorfrom skyvern.utils.templating import Constants, get_missing_variables@@ -46,3 +47,16 @@ def test_get_missing_variables(template, data, expected):def test_regex_missing_variable_pattern(template, expected):matches = set(re.findall(Constants.MissingVariablePattern, template))assert matches == expected+++ "gadget",+ [+ "{{ ''.__class__.__mro__[1].__subclasses__() }}",+ "{{ [].__class__.__base__ }}",+ "{% for cls in ().__class__.__bases__ %}{{ cls }}{% endfor %}",+ ],+)+def test_get_missing_variables_never_renders_gadgets_unsandboxed(gadget):+ with pytest.raises(SecurityError):+ get_missing_variables(gadget, {})