Skip to content

Automate browser based workflows with AI

Stop banning evaluate in Copilot-authored code blocks; fail closed on unparseable drafts (SKY-13043) (#7754)

Andrew Neilson86f5b830e46c
diff --git a/skyvern/cli/mcp_tools/code_block.py b/skyvern/cli/mcp_tools/code_block.py
index 355ee21b1..a295b3173 100644
--- a/skyvern/cli/mcp_tools/code_block.py
+++ b/skyvern/cli/mcp_tools/code_block.py
@@ -61,7 +61,7 @@ async def skyvern_code_block_lint(
"""Lint a Workflow Copilot `code` block with the copilot's own deterministic gates.
Runs CodeBlock.is_safe_code(), the security denylist
- (page.request/context/evaluate/evaluate_handle), the sandbox unresolved-name analysis,
+ (page.request/page.context), the sandbox unresolved-name analysis,
static preflight, and author-time AST diagnostics. Returns a structured pass/fail result.
No browser session or API call required.
"""
diff --git a/skyvern/forge/sdk/copilot/code_block_security.py b/skyvern/forge/sdk/copilot/code_block_security.py
index 3b68bd370..ffb1d1c8b 100644
--- a/skyvern/forge/sdk/copilot/code_block_security.py
+++ b/skyvern/forge/sdk/copilot/code_block_security.py
@@ -1,3 +1,13 @@
+"""Deterministic AST denylist over Copilot-synthesized code blocks, enforced at authoring and at runtime pre-dispatch.
+
+`page.request` and `page.context` remain denied at both seams because they reach the live browser session's HTTP
+client and its BrowserContext (cookies, `storage_state`, new pages, route interception), so prompt-injected
+synthesized code could exfiltrate session credentials out of band. Retaining them does not mitigate exfiltration
+through `page.evaluate`, which can read page-visible data and egress it directly from in-page JS; this module is a
+guardrail on one code path, not a sandbox, and Copilot code still executes in-process with credentials resolved
+before the run until out-of-process isolation lands.
+"""
+
from __future__ import annotations
import ast
@@ -9,14 +19,10 @@ COPILOT_CODE_SECURITY_FAILURE_CATEGORY = "COPILOT_CODE_SECURITY"
_AUTHOR_ATTR_REASONS = {
"request": "AUTHOR_PAGE_REQUEST",
"context": "AUTHOR_PAGE_CONTEXT",
- "evaluate": "AUTHOR_PAGE_EVALUATE",
- "evaluate_handle": "AUTHOR_PAGE_EVALUATE",
}
_RUNTIME_ATTR_REASONS = {
"request": "RUNTIME_PAGE_REQUEST",
"context": "RUNTIME_PAGE_CONTEXT",
- "evaluate": "RUNTIME_PAGE_EVALUATE",
- "evaluate_handle": "RUNTIME_PAGE_EVALUATE",
}
@@ -51,7 +57,7 @@ def author_time_code_security_errors(*, label: str, code: str) -> list[CodeBlock
try:
tree = ast.parse(code)
except SyntaxError:
- return []
+ return [_error(label, "AUTHOR_SYNTAX_ERROR")]
return _security_errors_for_tree(label, tree, _AUTHOR_ATTR_REASONS)
@@ -109,6 +115,8 @@ def _surface_for_reason(reason_code: str) -> str:
def _message_for_reason(*, label: str, reason_code: str, surface: str) -> str:
+ if reason_code == "AUTHOR_SYNTAX_ERROR":
+ return f"Code block `{label}` failed the Copilot code security check: the code does not parse as Python."
if reason_code.startswith("AUTHOR_"):
return f"Code block `{label}` failed the Copilot code security check: {surface} is not allowed."
return f"Code block `{label}` was blocked before browser dispatch: {surface} is not allowed at Copilot runtime."
diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.py
index fa798b435..428c492dd 100644
--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py
+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py
@@ -1095,6 +1095,7 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)
CodeBlock.is_safe_code(code)
except SyntaxError as exc:
errors.append(f"Code block `{label}` is not valid Python: {exc}")
+ continue
except InsecureCodeDetected as exc:
errors.append(
f"Code block `{label}` failed the sandbox safety check: {exc}. Rewrite without import "
diff --git a/tests/unit/test_code_block_lint_mcp_tool.py b/tests/unit/test_code_block_lint_mcp_tool.py
index bed44ea30..63fb4f2a6 100644
--- a/tests/unit/test_code_block_lint_mcp_tool.py
+++ b/tests/unit/test_code_block_lint_mcp_tool.py
@@ -64,12 +64,10 @@ async def test_import_is_blocked_by_code_safety_gate() -> None:
@pytest.mark.asyncio
-async def test_page_evaluate_is_blocked_by_security_denylist() -> None:
+async def test_page_evaluate_is_not_blocked_by_security_denylist() -> None:
result = await skyvern_code_block_lint(code='await page.evaluate("1+1")')
- assert result["ok"] is False
- assert result["data"]["lint_ok"] is False
- assert _has_security_error(result, reason_code="AUTHOR_PAGE_EVALUATE", surface="page.evaluate")
+ assert result["data"]["security_errors"] == []
@pytest.mark.asyncio
diff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.py
index c7077fa96..6eeb6fa30 100644
--- a/tests/unit/test_copilot_code_block_persist_seam.py
+++ b/tests/unit/test_copilot_code_block_persist_seam.py
@@ -837,21 +837,27 @@ class TestCodeSafetySeam:
def test_safe_code_passes(self) -> None:
assert _code_block_safety_errors(_SAFE_CODE_YAML, None) == []
+ def test_unparseable_code_block_reports_one_syntax_error(self) -> None:
+ errors = _code_block_safety_errors(_code_yaml("text = await page.evaluate("), None)
+
+ assert len(errors) == 1
+ assert "is not valid Python" in errors[0]
+
@pytest.mark.asyncio
async def test_denied_page_api_attribute_is_repairable_preflight_error_without_duplicate_generic_message(
self,
) -> None:
result = await _update_workflow(
- {"workflow_yaml": _code_yaml("text = await page.evaluate('() => document.body.innerText')")},
+ {"workflow_yaml": _code_yaml("state = await page.context.storage_state()")},
_standard_ctx(),
)
assert result["ok"] is False
joined = result["error"]
- assert "AUTHOR_PAGE_EVALUATE" in joined
+ assert "AUTHOR_PAGE_CONTEXT" in joined
assert "failed the generated-code preflight check" in joined
assert joined.count("failed the sandbox safety check") == 0
- assert joined.count("page.evaluate is not allowed") == 1
+ assert joined.count("page.context is not allowed") == 1
def test_unresolved_sandbox_names_are_seam_errors(self) -> None:
errors = _code_block_safety_errors(
@@ -1208,7 +1214,7 @@ class TestCodeSafetySeam:
"workflow_yaml",
[
_code_yaml("import os\nprint(provider_query)"),
- _code_yaml("await page.evaluate(provider_query)"),
+ _code_yaml("await page.request.get(provider_query)"),
],
)
@pytest.mark.asyncio
@@ -3677,7 +3683,7 @@ class TestCodeRepairProgressClassification:
"_code_block_safety_errors",
lambda workflow_yaml, prior_yaml: [
"Code block `search_registry` failed the generated-code preflight check: "
- "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."
+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."
],
)
ctx = _code_only_ctx()
@@ -3715,7 +3721,7 @@ class TestCodeRepairProgressClassification:
payload = _code_safety_reject_payload(
[
"Code block `search_registry` failed the generated-code preflight check: "
- "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."
+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."
]
)
diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.py
index c9e26c462..b3d4b917f 100644
--- a/tests/unit/test_copilot_code_block_synthesis.py
+++ b/tests/unit/test_copilot_code_block_synthesis.py
@@ -2160,8 +2160,6 @@ class TestPreflightSurfacesSyntaxError:
[
("await page.request.post('https://example.com/collect')", "AUTHOR_PAGE_REQUEST"),
("state = await page.context.storage_state()", "AUTHOR_PAGE_CONTEXT"),
- ("text = await page.evaluate('() => document.body.innerText')", "AUTHOR_PAGE_EVALUATE"),
- ("handle = await page.evaluate_handle('() => document.body')", "AUTHOR_PAGE_EVALUATE"),
],
)
def test_denied_page_api_attributes_surface_preflight_reason_codes(self, code: str, reason: str) -> None:
@@ -2170,12 +2168,31 @@ class TestPreflightSurfacesSyntaxError:
assert [diagnostic.code for diagnostic in diagnostics if diagnostic.code.startswith("AUTHOR_PAGE_")] == [reason]
assert any("not allowed in persisted workflow code blocks" in diagnostic.message for diagnostic in diagnostics)
+ @pytest.mark.parametrize(
+ "code",
+ [
+ "text = await page.evaluate('() => document.body.innerText')",
+ "handle = await page.evaluate_handle('() => document.body')",
+ ],
+ )
+ def test_in_page_javascript_is_not_an_author_time_security_error(self, code: str) -> None:
+ assert author_time_code_security_errors(label="search_registry", code=code) == []
+ assert [
+ diagnostic.code
+ for diagnostic in preflight_code_block(code, parameter_keys=())
+ if diagnostic.code.startswith("AUTHOR_PAGE_")
+ ] == []
+
+ def test_unparseable_code_is_not_silently_approved_at_author_time(self) -> None:
+ errors = author_time_code_security_errors(label="search_registry", code="text = await page.evaluate(")
+
+ assert [error.reason_code for error in errors] == ["AUTHOR_SYNTAX_ERROR"]
+ assert "does not parse as Python" in str(errors[0])
+
def test_denied_page_api_preflight_reason_codes_match_author_time_security_source(self) -> None:
code = """
await page.request.post("https://example.com/collect")
state = await page.context.storage_state()
- text = await page.evaluate("() => document.body.innerText")
- handle = await page.evaluate_handle("() => document.body")
"""
normalized_code = textwrap.dedent(code).strip()
@@ -2192,7 +2209,6 @@ class TestPreflightSurfacesSyntaxError:
== {
"AUTHOR_PAGE_REQUEST",
"AUTHOR_PAGE_CONTEXT",
- "AUTHOR_PAGE_EVALUATE",
}
)