Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Stop banning evaluate in Copilot-authored code blocks; fail closed on unparseable drafts (SKY-13043) (#7754)
Andrew Neilson86f5b830e46c
diff --git a/skyvern/cli/mcp_tools/code_block.py b/skyvern/cli/mcp_tools/code_block.pyindex 355ee21b1..a295b3173 100644--- a/skyvern/cli/mcp_tools/code_block.py+++ b/skyvern/cli/mcp_tools/code_block.py@@ -61,7 +61,7 @@ async def skyvern_code_block_lint("""Lint a Workflow Copilot `code` block with the copilot's own deterministic gates.Runs CodeBlock.is_safe_code(), the security denylist- (page.request/context/evaluate/evaluate_handle), the sandbox unresolved-name analysis,+ (page.request/page.context), the sandbox unresolved-name analysis,static preflight, and author-time AST diagnostics. Returns a structured pass/fail result.No browser session or API call required."""diff --git a/skyvern/forge/sdk/copilot/code_block_security.py b/skyvern/forge/sdk/copilot/code_block_security.pyindex 3b68bd370..ffb1d1c8b 100644--- a/skyvern/forge/sdk/copilot/code_block_security.py+++ b/skyvern/forge/sdk/copilot/code_block_security.py@@ -1,3 +1,13 @@+"""Deterministic AST denylist over Copilot-synthesized code blocks, enforced at authoring and at runtime pre-dispatch.++`page.request` and `page.context` remain denied at both seams because they reach the live browser session's HTTP+client and its BrowserContext (cookies, `storage_state`, new pages, route interception), so prompt-injected+synthesized code could exfiltrate session credentials out of band. Retaining them does not mitigate exfiltration+through `page.evaluate`, which can read page-visible data and egress it directly from in-page JS; this module is a+guardrail on one code path, not a sandbox, and Copilot code still executes in-process with credentials resolved+before the run until out-of-process isolation lands.+"""+from __future__ import annotationsimport ast@@ -9,14 +19,10 @@ COPILOT_CODE_SECURITY_FAILURE_CATEGORY = "COPILOT_CODE_SECURITY"_AUTHOR_ATTR_REASONS = {"request": "AUTHOR_PAGE_REQUEST","context": "AUTHOR_PAGE_CONTEXT",- "evaluate": "AUTHOR_PAGE_EVALUATE",- "evaluate_handle": "AUTHOR_PAGE_EVALUATE",}_RUNTIME_ATTR_REASONS = {"request": "RUNTIME_PAGE_REQUEST","context": "RUNTIME_PAGE_CONTEXT",- "evaluate": "RUNTIME_PAGE_EVALUATE",- "evaluate_handle": "RUNTIME_PAGE_EVALUATE",}@@ -51,7 +57,7 @@ def author_time_code_security_errors(*, label: str, code: str) -> list[CodeBlocktry:tree = ast.parse(code)except SyntaxError:- return []+ return [_error(label, "AUTHOR_SYNTAX_ERROR")]return _security_errors_for_tree(label, tree, _AUTHOR_ATTR_REASONS)@@ -109,6 +115,8 @@ def _surface_for_reason(reason_code: str) -> str:def _message_for_reason(*, label: str, reason_code: str, surface: str) -> str:+ if reason_code == "AUTHOR_SYNTAX_ERROR":+ return f"Code block `{label}` failed the Copilot code security check: the code does not parse as Python."if reason_code.startswith("AUTHOR_"):return f"Code block `{label}` failed the Copilot code security check: {surface} is not allowed."return f"Code block `{label}` was blocked before browser dispatch: {surface} is not allowed at Copilot runtime."diff --git a/skyvern/forge/sdk/copilot/tools/workflow_update.py b/skyvern/forge/sdk/copilot/tools/workflow_update.pyindex fa798b435..428c492dd 100644--- a/skyvern/forge/sdk/copilot/tools/workflow_update.py+++ b/skyvern/forge/sdk/copilot/tools/workflow_update.py@@ -1095,6 +1095,7 @@ def _code_block_safety_errors(workflow_yaml: str | None, prior_yaml: str | None)CodeBlock.is_safe_code(code)except SyntaxError as exc:errors.append(f"Code block `{label}` is not valid Python: {exc}")+ continueexcept InsecureCodeDetected as exc:errors.append(f"Code block `{label}` failed the sandbox safety check: {exc}. Rewrite without import "diff --git a/tests/unit/test_code_block_lint_mcp_tool.py b/tests/unit/test_code_block_lint_mcp_tool.pyindex bed44ea30..63fb4f2a6 100644--- a/tests/unit/test_code_block_lint_mcp_tool.py+++ b/tests/unit/test_code_block_lint_mcp_tool.py@@ -64,12 +64,10 @@ async def test_import_is_blocked_by_code_safety_gate() -> None:@pytest.mark.asyncio-async def test_page_evaluate_is_blocked_by_security_denylist() -> None:+async def test_page_evaluate_is_not_blocked_by_security_denylist() -> None:result = await skyvern_code_block_lint(code='await page.evaluate("1+1")')- assert result["ok"] is False- assert result["data"]["lint_ok"] is False- assert _has_security_error(result, reason_code="AUTHOR_PAGE_EVALUATE", surface="page.evaluate")+ assert result["data"]["security_errors"] == []@pytest.mark.asynciodiff --git a/tests/unit/test_copilot_code_block_persist_seam.py b/tests/unit/test_copilot_code_block_persist_seam.pyindex c7077fa96..6eeb6fa30 100644--- a/tests/unit/test_copilot_code_block_persist_seam.py+++ b/tests/unit/test_copilot_code_block_persist_seam.py@@ -837,21 +837,27 @@ class TestCodeSafetySeam:def test_safe_code_passes(self) -> None:assert _code_block_safety_errors(_SAFE_CODE_YAML, None) == []+ def test_unparseable_code_block_reports_one_syntax_error(self) -> None:+ errors = _code_block_safety_errors(_code_yaml("text = await page.evaluate("), None)++ assert len(errors) == 1+ assert "is not valid Python" in errors[0]+@pytest.mark.asyncioasync def test_denied_page_api_attribute_is_repairable_preflight_error_without_duplicate_generic_message(self,) -> None:result = await _update_workflow(- {"workflow_yaml": _code_yaml("text = await page.evaluate('() => document.body.innerText')")},+ {"workflow_yaml": _code_yaml("state = await page.context.storage_state()")},_standard_ctx(),)assert result["ok"] is Falsejoined = result["error"]- assert "AUTHOR_PAGE_EVALUATE" in joined+ assert "AUTHOR_PAGE_CONTEXT" in joinedassert "failed the generated-code preflight check" in joinedassert joined.count("failed the sandbox safety check") == 0- assert joined.count("page.evaluate is not allowed") == 1+ assert joined.count("page.context is not allowed") == 1def test_unresolved_sandbox_names_are_seam_errors(self) -> None:errors = _code_block_safety_errors(@@ -1208,7 +1214,7 @@ class TestCodeSafetySeam:"workflow_yaml",[_code_yaml("import os\nprint(provider_query)"),- _code_yaml("await page.evaluate(provider_query)"),+ _code_yaml("await page.request.get(provider_query)"),],)@pytest.mark.asyncio@@ -3677,7 +3683,7 @@ class TestCodeRepairProgressClassification:"_code_block_safety_errors",lambda workflow_yaml, prior_yaml: ["Code block `search_registry` failed the generated-code preflight check: "- "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."],)ctx = _code_only_ctx()@@ -3715,7 +3721,7 @@ class TestCodeRepairProgressClassification:payload = _code_safety_reject_payload(["Code block `search_registry` failed the generated-code preflight check: "- "AUTHOR_PAGE_EVALUATE: page.evaluate is not allowed."+ "AUTHOR_PAGE_REQUEST: page.request is not allowed."])diff --git a/tests/unit/test_copilot_code_block_synthesis.py b/tests/unit/test_copilot_code_block_synthesis.pyindex c9e26c462..b3d4b917f 100644--- a/tests/unit/test_copilot_code_block_synthesis.py+++ b/tests/unit/test_copilot_code_block_synthesis.py@@ -2160,8 +2160,6 @@ class TestPreflightSurfacesSyntaxError:[("await page.request.post('https://example.com/collect')", "AUTHOR_PAGE_REQUEST"),("state = await page.context.storage_state()", "AUTHOR_PAGE_CONTEXT"),- ("text = await page.evaluate('() => document.body.innerText')", "AUTHOR_PAGE_EVALUATE"),- ("handle = await page.evaluate_handle('() => document.body')", "AUTHOR_PAGE_EVALUATE"),],)def test_denied_page_api_attributes_surface_preflight_reason_codes(self, code: str, reason: str) -> None:@@ -2170,12 +2168,31 @@ class TestPreflightSurfacesSyntaxError:assert [diagnostic.code for diagnostic in diagnostics if diagnostic.code.startswith("AUTHOR_PAGE_")] == [reason]assert any("not allowed in persisted workflow code blocks" in diagnostic.message for diagnostic in diagnostics)+ @pytest.mark.parametrize(+ "code",+ [+ "text = await page.evaluate('() => document.body.innerText')",+ "handle = await page.evaluate_handle('() => document.body')",+ ],+ )+ def test_in_page_javascript_is_not_an_author_time_security_error(self, code: str) -> None:+ assert author_time_code_security_errors(label="search_registry", code=code) == []+ assert [+ diagnostic.code+ for diagnostic in preflight_code_block(code, parameter_keys=())+ if diagnostic.code.startswith("AUTHOR_PAGE_")+ ] == []++ def test_unparseable_code_is_not_silently_approved_at_author_time(self) -> None:+ errors = author_time_code_security_errors(label="search_registry", code="text = await page.evaluate(")++ assert [error.reason_code for error in errors] == ["AUTHOR_SYNTAX_ERROR"]+ assert "does not parse as Python" in str(errors[0])+def test_denied_page_api_preflight_reason_codes_match_author_time_security_source(self) -> None:code = """await page.request.post("https://example.com/collect")state = await page.context.storage_state()- text = await page.evaluate("() => document.body.innerText")- handle = await page.evaluate_handle("() => document.body")"""normalized_code = textwrap.dedent(code).strip()@@ -2192,7 +2209,6 @@ class TestPreflightSurfacesSyntaxError:== {"AUTHOR_PAGE_REQUEST","AUTHOR_PAGE_CONTEXT",- "AUTHOR_PAGE_EVALUATE",})