A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: edge cases for parsing pasted notebooks (#10033)
Akshay Agrawalbcb063b0910e
diff --git a/frontend/src/components/editor/cell/code/cell-editor.tsx b/frontend/src/components/editor/cell/code/cell-editor.tsxindex 419ee9115e..ac1206bb16 100644--- a/frontend/src/components/editor/cell/code/cell-editor.tsx+++ b/frontend/src/components/editor/cell/code/cell-editor.tsx@@ -10,9 +10,11 @@ import { Button } from "@/components/ui/button";import { DelayMount } from "@/components/utils/delay-mount";import { aiCompletionCellAtom } from "@/core/ai/state";import { maybeAddMarimoImport } from "@/core/cells/add-missing-import";-import { useCellActions } from "@/core/cells/cells";+import { getNotebook, useCellActions } from "@/core/cells/cells";+import { SETUP_CELL_ID } from "@/core/cells/ids";import { usePendingDeleteService } from "@/core/cells/pending-delete-service";import type { CellData, CellRuntimeState } from "@/core/cells/types";+import { notebookCellEditorViews } from "@/core/cells/utils";import { setupCodeMirror } from "@/core/codemirror/cm";import { acceptCompletionOnEnterAtom } from "@/core/codemirror/completion/accept-on-enter-atom";import { editorMountScheduler } from "@/core/codemirror/editor-mount-scheduler";@@ -22,6 +24,10 @@ import {reconfigureLanguageEffect,switchLanguage,} from "@/core/codemirror/language/extension";+import {+ getEditorCodeAsPython,+ updateEditorCodeFromPython,+} from "@/core/codemirror/language/utils";import { MARKDOWN_INITIAL_HIDE_CODE } from "@/core/codemirror/language/languages/markdown";import type { LanguageAdapterType } from "@/core/codemirror/language/types";import {@@ -214,6 +220,32 @@ const CellEditorInternal = ({});}},+ addOrAppendSetupCell: (code) => {+ const notebook = getNotebook();+ // No setup cell yet: create one with this code at the top.+ if (!notebook.cellIds.setupCellExists()) {+ cellActions.addSetupCellIfDoesntExist({ code });+ return;+ }+ // Setup cell exists: append the pasted code to it. Prefer updating+ // the mounted editor (keeps the view and state in sync, like+ // formatting does); fall back to a plain state update otherwise.+ const view = notebookCellEditorViews(notebook)[SETUP_CELL_ID];+ const existing = view+ ? getEditorCodeAsPython(view)+ : notebook.cellData[SETUP_CELL_ID].code;+ const merged = existing.trim()+ ? `${existing.trimEnd()}\n\n${code}`+ : code;+ if (view) {+ updateEditorCodeFromPython(view, merged);+ }+ cellActions.updateCellCode({+ cellId: SETUP_CELL_ID,+ code: merged,+ formattingChange: false,+ });+ },splitCell,toggleHideCode,aiCellCompletion: () => {diff --git a/frontend/src/core/codemirror/cells/state.ts b/frontend/src/core/codemirror/cells/state.tsindex 9e9f332de9..dd9721c40f 100644--- a/frontend/src/core/codemirror/cells/state.ts+++ b/frontend/src/core/codemirror/cells/state.ts@@ -8,6 +8,11 @@ export interface CodemirrorCellActions extends CellActions {toggleHideCode: () => boolean;aiCellCompletion: () => boolean;createManyBelow: (content: string[]) => void;+ /**+ * Create the setup cell with the given code, or append the code to the+ * existing setup cell if one already exists.+ */+ addOrAppendSetupCell: (code: string) => void;onRun: () => void;deleteCell: () => void;afterToggleMarkdown: () => void;diff --git a/frontend/src/core/codemirror/misc/__tests__/paste.test.ts b/frontend/src/core/codemirror/misc/__tests__/paste.test.tsindex f98656e129..34f3f279f0 100644--- a/frontend/src/core/codemirror/misc/__tests__/paste.test.ts+++ b/frontend/src/core/codemirror/misc/__tests__/paste.test.ts@@ -7,7 +7,7 @@ import {type CodemirrorCellActions,cellActionsState,} from "../../cells/state";-import { extractCells, pasteBundle } from "../paste";+import { extractCells, extractMarimoApp, pasteBundle } from "../paste";describe("extractCells", () => {it("returns empty array for non-marimo text", () => {@@ -158,6 +158,154 @@ def _(mo, px):]);});+ it("preserves decorators on nested functions", () => {+ const input = `+def _():+ @functools.cache+ def fib(n):+ return n+ return fib+`;+ expect(extractCells(input)).toEqual([+ "@functools.cache\ndef fib(n):\n return n",+ ]);+ });++ it("preserves decorated methods inside a class", () => {+ const input = `+def _():+ class A:+ @property+ def x(self):+ return self._x+ return A+`;+ expect(extractCells(input)).toEqual([+ "class A:\n @property\n def x(self):\n return self._x",+ ]);+ });++ it("handles async cells with multi-line args", () => {+ const input = `+async def _(+ a,+ b,+):+ x = await foo(a, b)+ return x+`;+ expect(extractCells(input)).toEqual(["x = await foo(a, b)"]);+ });++ it("strips multi-line returns using brackets", () => {+ const input = `+def _():+ a = 1+ b = 2+ return [+ a,+ b,+ ]+`;+ expect(extractCells(input)).toEqual(["a = 1\nb = 2"]);+ });++ it("strips multi-line returns using braces", () => {+ const input = `+def _():+ a = 1+ return {+ "a": a,+ }+`;+ expect(extractCells(input)).toEqual(["a = 1"]);+ });++ it("does not corrupt the following cell after a bracketed return", () => {+ const input = `+def _():+ a = 1+ return [+ a,+ ]++def _():+ b = 2+ return b+`;+ expect(extractCells(input)).toEqual(["a = 1", "b = 2"]);+ });++ it("preserves comments in the cell body", () => {+ const input = `+def _():+ # leading comment+ x = 1+ return x+`;+ expect(extractCells(input)).toEqual(["# leading comment\nx = 1"]);+ });++ it("extracts the setup block separately from cells", () => {+ const input = `+import marimo+app = marimo.App()++with app.setup(hide_code=True):+ import marimo as mo+ import numpy as np+ from numpy.linalg import eigh+`;+ expect(extractMarimoApp(input)).toEqual({+ setup:+ "import marimo as mo\nimport numpy as np\nfrom numpy.linalg import eigh",+ cells: [],+ });+ // The convenience wrapper excludes the setup block.+ expect(extractCells(input)).toEqual([]);+ });++ it("extracts @app.function definitions, keeping their return", () => {+ const input = `+def laplacian_matrix(adjacency_matrix):+ degree_matrix = np.diag(np.sum(adjacency_matrix, axis=1))+ L = degree_matrix - adjacency_matrix+ return L+`;+ expect(extractCells(input)).toEqual([+ "def laplacian_matrix(adjacency_matrix):\n degree_matrix = np.diag(np.sum(adjacency_matrix, axis=1))\n L = degree_matrix - adjacency_matrix\n return L",+ ]);+ });++ it("extracts setup separately while keeping cells and functions in order", () => {+ const input = `+with app.setup:+ import numpy as np++def _():+ a = 1+ return (a,)++def double(x):+ return x * 2+`;+ expect(extractMarimoApp(input)).toEqual({+ setup: "import numpy as np",+ cells: ["a = 1", "def double(x):\n return x * 2"],+ });+ });+it("handles cells with config", () => {const input = `@app.cell(hide_code=True, column=2)@@ -254,4 +402,45 @@ def _():expect(createManyBelow).not.toHaveBeenCalled();});++ it("routes the setup block to the setup cell, not a normal cell", () => {+ const createManyBelow = vi.fn();+ const addOrAppendSetupCell = vi.fn();+ const extension = pasteBundle();+ const view = new EditorView({+ state: EditorState.create({+ doc: "",+ extensions: [+ extension,+ cellActionsState.of({+ createManyBelow,+ addOrAppendSetupCell,+ } as never as CodemirrorCellActions),+ ],+ }),+ });++ const clipboardData = new MockDataTransfer();+ clipboardData.setData(+ "text/plain",+ `+with app.setup:+ import numpy as np++def _():+ x = 1+ return x+`,+ );+ const event = new MockClipboardEvent("paste", { clipboardData });++ // HACK: manually add the paste event listener+ // @ts-expect-error extension not typed+ view.dom.onpaste = (evt) => extension[0].domEventHandlers.paste(evt, view);+ view.dom.dispatchEvent(event);++ expect(addOrAppendSetupCell).toHaveBeenCalledWith("import numpy as np");+ expect(createManyBelow).toHaveBeenCalledWith(["x = 1"]);+ });});diff --git a/frontend/src/core/codemirror/misc/paste.ts b/frontend/src/core/codemirror/misc/paste.tsindex 82428c65c0..923415f515 100644--- a/frontend/src/core/codemirror/misc/paste.ts+++ b/frontend/src/core/codemirror/misc/paste.ts@@ -1,6 +1,8 @@/* Copyright 2026 Marimo. All rights reserved. */import type { Extension } from "@codemirror/state";import { EditorView } from "@codemirror/view";+import type { SyntaxNode } from "@lezer/common";+import { parser } from "@lezer/python";import { cellActionsState } from "../cells/state";export function pasteBundle(): Extension[] {@@ -8,152 +10,166 @@ export function pasteBundle(): Extension[] {EditorView.domEventHandlers({paste: (event: ClipboardEvent, view: EditorView) => {const text = event.clipboardData?.getData("text/plain");- if (!text?.includes("@app.cell")) {+ if (!text || !looksLikeMarimoApp(text)) {return false;}- const cells = extractCells(text);- if (cells.length === 0) {+ const { setup, cells } = extractMarimoApp(text);+ if (setup === null && cells.length === 0) {return false;}const actions = view.state.facet(cellActionsState);- actions.createManyBelow(cells);+ // The `with app.setup` block belongs in the special setup cell, not in+ // an ordinary cell.+ if (setup !== null) {+ actions.addOrAppendSetupCell(setup);+ }+ if (cells.length > 0) {+ actions.createManyBelow(cells);+ }return true;},}),];}+/** Whether the pasted text looks like marimo app source worth parsing. */+function looksLikeMarimoApp(text: string): boolean {+ return /@app\.cell|@app\.function|\bapp\.setup\b/.test(text);+}++export interface ExtractedApp {+ /** The `with app.setup` block's body, if present. Belongs in the setup cell. */+ setup: string | null;+ /** Ordinary cells, in source order. */+ cells: string[];+}+/**- * Extract the cells from a marimo app.+ * Extract the cells of a marimo app from its source.+ *+ * Parses the pasted source with the Python grammar (the same one that powers+ * the editor) and turns each marimo construct into a cell:+ * - `with app.setup(...)` contributes the setup block's body, returned+ * separately as `setup` because it belongs in the dedicated setup cell.+ * - `@app.cell` functions contribute their body, minus the trailing+ * auto-generated `return`.+ * - `@app.function` definitions contribute the whole function (the decorator+ * is dropped, but the function — including its `return` — is kept).+ *+ * Using the real syntax tree means decorators, `async def`, multi-line+ * signatures, nested functions, and multi-line returns of any bracket type are+ * all handled structurally rather than by line-based heuristics.*/-export function extractCells(text: string): string[] {- // Quick check if this looks like a marimo app- if (!text.includes("@app.cell")) {- return [];+export function extractMarimoApp(text: string): ExtractedApp {+ if (!looksLikeMarimoApp(text)) {+ return { setup: null, cells: [] };}const cells: string[] = [];- const lines = text.split("\n");- let currentCell: string[] = [];- let inCell = false;- let skipLines = 0;- let inMultilineArgs = false;- let inMultilineReturn = false;- let parenCount = 0;- let cellBaseIndent: number | null = null;-- // Pre-compile regex patterns- const leadingParenRegex = /\(/g;- const trailingParenRegex = /\)/g;- const cellEndMarkers = new Set(["@"]);-- function countParens(line: string): number {- return (- (line.match(leadingParenRegex) || []).length -- (line.match(trailingParenRegex) || []).length- );- }-- function getIndent(line: string): number {- const match = line.match(/^\s*/);- return match ? match[0].length : 0;- }-- function finalizeCellIfNeeded() {- if (currentCell.length === 0) {- return;+ const setupBlocks: string[] = [];+ const tree = parser.parse(text);++ // A node's `from` starts after its line's leading indentation; extend back to+ // the start of the line so `dedent` sees consistent indentation, then dedent.+ const blockText = (from: number, to: number): string => {+ const lineStart = text.lastIndexOf("\n", from - 1) + 1;+ return dedent(text.slice(lineStart, to));+ };++ const pushCell = (from: number, to: number) => {+ const cell = blockText(from, to);+ if (cell.trim()) {+ cells.push(cell);}-- // Only add non-empty cells- if (currentCell.some((l) => l.trim() !== "")) {- cells.push(dedent(currentCell.join("\n")));+ };++ // The text of a block's body (the statements after its `:`), optionally+ // dropping a trailing auto-generated `return`. Returns null if empty.+ const bodyText = (+ body: SyntaxNode,+ stripTrailingReturn: boolean,+ ): string | null => {+ const children: SyntaxNode[] = [];+ for (let child = body.firstChild; child; child = child.nextSibling) {+ if (child.name !== ":") {+ children.push(child);+ }}- currentCell = [];- }-- for (const line of lines) {- const trimmed = line.trim();-- // Skip empty lines between cells- if (!trimmed && !inCell) {- continue;+ if (stripTrailingReturn && children.at(-1)?.name === "ReturnStatement") {+ children.pop();}-- // Start of a new cell- if (trimmed.startsWith("@app.cell")) {- finalizeCellIfNeeded();- inCell = true;- skipLines = 1; // Skip the def line- cellBaseIndent = null;- continue;+ const first = children[0];+ const last = children.at(-1);+ if (!first || !last) {+ return null;}-- // Handle function definition and args- if (skipLines > 0) {- if (- trimmed.startsWith("def") &&- trimmed.includes("(") &&- !trimmed.includes("):")- ) {- inMultilineArgs = true;- parenCount = countParens(trimmed);+ const cell = blockText(first.from, last.to);+ return cell.trim() ? cell : null;+ };++ tree.iterate({+ enter: (node) => {+ // Setup cell: `with app.setup:` / `with app.setup(...):`+ if (node.name === "WithStatement") {+ const body = node.node.getChild("Body");+ if (body && text.slice(node.from, body.from).includes("app.setup")) {+ const setup = bodyText(body, false);+ if (setup !== null) {+ setupBlocks.push(setup);+ }+ }+ return false;}- skipLines--;- continue;- }- // Track multi-line arguments- if (inMultilineArgs) {- parenCount += countParens(trimmed);- if (parenCount === 0) {- inMultilineArgs = false;+ if (node.name !== "DecoratedStatement") {+ // Keep descending until we reach the decorated definitions.+ return true;}- continue;- }-- if (!inCell) {- continue;- }-- // Handle cell content- if (cellEndMarkers.has(trimmed[0]) || trimmed.startsWith("if __name__")) {- finalizeCellIfNeeded();- inCell = trimmed.startsWith("@");- continue;- }-- // Detect base indentation of cell body from first content line- if (cellBaseIndent === null && trimmed) {- cellBaseIndent = getIndent(line);- }- // Handle return statements — only strip cell-level returns- if (trimmed.startsWith("return") && getIndent(line) === cellBaseIndent) {- if (trimmed.includes("(") && !trimmed.endsWith(")")) {- inMultilineReturn = true;- parenCount = countParens(trimmed);+ const decorator = node.node.getChild("Decorator");+ const fn = node.node.getChild("FunctionDefinition");+ if (!decorator || !fn) {+ return false;+ }+ const decoratorText = text.slice(decorator.from, decorator.to).trim();++ // `@app.cell` / `@app.cell(...)`: the body becomes a cell, minus the+ // trailing auto-generated return.+ if (decoratorText.startsWith("@app.cell")) {+ const body = fn.getChild("Body");+ if (body) {+ const cell = bodyText(body, true);+ if (cell !== null) {+ cells.push(cell);+ }+ }+ return false;}- continue;- }- if (inMultilineReturn) {- parenCount += countParens(trimmed);- if (parenCount === 0) {- inMultilineReturn = false;+ // `@app.function`: the whole function definition is the cell (the+ // decorator is dropped, but the function — including its return — stays).+ if (decoratorText.startsWith("@app.function")) {+ pushCell(fn.from, fn.to);}- continue;- }- // Add line to current cell- currentCell.push(line);- }+ return false;+ },+ });- // Handle last cell- finalizeCellIfNeeded();+ return {+ setup: setupBlocks.length > 0 ? setupBlocks.join("\n\n") : null,+ cells,+ };+}- return cells;+/**+ * Convenience wrapper returning only the ordinary cells (excluding the setup+ * block). Prefer {@link extractMarimoApp} when the setup block matters.+ */+export function extractCells(text: string): string[] {+ return extractMarimoApp(text).cells;}function dedent(text: string): string {