A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
chore: migrate out smart cell logic to its own package (#6808)
Myles Scolnick16139c51e152
diff --git a/frontend/package.json b/frontend/package.jsonindex b6f282979d..58cbb38822 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -55,6 +55,7 @@"@marimo-team/codemirror-sql": "^0.2.3","@marimo-team/llm-info": "workspace:*","@marimo-team/marimo-api": "workspace:*",+ "@marimo-team/smart-cells": "workspace:*","@marimo-team/react-slotz": "^0.2.0","@mui/material": "^6.5.0","@open-rpc/client-js": "^1.8.1",diff --git a/frontend/src/core/codemirror/language/__tests__/ast.test.ts b/frontend/src/core/codemirror/language/__tests__/ast.test.tsdeleted file mode 100644index b586675b21..0000000000--- a/frontend/src/core/codemirror/language/__tests__/ast.test.ts+++ /dev/null@@ -1,124 +0,0 @@-/* Copyright 2024 Marimo. All rights reserved. */--import type { TreeCursor } from "@lezer/common";-import { parser } from "@lezer/python";-import { describe, expect, it } from "vitest";-import { parseArgsKwargs } from "../utils/ast";--function moveToArgList(cursor: TreeCursor) {- cursor.next();- cursor.next();- cursor.next();- cursor.next();- expect(cursor.name).toBe("ArgList");-}--function createCursor(code: string) {- const tree = parser.parse(code);- const cursor = tree.cursor();- moveToArgList(cursor);- return cursor;-}--function printResults(- results: ReturnType<typeof parseArgsKwargs>,- code: string,-) {- return {- args: results.args.map(- (arg) =>- `"${arg.name}" (${arg.from}, ${arg.to}, ${code.slice(arg.from, arg.to)})`,- ),- kwargs: results.kwargs.map((kwarg) => ({- key: kwarg.key,- value: kwarg.value,- })),- };-}--describe("parseArgsKwargs", () => {- it("should parse empty arglist", () => {- const code = "fn()";- const cursor = createCursor(code);- expect(- printResults(parseArgsKwargs(cursor, code), code),- ).toMatchInlineSnapshot(`- {- "args": [],- "kwargs": [],- }- `);- });-- it("should parse positional arguments", () => {- const code = "fn(a, b)";- const cursor = createCursor(code);- expect(- printResults(parseArgsKwargs(cursor, code), code),- ).toMatchInlineSnapshot(`- {- "args": [- ""VariableName" (3, 4, a)",- ""VariableName" (6, 7, b)",- ],- "kwargs": [],- }- `);- });-- it("should parse keyword arguments", () => {- const code = "fn(a=b, c=d)";- const cursor = createCursor(code);- expect(- printResults(parseArgsKwargs(cursor, code), code),- ).toMatchInlineSnapshot(`- {- "args": [],- "kwargs": [- {- "key": "a",- "value": "b",- },- {- "key": "c",- "value": "d",- },- ],- }- `);- });-- it("should parse mixed arguments", () => {- const code = "fn(a, b=c)";- const cursor = createCursor(code);- expect(- printResults(parseArgsKwargs(cursor, code), code),- ).toMatchInlineSnapshot(`- {- "args": [- ""VariableName" (3, 4, a)",- ],- "kwargs": [- {- "key": "b",- "value": "c",- },- ],- }- `);- });-- it("should handle non-ArgList input", () => {- const code = "x";- const tree = parser.parse(code);- const cursor = tree.cursor();- expect(- printResults(parseArgsKwargs(cursor, code), code),- ).toMatchInlineSnapshot(`- {- "args": [],- "kwargs": [],- }- `);- });-});diff --git a/frontend/src/core/codemirror/language/__tests__/extension.test.ts b/frontend/src/core/codemirror/language/__tests__/extension.test.tsindex 926989b674..ee1f621f86 100644--- a/frontend/src/core/codemirror/language/__tests__/extension.test.ts+++ b/frontend/src/core/codemirror/language/__tests__/extension.test.ts@@ -174,8 +174,12 @@ describe("switchLanguage", () => {// Switch to sqlswitchLanguage(mockEditor, { language: "sql", keepCodeAsIs: false });expect(mockEditor.state.doc.toString()).toMatchInlineSnapshot(`- "print('Hello')- print('Goodbye')"+ "mo.md(+ r"""+ print('Hello')+ print('Goodbye')+ """+ )"`);// Switch back to python@@ -183,8 +187,12 @@ describe("switchLanguage", () => {expect(mockEditor.state.doc.toString()).toMatchInlineSnapshot(`"_df = mo.sql(f"""+ mo.md(+ r\\"""print('Hello')print('Goodbye')+ \\"""+ )""")"`);diff --git a/frontend/src/core/codemirror/language/languages/markdown.ts b/frontend/src/core/codemirror/language/languages/markdown.tsindex 76249160bf..a03e86016c 100644--- a/frontend/src/core/codemirror/language/languages/markdown.ts+++ b/frontend/src/core/codemirror/language/languages/markdown.ts@@ -8,7 +8,10 @@ import { languages } from "@codemirror/language-data";import { stexMath } from "@codemirror/legacy-modes/mode/stex";import type { Extension } from "@codemirror/state";import { type EditorView, ViewPlugin } from "@codemirror/view";-import dedent from "string-dedent";+import {+ type MarkdownMetadata,+ MarkdownParser,+} from "@marimo-team/smart-cells";import type { CellId } from "@/core/cells/ids";import type { CompletionConfig } from "@/core/config/config-schema";import type { HotkeyProvider } from "@/core/hotkeys/hotkeys";@@ -22,36 +25,8 @@ import { parsePython } from "../embedded/embedded-python";import { parseLatex } from "../embedded/latex";import { languageMetadataField } from "../metadata";import type { LanguageAdapter } from "../types";-import {- QUOTE_PREFIX_KINDS,- type QuotePrefixKind,- splitQuotePrefix,-} from "../utils/quotes";--export interface MarkdownLanguageAdapterMetadata {- quotePrefix: QuotePrefixKind;-}-const quoteKinds = [- ['"""', '"""'],- ["'''", "'''"],- ['"', '"'],- ["'", "'"],-];--// explode into all combinations-const pairs = QUOTE_PREFIX_KINDS.flatMap((prefix) =>- quoteKinds.map(([start, end]) => [prefix + start, end]),-);--const regexes = pairs.map(- ([start, end]) =>- // mo.md( + any number of spaces + start + capture + any number of spaces + end)- [- start,- new RegExp(`^mo\\.md\\(\\s*${start}(.*)${end}\\s*\\)$`, "s"),- ] as const,-);+export interface MarkdownLanguageAdapterMetadata extends MarkdownMetadata {}/*** Language adapter for Markdown.@@ -59,146 +34,34 @@ const regexes = pairs.map(export class MarkdownLanguageAdapterimplements LanguageAdapter<MarkdownLanguageAdapterMetadata>{+ private parser = new MarkdownParser();+readonly type = "markdown";- readonly defaultCode = 'mo.md(r"""\n""")';- readonly defaultMetadata: MarkdownLanguageAdapterMetadata = {- quotePrefix: "r",- };+ readonly defaultCode = this.parser.defaultCode;+ readonly defaultMetadata: MarkdownLanguageAdapterMetadata =+ this.parser.defaultMetadata;static fromMarkdown(markdown: string) {- return `mo.md(r"""\n${markdown}\n""")`;+ return MarkdownParser.fromMarkdown(markdown);}transformIn(pythonCode: string,): [string, number, MarkdownLanguageAdapterMetadata] {- pythonCode = pythonCode.trim();-- const metadata = { ...this.defaultMetadata };-- // empty string- if (pythonCode === "") {- return ["", 0, metadata];- }-- for (const [start, regex] of regexes) {- const match = pythonCode.match(regex);- if (match) {- const innerCode = match[1];- const [quotePrefix, quoteType] = splitQuotePrefix(start);- metadata.quotePrefix = quotePrefix;- const unescapedCode = innerCode.replaceAll(`\\${quoteType}`, quoteType);-- const offset = pythonCode.indexOf(innerCode);- // string-dedent expects the first and last line to be empty / contain only whitespace, so we pad with \n- return [dedent(`\n${unescapedCode}\n`).trim(), offset, metadata];- }- }-- // no match- return [pythonCode, 0, metadata];+ const result = this.parser.transformIn(pythonCode);+ return [result.code, result.offset, result.metadata];}transformOut(code: string,metadata: MarkdownLanguageAdapterMetadata,): [string, number] {- // NB. Must be kept consistent with marimo/_convert/utils.py- // ::markdown_to_marimo-- // Empty string- if (code === "") {- // Need at least a space, otherwise the output will be 6 quotes- code = " ";- }-- const { quotePrefix } = metadata;-- // We always transform back with triple quotes, as to avoid needing to- // escape single quotes.- const escapedCode = code.replaceAll('"""', String.raw`\"""`);-- // If its one line and not bounded by quotes, write it as single line- const isOneLine = !code.includes("\n");- const boundedByQuote = code.startsWith('"') || code.endsWith('"');- if (isOneLine && !boundedByQuote) {- const start = `mo.md(${quotePrefix}"""`;- const end = `""")`;- return [start + escapedCode + end, start.length];- }-- // Multiline code- const start = `mo.md(\n ${quotePrefix}"""\n`;- const end = `\n"""\n)`;- return [start + escapedCode + end, start.length + 1];+ const result = this.parser.transformOut(code, metadata);+ return [result.code, result.offset];}isSupported(pythonCode: string): boolean {- pythonCode = pythonCode.trim();-- // Empty strings are supported- if (pythonCode === "") {- return true;- }-- // Must start with mo.md(- if (!pythonCode.startsWith("mo.md(")) {- return false;- }-- // Empty function calls are supported- if (pythonCode === "mo.md()") {- return true;- }-- // Parse the code using Lezer and check for the exact match of mo.md() signature- const tree = pythonLanguage.parser.parse(pythonCode);-- // This is the exact match of mo.md() signature- const enterOrder: { match: string | RegExp; stop?: boolean }[] = [- { match: "Script" },- { match: "ExpressionStatement" },- { match: "CallExpression" },- { match: "MemberExpression" },- { match: "VariableName" },- { match: "." },- { match: "PropertyName" },- { match: "ArgList" },- { match: "(" },- { match: /String|FormatString/, stop: true },- { match: ")" },- ];-- let isValid = true;-- // Parse the code using Lezer to check for multiple function calls and string content- tree.iterate({- enter: (node) => {- const current = enterOrder.shift();- if (current === undefined) {- // If our list is empty, but we are still going- // then this is not a valid call- isValid = false;- return false;- }-- const match = current.match;-- if (typeof match === "string") {- isValid = isValid && match === node.name;- return isValid && !current.stop;- }-- if (!match.test(node.name)) {- isValid = false;- return isValid && !current.stop;- }-- return isValid && !current.stop;- },- });-- return isValid;+ return this.parser.isSupported(pythonCode);}getExtension(diff --git a/frontend/src/core/codemirror/language/languages/python.ts b/frontend/src/core/codemirror/language/languages/python.tsindex b37a2c7de4..1d74cf7750 100644--- a/frontend/src/core/codemirror/language/languages/python.ts+++ b/frontend/src/core/codemirror/language/languages/python.ts@@ -17,6 +17,7 @@ import {LanguageServerClient,languageServerWithClient,} from "@marimo-team/codemirror-languageserver";+import { PythonParser } from "@marimo-team/smart-cells";import type { CellId } from "@/core/cells/ids";import { hasCapability } from "@/core/config/capabilities";import type {@@ -172,20 +173,24 @@ const pyrightClient = once((_: LSPConfig) => {* Language adapter for Python.*/export class PythonLanguageAdapter implements LanguageAdapter<{}> {+ private parser = new PythonParser();+readonly type = "python";- readonly defaultCode = "";- readonly defaultMetadata = {};+ readonly defaultCode = this.parser.defaultCode;+ readonly defaultMetadata = this.parser.defaultMetadata;transformIn(code: string): [string, number, {}] {- return [code, 0, {}];+ const result = this.parser.transformIn(code);+ return [result.code, result.offset, result.metadata];}- transformOut(code: string, _metadata: {}): [string, number] {- return [code, 0];+ transformOut(code: string, metadata: {}): [string, number] {+ const result = this.parser.transformOut(code, metadata);+ return [result.code, result.offset];}- isSupported(_code: string): boolean {- return true;+ isSupported(code: string): boolean {+ return this.parser.isSupported(code);}getExtension(diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.tsindex 9f8c8d6bfe..b11d1e604e 100644--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts@@ -6,8 +6,6 @@ import { type SQLDialect, type SQLNamespace, sql } from "@codemirror/lang-sql";import type { EditorState, Extension } from "@codemirror/state";import { Compartment } from "@codemirror/state";import { EditorView, keymap } from "@codemirror/view";-import type { SyntaxNode, TreeCursor } from "@lezer/common";-import { parser } from "@lezer/python";import {defaultSqlHoverTheme,NodeSqlParser,@@ -17,7 +15,7 @@ import {sqlExtension,} from "@marimo-team/codemirror-sql";import { DuckDBDialect } from "@marimo-team/codemirror-sql/dialects";-import dedent from "string-dedent";+import { type SQLMetadata, SQLParser } from "@marimo-team/smart-cells";import type { CellId } from "@/core/cells/ids";import { cellIdState } from "@/core/codemirror/cells/state";import type { PlaceholderType } from "@/core/codemirror/config/types";@@ -44,10 +42,6 @@ import { Logger } from "@/utils/Logger";import { variableCompletionSource } from "../../embedded/embedded-python";import { languageMetadataField } from "../../metadata";import type { LanguageAdapter } from "../../types";-import { parseArgsKwargs } from "../../utils/ast";-import { indentOneTab } from "../../utils/indentOneTab";-import type { QuotePrefixKind } from "../../utils/quotes";-import { MarkdownLanguageAdapter } from "../markdown";import {clearSqlValidationError,setSqlValidationError,@@ -65,11 +59,7 @@ const DEFAULT_PARSER_DIALECT = "DuckDB";// A compartment for the SQL config, so we can update the config of codemirrorconst sqlConfigCompartment = new Compartment();-export interface SQLLanguageAdapterMetadata {- dataframeName: string;- quotePrefix: QuotePrefixKind;- commentLines: string[];- showOutput: boolean;+export interface SQLLanguageAdapterMetadata extends SQLMetadata {engine: ConnectionName;}@@ -83,6 +73,7 @@ function getLatestEngine(): ConnectionName {export class SQLLanguageAdapterimplements LanguageAdapter<SQLLanguageAdapterMetadata>{+ private parser = new SQLParser();readonly type = "sql";sqlModeEnabled: boolean;@@ -92,25 +83,20 @@ export class SQLLanguageAdapterget defaultMetadata(): SQLLanguageAdapterMetadata {return {- dataframeName: "_df",- quotePrefix: "f",- commentLines: [],- showOutput: true,- engine: getLatestEngine() || this.defaultEngine,+ ...this.parser.defaultMetadata,+ engine: getLatestEngine() || DUCKDB_ENGINE,};}get defaultCode(): string {- const latestEngine = getLatestEngine();- if (latestEngine === this.defaultEngine) {- return `_df = mo.sql(f"""SELECT * FROM """)`;+ const engine = getLatestEngine();+ if (engine && engine !== DUCKDB_ENGINE) {+ return `_df = mo.sql(f"""SELECT * FROM """, engine=${engine})`;}- return `_df = mo.sql(f"""SELECT * FROM """, engine=${latestEngine})`;+ return this.parser.defaultCode;}- static fromQuery = (query: string) => `_df = mo.sql(f"""${query.trim()}""")`;-- private readonly defaultEngine = DUCKDB_ENGINE;+ static fromQuery = (query: string) => SQLParser.fromQuery(query);transformIn(pythonCode: string,@@ -119,100 +105,29 @@ export class SQLLanguageAdapterqueryStartOffset: number,metadata: SQLLanguageAdapterMetadata,] {- pythonCode = pythonCode.trim();-- // Default metadata- const metadata: SQLLanguageAdapterMetadata = {- ...this.defaultMetadata,- commentLines: this.extractCommentLines(pythonCode),- };-- if (!this.isSupported(pythonCode)) {- // Attempt to remove any markdown wrappers- const [transformedCode, offset] =- new MarkdownLanguageAdapter().transformIn(pythonCode);- // Just return the original code- return [transformedCode, offset, metadata];- }+ this.parser.defaultMetadata.engine = getLatestEngine() || DUCKDB_ENGINE;+ const result = this.parser.transformIn(pythonCode);- // Handle empty strings- if (pythonCode === "") {- return ["", 0, metadata];- }-- const sqlStatement = parseSQLStatement(pythonCode);- if (sqlStatement) {- metadata.dataframeName = sqlStatement.dfName;- metadata.showOutput = sqlStatement.output ?? true;- metadata.engine =- (sqlStatement.engine as ConnectionName) ?? this.defaultEngine;-- if (metadata.engine !== this.defaultEngine) {- // User selected a new engine, set it as latest.- // This makes new SQL statements use the new engine by default.- setLatestEngineSelected(metadata.engine);- }+ // Handle engine selection side effect+ const metadata = result.metadata as SQLLanguageAdapterMetadata;- return [- dedent(`\n${sqlStatement.sqlString}\n`).trim(),- sqlStatement.startPosition,- metadata,- ];+ if (metadata.engine && metadata.engine !== DUCKDB_ENGINE) {+ setLatestEngineSelected(metadata.engine as ConnectionName);}- return [pythonCode, 0, metadata];+ return [result.code, result.offset, metadata];}transformOut(code: string,metadata: SQLLanguageAdapterMetadata,): [string, number] {- const { quotePrefix, commentLines, showOutput, engine, dataframeName } =- metadata;-- const start = `${dataframeName} = mo.sql(\n ${quotePrefix}"""\n`;- const escapedCode = code.replaceAll('"""', String.raw`\"""`);-- const showOutputParam = showOutput ? "" : ",\n output=False";- const engineParam =- engine === this.defaultEngine ? "" : `,\n engine=${engine}`;- const end = `\n """${showOutputParam}${engineParam}\n)`;-- return [- [...commentLines, start].join("\n") + indentOneTab(escapedCode) + end,- start.length + 1,- ];+ const result = this.parser.transformOut(code, metadata);+ return [result.code, result.offset];}isSupported(pythonCode: string): boolean {- if (pythonCode.trim() === "") {- return true;- }-- // Has at least one `mo.sql` call- if (!pythonCode.includes("mo.sql")) {- return false;- }-- // Does not have 2 `mo.sql` calls- if (pythonCode.split("mo.sql").length > 2) {- return false;- }-- return parseSQLStatement(pythonCode) !== null;- }-- private extractCommentLines(pythonCode: string): string[] {- const lines = pythonCode.split("\n");- const commentLines = [];- for (const line of lines) {- if (line.startsWith("#")) {- commentLines.push(line);- } else {- break;- }- }- return commentLines;+ return this.parser.isSupported(pythonCode);}getExtension(@@ -476,194 +391,6 @@ function connectionNameToParserDialect(}}-interface SQLParseInfo {- dfName: string;- sqlString: string;- engine: string | undefined;- output: boolean | undefined;- startPosition: number;-}--// Finds an assignment node that is preceded only by comments.-function findAssignment(cursor: TreeCursor): SyntaxNode | null {- do {- if (cursor.name === "AssignStatement") {- return cursor.node;- }-- if (cursor.name !== "Comment") {- return null;- }- } while (cursor.next());- return null;-}--function getStringContent(node: SyntaxNode, code: string): string | null {- // Handle triple quoted strings- if (node.name === "String") {- const content = code.slice(node.from, node.to);- // Remove quotes and trim- if (content.startsWith('"""') || content.startsWith("'''")) {- return safeDedent(content.slice(3, -3));- }- // Handle single quoted strings- return safeDedent(content.slice(1, -1));- }- // Handle f-strings- if (node.name === "FormatString") {- const content = code.slice(node.from, node.to);- if (content.startsWith('f"""') || content.startsWith("f'''")) {- return safeDedent(content.slice(4, -3));- }- return safeDedent(content.slice(2, -1));- }- return null;-}--/**- * Parses a SQL statement from a Python code string.- *- * @param code - The Python code string to parse.- * @returns The parsed SQL statement or null if parsing fails.- */-function parseSQLStatement(code: string): SQLParseInfo | null {- try {- const tree = parser.parse(code);- const cursor = tree.cursor();-- // Trees start with a Script node.- if (cursor.name === "Script") {- cursor.next();- }- const assignStmt = findAssignment(cursor);- if (!assignStmt) {- return null;- }-- // Code after the assignment statement is not allowed.- if (code.slice(assignStmt.to).trim().length > 0) {- return null;- }-- let dfName: string | null = null;- let sqlString: string | null = null;- let engine: string | undefined;- let output: boolean | undefined;- let startPosition = 0;-- // Parse the assignment- const assignCursor = assignStmt.cursor();- assignCursor.firstChild(); // Move to first child of assignment-- // First child should be the variable name- if (assignCursor.name === "VariableName") {- dfName = code.slice(assignCursor.from, assignCursor.to);- }-- if (!dfName) {- return null;- }-- // Move to the expression part (after the =)- while (assignCursor.next()) {- if (assignCursor.name === "CallExpression") {- // Check if it's mo.sql call- const callCursor = assignCursor.node.cursor();- let isMoSql = false;-- callCursor.firstChild(); // Move to first child of call- if (callCursor.name === "MemberExpression") {- const memberText = code.slice(callCursor.from, callCursor.to);- isMoSql = memberText === "mo.sql";- }-- if (!isMoSql) {- return null;- }-- // Move to arguments- while (callCursor.next()) {- if (callCursor.name === "ArgList") {- const argListCursor = callCursor.node.cursor();-- const { args, kwargs } = parseArgsKwargs(argListCursor, code);-- // Parse positional args (SQL query)- if (args.length === 1) {- sqlString = getStringContent(args[0], code);- startPosition =- args[0].from +- getPrefixLength(code.slice(args[0].from, args[0].to));- }-- // Parse kwargs (engine and output)- for (const { key, value } of kwargs) {- switch (key) {- case "engine":- engine = value;- break;- case "output":- output = value === "True";- break;- }- }-- // Check if sql string is empty- if (sqlString === "") {- return { dfName, sqlString: "", engine, output, startPosition };- }-- break;- }- }- }- }-- if (!dfName || !sqlString) {- return null;- }-- return {- dfName,- sqlString,- engine,- output,- startPosition,- };- } catch (error) {- Logger.warn("Failed to parse SQL statement", { error: error });- return null;- }-}--function getPrefixLength(code: string): number {- if (code === "") {- return 0;- }- if (code.startsWith('f"""') || code.startsWith("f'''")) {- return 4;- }- if (code.startsWith('"""') || code.startsWith("'''")) {- return 3;- }- if (code.startsWith("f'") || code.startsWith('f"')) {- return 2;- }- if (code.startsWith("'") || code.startsWith('"')) {- return 1;- }- return 0;-}--function safeDedent(code: string): string {- try {- // Dedent expects the first and last line to be empty / contain only whitespace, so we pad with \n- return dedent(`\n${code}\n`).trim();- } catch {- return code;- }-}-const SQL_VALIDATION_DEBOUNCE_MS = 300;/**diff --git a/frontend/src/core/codemirror/language/panel/markdown.tsx b/frontend/src/core/codemirror/language/panel/markdown.tsxindex 19aed2c4be..30eef12d65 100644--- a/frontend/src/core/codemirror/language/panel/markdown.tsx+++ b/frontend/src/core/codemirror/language/panel/markdown.tsx@@ -1,5 +1,6 @@/* Copyright 2024 Marimo. All rights reserved. */-import type { QuotePrefixKind } from "../utils/quotes";++import type { QuotePrefixKind } from "@marimo-team/smart-cells";// Based on the current quote prefix and the checkbox state, return the new quote prefixexport function getQuotePrefix({diff --git a/frontend/src/core/codemirror/language/panel/panel.tsx b/frontend/src/core/codemirror/language/panel/panel.tsxindex a61dcedec4..919deebe11 100644--- a/frontend/src/core/codemirror/language/panel/panel.tsx+++ b/frontend/src/core/codemirror/language/panel/panel.tsx@@ -1,5 +1,6 @@/* Copyright 2024 Marimo. All rights reserved. */import type { EditorView } from "@codemirror/view";+import type { QuotePrefixKind } from "@marimo-team/smart-cells";import { InfoIcon, PaintRollerIcon } from "lucide-react";import { Button } from "@/components/ui/button";import { Checkbox } from "@/components/ui/checkbox";@@ -20,7 +21,6 @@ import {updateLanguageMetadata,} from "../metadata";import type { LanguageMetadataOf } from "../types";-import type { QuotePrefixKind } from "../utils/quotes";import { getQuotePrefix, MarkdownQuotePrefixTooltip } from "./markdown";import { SQLEngineSelect, SQLModeSelect } from "./sql";diff --git a/frontend/src/core/codemirror/language/utils/ast.ts b/frontend/src/core/codemirror/language/utils/ast.tsdeleted file mode 100644index 331619c39a..0000000000--- a/frontend/src/core/codemirror/language/utils/ast.ts+++ /dev/null@@ -1,84 +0,0 @@-/* Copyright 2024 Marimo. All rights reserved. */--import type { SyntaxNode, TreeCursor } from "@lezer/common";-import { Logger } from "@/utils/Logger";--const SYNTAX_TOKENS = new Set(["(", "[", "{", ",", ")", "]", "}"]);--export function parseArgsKwargs(- argCursor: TreeCursor,- code: string,-): {- args: SyntaxNode[];- kwargs: { key: string; value: string }[];-} {- // Check we are in an ArgList- const name = argCursor.name;-- if (name !== "ArgList") {- Logger.warn(`Not an ArgList, name: ${name}`);- return { args: [], kwargs: [] };- }-- return { args: parseArgs(argCursor), kwargs: parseKwargs(argCursor, code) };-}--function parseArgs(argCursor: TreeCursor): SyntaxNode[] {- // Move to first argument- argCursor.firstChild();-- const args: SyntaxNode[] = [];- let name = argCursor.name;-- do {- name = argCursor.name;- if (SYNTAX_TOKENS.has(name)) {- continue;- }-- if (name === "VariableName") {- // Stop if we hit a kwarg (next token is AssignOp)- const peek = argCursor.node.nextSibling;- if (peek?.name === "AssignOp") {- argCursor.prev();- break;- }- }-- args.push(argCursor.node);- } while (argCursor.nextSibling());-- return args;-}--function parseKwargs(- argCursor: TreeCursor,- code: string,-): { key: string; value: string }[] {- const kwargs: { key: string; value: string }[] = [];- let name = argCursor.name;-- do {- name = argCursor.name;- if (name === "VariableName") {- const key = code.slice(argCursor.from, argCursor.to);-- // Check for AssignOp- const assignNode = argCursor.node.nextSibling;- if (!assignNode || assignNode.name !== "AssignOp") {- continue;- }-- // Get the value node- const valueNode = assignNode.nextSibling;- if (!valueNode) {- continue;- }-- const value = code.slice(valueNode.from, valueNode.to).trim();- kwargs.push({ key, value });- }- } while (argCursor.nextSibling());-- return kwargs;-}diff --git a/frontend/src/core/codemirror/language/utils/quotes.ts b/frontend/src/core/codemirror/language/utils/quotes.tsdeleted file mode 100644index f569c6b4a7..0000000000--- a/frontend/src/core/codemirror/language/utils/quotes.ts+++ /dev/null@@ -1,18 +0,0 @@-/* Copyright 2024 Marimo. All rights reserved. */--export const QUOTE_PREFIX_KINDS = ["", "f", "r", "fr", "rf"] as const;-export type QuotePrefixKind = (typeof QUOTE_PREFIX_KINDS)[number];--// Remove the f, r, fr, rf prefixes from the quote-export function splitQuotePrefix(quote: string): [QuotePrefixKind, string] {- // start with the longest prefix- const prefixKindsByLength = [...QUOTE_PREFIX_KINDS].sort(- (a, b) => b.length - a.length,- );- for (const prefix of prefixKindsByLength) {- if (quote.startsWith(prefix)) {- return [prefix, quote.slice(prefix.length)];- }- }- return ["", quote];-}diff --git a/packages/openapi/package.json b/packages/openapi/package.jsonindex 908b65b8bc..c37864c9f6 100644--- a/packages/openapi/package.json+++ b/packages/openapi/package.json@@ -2,6 +2,7 @@"name": "@marimo-team/marimo-api","version": "0.1.0","private": true,+ "sideEffects": false,"main": "src/index.ts","types": "src/index.ts","type": "module",diff --git a/packages/smart-cells/README.md b/packages/smart-cells/README.mdnew file mode 100644index 0000000000..f1b9b35ae3--- /dev/null+++ b/packages/smart-cells/README.md@@ -0,0 +1,3 @@+# @marimo-team/smart-cells++Parse "smart cells" (markdown, SQL) to-and-from Python code.diff --git a/packages/smart-cells/package.json b/packages/smart-cells/package.jsonnew file mode 100644index 0000000000..86f22940d9--- /dev/null+++ b/packages/smart-cells/package.json@@ -0,0 +1,37 @@+{+ "name": "@marimo-team/smart-cells",+ "version": "0.1.0",+ "description": "Pure parsing library for marimo smart cells (markdown, SQL, etc.)",+ "sideEffects": false,+ "main": "src/index.ts",+ "types": "src/index.ts",+ "type": "module",+ "files": [+ "src"+ ],+ "scripts": {+ "build": "echo 'No build step'",+ "test": "vitest",+ "typecheck": "tsgo"+ },+ "dependencies": {+ "@codemirror/lang-python": "^6.2.1",+ "@lezer/common": "^1.2.1",+ "@lezer/python": "^1.1.14",+ "string-dedent": "^3.0.1"+ },+ "devDependencies": {+ "@marimo-team/tsconfig": "workspace:*",+ "@types/node": "^20.0.0",+ "typescript": "^5.3.3",+ "vitest": "^3.2.4"+ },+ "keywords": [+ "marimo",+ "parser",+ "markdown",+ "sql",+ "python"+ ],+ "license": "Apache-2.0"+}diff --git a/packages/smart-cells/src/__tests__/index.test.ts b/packages/smart-cells/src/__tests__/index.test.tsnew file mode 100644index 0000000000..bba4a94f9d--- /dev/null+++ b/packages/smart-cells/src/__tests__/index.test.ts@@ -0,0 +1,13 @@+import { expect, it } from "vitest";++import * as index from "../index.js";++it("index.ts should export the correct modules", () => {+ expect(Object.keys(index)).toMatchInlineSnapshot(`+ [+ "MarkdownParser",+ "PythonParser",+ "SQLParser",+ ]+ `);+});diff --git a/packages/smart-cells/src/__tests__/markdown-parser.test.ts b/packages/smart-cells/src/__tests__/markdown-parser.test.tsnew file mode 100644index 0000000000..4626c6ff1a--- /dev/null+++ b/packages/smart-cells/src/__tests__/markdown-parser.test.ts@@ -0,0 +1,193 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { describe, expect, it } from "vitest";+import { MarkdownParser } from "../parsers/markdown-parser.js";++const parser = new MarkdownParser();++describe("MarkdownParser", () => {+ describe("transformIn", () => {+ it("should extract markdown from triple double-quoted strings", () => {+ const pythonCode = 'mo.md("""# Markdown Title\n\nSome content here.""")';+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("# Markdown Title\n\nSome content here.");+ expect(offset).toBe(9);+ expect(metadata.quotePrefix).toBe("");+ });++ it("should handle single double-quoted strings", () => {+ const pythonCode = 'mo.md("Some *markdown* content")';+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("Some *markdown* content");+ expect(offset).toBe(7);+ expect(metadata.quotePrefix).toBe("");+ });++ it("should handle f-strings", () => {+ const pythonCode = 'mo.md(f"""# Title\n{some_variable}""")';+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("# Title\n{some_variable}");+ expect(offset).toBe(10);+ expect(metadata.quotePrefix).toBe("f");+ });++ it("should handle r-strings", () => {+ const pythonCode = `mo.md(r"$\\nu = \\mathllap{}\\cdot\\mathllap{\\alpha}$")`;+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe(String.raw`$\nu = \mathllap{}\cdot\mathllap{\alpha}$`);+ expect(offset).toBe(8);+ expect(metadata.quotePrefix).toBe("r");+ });++ it("should handle rf-strings", () => {+ const pythonCode = 'mo.md(rf"""# Title\n{some_variable}""")';+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("# Title\n{some_variable}");+ expect(offset).toBe(11);+ expect(metadata.quotePrefix).toBe("rf");+ });++ it("should unescape quotes", () => {+ const pythonCode = String.raw`mo.md("""This is some \"""content\"""!""")`;+ const { code } = parser.transformIn(pythonCode);+ expect(code).toBe(`This is some """content"""!`);+ });++ it("should dedent indented strings", () => {+ const pythonCode =+ 'mo.md(\n\t"""\n\t- item 1\n\t-item 2\n\t-item3\n\t"""\n)';+ const { code } = parser.transformIn(pythonCode);+ expect(code).toBe("- item 1\n-item 2\n-item3");+ });++ it("should handle empty strings", () => {+ const { code, offset } = parser.transformIn('mo.md("")');+ expect(code).toBe("");+ expect(offset).toBe(0);+ });++ it("should return original code for unsupported format", () => {+ const pythonCode = 'print("Hello, World!")';+ const { code, offset } = parser.transformIn(pythonCode);+ expect(code).toBe('print("Hello, World!")');+ expect(offset).toBe(0);+ });+ });++ describe("transformOut", () => {+ it("should wrap single line markdown", () => {+ const code = "Hello world";+ const { code: pythonCode, offset } = parser.transformOut(code, {+ quotePrefix: "",+ });+ expect(pythonCode).toBe(`mo.md("""Hello world""")`);+ expect(offset).toBe(9);+ });++ it("should wrap multiline markdown", () => {+ const code = "# Markdown Title\n\nSome content here.";+ const { code: pythonCode, offset } = parser.transformOut(code, {+ quotePrefix: "",+ });+ expect(pythonCode).toMatchInlineSnapshot(`+ "mo.md(+ """+ # Markdown Title++ Some content here.+ """+ )"+ `);+ expect(offset).toBe(16);+ });++ it("should preserve r-string prefix", () => {+ const code = String.raw`$\nu = \mathllap{}\cdot\mathllap{\alpha}$`;+ const { code: pythonCode } = parser.transformOut(code, {+ quotePrefix: "r",+ });+ expect(pythonCode).toBe(+ `mo.md(r"""$\\nu = \\mathllap{}\\cdot\\mathllap{\\alpha}$""")`,+ );+ });++ it("should preserve f-string prefix", () => {+ const code = "# Title\n{some_variable}";+ const { code: pythonCode } = parser.transformOut(code, {+ quotePrefix: "f",+ });+ expect(pythonCode).toMatchInlineSnapshot(`+ "mo.md(+ f"""+ # Title+ {some_variable}+ """+ )"+ `);+ });++ it("should escape triple quotes", () => {+ const code = 'Markdown with an escaped """quote"""!!';+ const { code: pythonCode } = parser.transformOut(code, {+ quotePrefix: "",+ });+ expect(pythonCode).toBe(+ `mo.md("""Markdown with an escaped \\"""quote\\"""!!""")`,+ );+ });++ it("should handle empty string", () => {+ const { code: pythonCode } = parser.transformOut("", {+ quotePrefix: "r",+ });+ expect(pythonCode).toBe(`mo.md(r""" """)`);+ });+ });++ describe("isSupported", () => {+ it("should return true for valid formats", () => {+ const validCases = [+ 'mo.md("""# Markdown""")',+ "mo.md()",+ 'mo.md("")',+ "mo.md(f'hello world')",+ 'mo.md(r"hello world")',+ "mo.md(rf'hello world')",+ 'mo.md(fr"hello world")',+ ];++ for (const pythonCode of validCases) {+ expect(parser.isSupported(pythonCode)).toBe(true);+ }+ });++ it("should return false for invalid formats", () => {+ const invalidCases = [+ 'print("Hello, World!")',+ 'mo.md("test"), mo.md("test2")', // Multiple calls+ 'mo.md("test"); print("hi")', // Additional statements+ ];++ for (const pythonCode of invalidCases) {+ expect(parser.isSupported(pythonCode)).toBe(false);+ }+ });++ it("should return true for empty string", () => {+ expect(parser.isSupported("")).toBe(true);+ });+ });++ describe("roundtrip", () => {+ it("should roundtrip markdown correctly", () => {+ const originalMarkdown = "# Hello\n\nThis is **bold** text.";+ const pythonCode = MarkdownParser.fromMarkdown(originalMarkdown);+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe(originalMarkdown);++ const { code: backToPython } = parser.transformOut(code, metadata);+ const { code: roundtripped } = parser.transformIn(backToPython);+ expect(roundtripped).toBe(originalMarkdown);+ });+ });+});diff --git a/packages/smart-cells/src/__tests__/python-ast.test.ts b/packages/smart-cells/src/__tests__/python-ast.test.tsnew file mode 100644index 0000000000..8644c77346--- /dev/null+++ b/packages/smart-cells/src/__tests__/python-ast.test.ts@@ -0,0 +1,363 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { TreeCursor } from "@lezer/common";+import { describe, expect, it } from "vitest";+import {+ getPrefixLength,+ getStringContent,+ parseArgsKwargs,+ parsePythonAST,+} from "../utils/python-ast.js";++function moveToArgList(cursor: TreeCursor) {+ cursor.next();+ cursor.next();+ cursor.next();+ cursor.next();+ expect(cursor.name).toBe("ArgList");+}++function createCursor(code: string) {+ const tree = parsePythonAST(code);+ const cursor = tree.cursor();+ moveToArgList(cursor);+ return cursor;+}++function printResults(+ results: ReturnType<typeof parseArgsKwargs>,+ code: string,+) {+ return {+ args: results.args.map(+ (arg) =>+ `"${arg.name}" (${arg.from}, ${arg.to}, ${code.slice(arg.from, arg.to)})`,+ ),+ kwargs: results.kwargs.map((kwarg) => ({+ key: kwarg.key,+ value: kwarg.value,+ })),+ };+}++describe("parsePythonAST", () => {+ it("should parse valid Python code", () => {+ const tree = parsePythonAST("print('hello')");+ expect(tree).toBeDefined();+ expect(tree.length).toBeGreaterThan(0);+ });++ it("should parse empty code", () => {+ const tree = parsePythonAST("");+ expect(tree).toBeDefined();+ });+});++describe("parseArgsKwargs", () => {+ it("should parse empty arglist", () => {+ const code = "fn()";+ const cursor = createCursor(code);+ expect(+ printResults(parseArgsKwargs(cursor, code), code),+ ).toMatchInlineSnapshot(`+ {+ "args": [],+ "kwargs": [],+ }+ `);+ });++ it("should parse positional arguments", () => {+ const code = "fn(a, b)";+ const cursor = createCursor(code);+ expect(+ printResults(parseArgsKwargs(cursor, code), code),+ ).toMatchInlineSnapshot(`+ {+ "args": [+ ""VariableName" (3, 4, a)",+ ""VariableName" (6, 7, b)",+ ],+ "kwargs": [],+ }+ `);+ });++ it("should parse keyword arguments", () => {+ const code = "fn(a=b, c=d)";+ const cursor = createCursor(code);+ expect(+ printResults(parseArgsKwargs(cursor, code), code),+ ).toMatchInlineSnapshot(`+ {+ "args": [],+ "kwargs": [+ {+ "key": "a",+ "value": "b",+ },+ {+ "key": "c",+ "value": "d",+ },+ ],+ }+ `);+ });++ it("should parse mixed arguments", () => {+ const code = "fn(a, b=c)";+ const cursor = createCursor(code);+ expect(+ printResults(parseArgsKwargs(cursor, code), code),+ ).toMatchInlineSnapshot(`+ {+ "args": [+ ""VariableName" (3, 4, a)",+ ],+ "kwargs": [+ {+ "key": "b",+ "value": "c",+ },+ ],+ }+ `);+ });++ it("should handle non-ArgList input", () => {+ const code = "x";+ const tree = parsePythonAST(code);+ const cursor = tree.cursor();+ expect(+ printResults(parseArgsKwargs(cursor, code), code),+ ).toMatchInlineSnapshot(`+ {+ "args": [],+ "kwargs": [],+ }+ `);+ });++ it("should parse complex keyword arguments", () => {+ const code = "fn(output=True, engine=postgres_engine)";+ const cursor = createCursor(code);+ const result = parseArgsKwargs(cursor, code);+ expect(result.args).toHaveLength(0);+ expect(result.kwargs).toHaveLength(2);+ expect(result.kwargs[0]).toEqual({ key: "output", value: "True" });+ expect(result.kwargs[1]).toEqual({+ key: "engine",+ value: "postgres_engine",+ });+ });++ it("should parse string literal arguments", () => {+ const code = 'fn("hello", x=42)';+ const cursor = createCursor(code);+ const result = parseArgsKwargs(cursor, code);+ expect(result.args).toHaveLength(1);+ expect(result.kwargs).toHaveLength(1);+ expect(result.kwargs[0]).toEqual({ key: "x", value: "42" });+ });+});++describe("getStringContent", () => {+ function findStringNode(code: string) {+ const tree = parsePythonAST(code);+ const cursor = tree.cursor();+ while (cursor.next()) {+ if (cursor.name === "String" || cursor.name === "FormatString") {+ return cursor.node;+ }+ }+ return null;+ }++ it("should extract content from triple double-quoted strings", () => {+ const code = '"""hello world"""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello world");+ }+ });++ it("should extract content from triple single-quoted strings", () => {+ const code = "'''hello world'''";+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello world");+ }+ });++ it("should extract content from single double-quoted strings", () => {+ const code = '"hello"';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello");+ }+ });++ it("should extract content from single single-quoted strings", () => {+ const code = "'hello'";+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello");+ }+ });++ it("should extract content from f-strings with triple quotes", () => {+ const code = 'f"""hello {world}"""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello {world}");+ }+ });++ it("should extract content from f-strings with single quotes", () => {+ const code = 'f"hello {world}"';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello {world}");+ }+ });++ it("should extract content from rf-strings", () => {+ const code = 'rf"""hello {world}"""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello {world}");+ }+ });++ it("should extract content from fr-strings", () => {+ const code = 'fr"""hello {world}"""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello {world}");+ }+ });++ it("should extract content from r-strings with triple quotes", () => {+ const code = 'r"""hello\\nworld"""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello\\nworld");+ }+ });++ it("should extract content from r-strings with single quotes", () => {+ const code = 'r"hello\\nworld"';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("hello\\nworld");+ }+ });++ it("should return null for non-string nodes", () => {+ const code = "42";+ const tree = parsePythonAST(code);+ const node = tree.topNode.firstChild;+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBeNull();+ }+ });++ it("should handle empty strings", () => {+ const code = '""';+ const node = findStringNode(code);+ expect(node).not.toBeNull();+ if (node) {+ const content = getStringContent(node, code);+ expect(content).toBe("");+ }+ });+});++describe("getPrefixLength", () => {+ it("should return 0 for empty string", () => {+ expect(getPrefixLength("")).toBe(0);+ });++ it("should return correct length for triple double quotes", () => {+ expect(getPrefixLength('"""hello"""')).toBe(3);+ });++ it("should return correct length for triple single quotes", () => {+ expect(getPrefixLength("'''hello'''")).toBe(3);+ });++ it("should return correct length for single double quote", () => {+ expect(getPrefixLength('"hello"')).toBe(1);+ });++ it("should return correct length for single single quote", () => {+ expect(getPrefixLength("'hello'")).toBe(1);+ });++ it("should return correct length for f-strings with triple quotes", () => {+ expect(getPrefixLength('f"""hello"""')).toBe(4);+ expect(getPrefixLength("f'''hello'''")).toBe(4);+ });++ it("should return correct length for f-strings with single quotes", () => {+ expect(getPrefixLength('f"hello"')).toBe(2);+ expect(getPrefixLength("f'hello'")).toBe(2);+ });++ it("should return correct length for r-strings with triple quotes", () => {+ expect(getPrefixLength('r"""hello"""')).toBe(4);+ expect(getPrefixLength("r'''hello'''")).toBe(4);+ });++ it("should return correct length for r-strings with single quotes", () => {+ expect(getPrefixLength('r"hello"')).toBe(2);+ expect(getPrefixLength("r'hello'")).toBe(2);+ });++ it("should return correct length for rf-strings with triple quotes", () => {+ expect(getPrefixLength('rf"""hello"""')).toBe(5);+ expect(getPrefixLength("rf'''hello'''")).toBe(5);+ });++ it("should return correct length for rf-strings with single quotes", () => {+ expect(getPrefixLength('rf"hello"')).toBe(3);+ expect(getPrefixLength("rf'hello'")).toBe(3);+ });++ it("should return correct length for fr-strings with triple quotes", () => {+ expect(getPrefixLength('fr"""hello"""')).toBe(5);+ expect(getPrefixLength("fr'''hello'''")).toBe(5);+ });++ it("should return correct length for fr-strings with single quotes", () => {+ expect(getPrefixLength('fr"hello"')).toBe(3);+ expect(getPrefixLength("fr'hello'")).toBe(3);+ });++ it("should return 0 for non-string content", () => {+ expect(getPrefixLength("hello")).toBe(0);+ expect(getPrefixLength("42")).toBe(0);+ });+});diff --git a/packages/smart-cells/src/__tests__/python-parser.test.ts b/packages/smart-cells/src/__tests__/python-parser.test.tsnew file mode 100644index 0000000000..ab70feb7a1--- /dev/null+++ b/packages/smart-cells/src/__tests__/python-parser.test.ts@@ -0,0 +1,28 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { describe, expect, it } from "vitest";+import { PythonParser } from "../parsers/python-parser.js";++const parser = new PythonParser();++describe("PythonParser", () => {+ it("should pass through code unchanged", () => {+ const code = 'print("Hello, World!")';+ const { code: result, offset, metadata } = parser.transformIn(code);+ expect(result).toBe(code);+ expect(offset).toBe(0);+ expect(metadata).toEqual({});+ });++ it("should support any code", () => {+ expect(parser.isSupported("anything goes")).toBe(true);+ expect(parser.isSupported("")).toBe(true);+ });++ it("should roundtrip correctly", () => {+ const code = "x = 42\nprint(x)";+ const { code: transformed, metadata } = parser.transformIn(code);+ const { code: backToOriginal } = parser.transformOut(transformed, metadata);+ expect(backToOriginal).toBe(code);+ });+});diff --git a/packages/smart-cells/src/__tests__/sql-parser.test.ts b/packages/smart-cells/src/__tests__/sql-parser.test.tsnew file mode 100644index 0000000000..72a51a0aee--- /dev/null+++ b/packages/smart-cells/src/__tests__/sql-parser.test.ts@@ -0,0 +1,228 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { describe, expect, it } from "vitest";+import { SQLParser } from "../parsers/sql-parser.js";++const parser = new SQLParser();++describe("SQLParser", () => {+ describe("transformIn", () => {+ it("should extract SQL from triple double-quoted strings", () => {+ const pythonCode = '_df = mo.sql("""SELECT * FROM {df}""")';+ const { code, offset, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM {df}");+ expect(metadata.dataframeName).toBe("_df");+ expect(offset).toBe(16);+ });++ it("should handle single double-quoted strings", () => {+ const pythonCode = 'next_df = mo.sql("SELECT * FROM {df}")';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM {df}");+ expect(metadata.dataframeName).toBe("next_df");+ });++ it("should handle output flag set to True", () => {+ const pythonCode = '_df = mo.sql("""SELECT * FROM table""", output=True)';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM table");+ expect(metadata.showOutput).toBe(true);+ });++ it("should handle output flag set to False", () => {+ const pythonCode =+ '_df = mo.sql("""SELECT * FROM table""", output=False)';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM table");+ expect(metadata.showOutput).toBe(false);+ });++ it("should handle engine param when provided", () => {+ const pythonCode =+ '_df = mo.sql("""SELECT * FROM table""", engine=postgres_engine)';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM table");+ expect(metadata.engine).toBe("postgres_engine");+ });++ it("should handle engine and output params together", () => {+ const pythonCode =+ '_df = mo.sql("""SELECT * FROM table""", output=False, engine=postgres_engine)';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM table");+ expect(metadata.showOutput).toBe(false);+ expect(metadata.engine).toBe("postgres_engine");+ });++ it("should handle parametrized SQL with f-strings", () => {+ const pythonCode = `+_df = mo.sql(+ f"""+ SELECT name, price, category+ FROM products+ WHERE price < {price_threshold.value}+ ORDER BY price DESC+ """,+ engine=sqlite,+)+`;+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe(+ `SELECT name, price, category+FROM products+WHERE price < {price_threshold.value}+ORDER BY price DESC`.trim(),+ );+ expect(metadata.engine).toBe("sqlite");+ });++ it("should handle empty SQL string", () => {+ const pythonCode = 'next_df = mo.sql("")';+ const { code, offset } = parser.transformIn(pythonCode);+ expect(code).toBe("");+ expect(offset).toBe(18);+ });++ it("should preserve Python comments", () => {+ const pythonCode = '# hello\nmy_df = mo.sql("""SELECT * FROM {df}""")';+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe("SELECT * FROM {df}");+ expect(metadata.commentLines).toEqual(["# hello"]);+ });+ });++ describe("transformOut", () => {+ it("should wrap SQL code with default settings", () => {+ const code = "SELECT * FROM {df}";+ const metadata = {+ dataframeName: "my_df",+ quotePrefix: "f" as const,+ commentLines: [],+ showOutput: true,+ engine: "__marimo_duckdb",+ };+ const { code: pythonCode } = parser.transformOut(code, metadata);+ expect(pythonCode).toMatchInlineSnapshot(`+ "my_df = mo.sql(+ f"""+ SELECT * FROM {df}+ """+ )"+ `);+ });++ it("should include output flag when set to False", () => {+ const code = "SELECT * FROM table";+ const metadata = {+ dataframeName: "my_df",+ quotePrefix: "f" as const,+ commentLines: [],+ showOutput: false,+ engine: "__marimo_duckdb",+ };+ const { code: pythonCode } = parser.transformOut(code, metadata);+ expect(pythonCode).toMatchInlineSnapshot(`+ "my_df = mo.sql(+ f"""+ SELECT * FROM table+ """,+ output=False+ )"+ `);+ });++ it("should add engine connection when provided", () => {+ const code = "SELECT * FROM table";+ const metadata = {+ dataframeName: "_df",+ quotePrefix: "f" as const,+ commentLines: ["# hello"],+ showOutput: true,+ engine: "postgres_engine",+ };+ const { code: pythonCode } = parser.transformOut(code, metadata);+ expect(pythonCode).toMatchInlineSnapshot(`+ "# hello+ _df = mo.sql(+ f"""+ SELECT * FROM table+ """,+ engine=postgres_engine+ )"+ `);+ });++ it("should add both engine and output params", () => {+ const code = "SELECT * FROM table";+ const metadata = {+ dataframeName: "_df",+ quotePrefix: "f" as const,+ commentLines: ["# hello"],+ showOutput: false,+ engine: "postgres_engine",+ };+ const { code: pythonCode } = parser.transformOut(code, metadata);+ expect(pythonCode).toMatchInlineSnapshot(`+ "# hello+ _df = mo.sql(+ f"""+ SELECT * FROM table+ """,+ output=False,+ engine=postgres_engine+ )"+ `);+ });+ });++ describe("isSupported", () => {+ it("should return true for supported formats", () => {+ const validCases = [+ 'df2 = mo.sql("""SELECT * FROM {df}""")',+ "my_df = mo.sql('')",+ 'df = mo.sql("")',+ '# this is a sql cell\ndf = mo.sql("")',+ '# multiple\n# comments\ndf = mo.sql("")',+ 'df = mo.sql("""SELECT 1""", output=True)',+ 'df = mo.sql("""SELECT 1""", engine=postgres)',+ ];++ for (const pythonCode of validCases) {+ expect(parser.isSupported(pythonCode)).toBe(true);+ }+ });++ it("should return false for unsupported formats", () => {+ const invalidCases = [+ 'print("Hello, World!")',+ "mo.sql()", // No assignment+ 'mo.sql("")', // No assignment+ "_df = mo.sql()", // Empty call+ 'df := mo.sql("")', // Wrong assignment operator+ // Multiple SQL calls+ 'df = mo.sql("""SELECT 1""")\ndf2 = mo.sql("""SELECT 2""")',+ ];++ for (const pythonCode of invalidCases) {+ expect(parser.isSupported(pythonCode)).toBe(false);+ }+ });++ it("should return true for empty string", () => {+ expect(parser.isSupported("")).toBe(true);+ });+ });++ describe("roundtrip", () => {+ it("should roundtrip SQL correctly", () => {+ const originalSQL = "SELECT id, name FROM users WHERE active = true";+ const pythonCode = SQLParser.fromQuery(originalSQL);+ const { code, metadata } = parser.transformIn(pythonCode);+ expect(code).toBe(originalSQL);++ const { code: backToPython } = parser.transformOut(code, metadata);+ const { code: roundtripped } = parser.transformIn(backToPython);+ expect(roundtripped).toBe(originalSQL);+ });+ });+});diff --git a/packages/smart-cells/src/index.ts b/packages/smart-cells/src/index.tsnew file mode 100644index 0000000000..2573377841--- /dev/null+++ b/packages/smart-cells/src/index.ts@@ -0,0 +1,13 @@+/* Copyright 2024 Marimo. All rights reserved. */++export type { MarkdownMetadata } from "./parsers/markdown-parser.js";+export { MarkdownParser } from "./parsers/markdown-parser.js";+export { PythonParser } from "./parsers/python-parser.js";+export type { SQLMetadata } from "./parsers/sql-parser.js";+export { SQLParser } from "./parsers/sql-parser.js";+export type {+ FormatResult,+ LanguageParser,+ ParseResult,+ QuotePrefixKind,+} from "./types.js";diff --git a/packages/smart-cells/src/parsers/index.ts b/packages/smart-cells/src/parsers/index.tsnew file mode 100644index 0000000000..0eedb246de--- /dev/null+++ b/packages/smart-cells/src/parsers/index.ts@@ -0,0 +1,5 @@+/* Copyright 2024 Marimo. All rights reserved. */++export * from "./markdown-parser.js";+export * from "./python-parser.js";+export * from "./sql-parser.js";diff --git a/packages/smart-cells/src/parsers/markdown-parser.ts b/packages/smart-cells/src/parsers/markdown-parser.tsnew file mode 100644index 0000000000..8be438eb2d--- /dev/null+++ b/packages/smart-cells/src/parsers/markdown-parser.ts@@ -0,0 +1,196 @@+/* Copyright 2024 Marimo. All rights reserved. */++import { pythonLanguage } from "@codemirror/lang-python";+import dedent from "string-dedent";+import type {+ FormatResult,+ LanguageParser,+ ParseResult,+ QuotePrefixKind,+} from "../types.js";+import { QUOTE_PREFIX_KINDS } from "../types.js";+import { splitQuotePrefix, unescapeQuotes } from "../utils/index.js";++export interface MarkdownMetadata {+ quotePrefix: QuotePrefixKind;+}++const quoteKinds = [+ ['"""', '"""'],+ ["'''", "'''"],+ ['"', '"'],+ ["'", "'"],+] as const;++// Explode into all combinations of prefixes and quote types+const pairs = QUOTE_PREFIX_KINDS.flatMap((prefix) =>+ quoteKinds.map(([start, end]) => [prefix + start, end] as const),+);++const regexes = pairs.map(+ ([start, end]) =>+ [+ start,+ new RegExp(+ `^mo\\.md\\(\\s*${escapeRegex(start)}(.*)${escapeRegex(end)}\\s*\\)$`,+ "s",+ ),+ ] as const,+);++function escapeRegex(str: string): string {+ return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");+}++/**+ * Parser for marimo Markdown cells (mo.md()).+ *+ * Converts between Python code like `mo.md(r"""# Hello""")` and+ * plain Markdown like `# Hello`.+ */+export class MarkdownParser implements LanguageParser<MarkdownMetadata> {+ readonly type = "markdown";+ readonly defaultCode = 'mo.md(r"""\n""")';+ readonly defaultMetadata: MarkdownMetadata = {+ quotePrefix: "r",+ };++ /**+ * Create a markdown cell from markdown content.+ */+ static fromMarkdown(markdown: string): string {+ return `mo.md(r"""\n${markdown}\n""")`;+ }++ transformIn(pythonCode: string): ParseResult<MarkdownMetadata> {+ pythonCode = pythonCode.trim();++ const metadata: MarkdownMetadata = { ...this.defaultMetadata };++ // Empty string+ if (pythonCode === "") {+ return { code: "", offset: 0, metadata };+ }++ // Try to match against all known patterns+ for (const [start, regex] of regexes) {+ const match = pythonCode.match(regex);+ if (match) {+ const innerCode = match[1];+ const [quotePrefix, quoteType] = splitQuotePrefix(start);+ metadata.quotePrefix = quotePrefix;+ const unescapedCode = unescapeQuotes(innerCode, quoteType);++ const offset = pythonCode.indexOf(innerCode);+ // string-dedent expects the first and last line to be empty / contain only whitespace,+ // so we pad with \n+ return {+ code: dedent(`\n${unescapedCode}\n`).trim(),+ offset,+ metadata,+ };+ }+ }++ // No match - return original code+ return { code: pythonCode, offset: 0, metadata };+ }++ transformOut(code: string, metadata: MarkdownMetadata): FormatResult {+ // NB. Must be kept consistent with marimo/_convert/utils.py::markdown_to_marimo++ // Empty string+ if (code === "") {+ // Need at least a space, otherwise the output will be 6 quotes+ code = " ";+ }++ const { quotePrefix } = metadata;++ // We always transform back with triple quotes, as to avoid needing to+ // escape single quotes.+ const escapedCode = code.replaceAll('"""', String.raw`\"""`);++ // If it's one line and not bounded by quotes, write it as single line+ const isOneLine = !code.includes("\n");+ const boundedByQuote = code.startsWith('"') || code.endsWith('"');+ if (isOneLine && !boundedByQuote) {+ const start = `mo.md(${quotePrefix}"""`;+ const end = `""")`;+ return { code: start + escapedCode + end, offset: start.length };+ }++ // Multiline code+ const start = `mo.md(\n ${quotePrefix}"""\n`;+ const end = `\n"""\n)`;+ return { code: start + escapedCode + end, offset: start.length + 1 };+ }++ isSupported(pythonCode: string): boolean {+ pythonCode = pythonCode.trim();++ // Empty strings are supported+ if (pythonCode === "") {+ return true;+ }++ // Must start with mo.md(+ if (!pythonCode.startsWith("mo.md(")) {+ return false;+ }++ // Empty function calls are supported+ if (pythonCode === "mo.md()") {+ return true;+ }++ // Parse the code using Lezer and check for the exact match of mo.md() signature+ const tree = pythonLanguage.parser.parse(pythonCode);++ // This is the exact match of mo.md() signature+ const enterOrder: Array<{ match: string | RegExp; stop?: boolean }> = [+ { match: "Script" },+ { match: "ExpressionStatement" },+ { match: "CallExpression" },+ { match: "MemberExpression" },+ { match: "VariableName" },+ { match: "." },+ { match: "PropertyName" },+ { match: "ArgList" },+ { match: "(" },+ { match: /String|FormatString/, stop: true },+ { match: ")" },+ ];++ let isValid = true;++ // Parse the code using Lezer to check for multiple function calls and string content+ tree.iterate({+ enter: (node) => {+ const current = enterOrder.shift();+ if (current === undefined) {+ // If our list is empty, but we are still going+ // then this is not a valid call+ isValid = false;+ return false;+ }++ const match = current.match;++ if (typeof match === "string") {+ isValid = isValid && match === node.name;+ return isValid && !current.stop;+ }++ if (!match.test(node.name)) {+ isValid = false;+ return isValid && !current.stop;+ }++ return isValid && !current.stop;+ },+ });++ return isValid;+ }+}diff --git a/packages/smart-cells/src/parsers/python-parser.ts b/packages/smart-cells/src/parsers/python-parser.tsnew file mode 100644index 0000000000..686e8a2027--- /dev/null+++ b/packages/smart-cells/src/parsers/python-parser.ts@@ -0,0 +1,26 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { FormatResult, LanguageParser, ParseResult } from "../types.js";++/**+ * Parser for Python cells (identity transformation).+ *+ * This parser simply passes through Python code unchanged.+ */+export class PythonParser implements LanguageParser<Record<string, never>> {+ readonly type = "python";+ readonly defaultCode = "";+ readonly defaultMetadata = {};++ transformIn(code: string): ParseResult<Record<string, never>> {+ return { code, offset: 0, metadata: {} };+ }++ transformOut(code: string, _metadata: Record<string, never>): FormatResult {+ return { code, offset: 0 };+ }++ isSupported(_code: string): boolean {+ return true;+ }+}diff --git a/packages/smart-cells/src/parsers/sql-parser.ts b/packages/smart-cells/src/parsers/sql-parser.tsnew file mode 100644index 0000000000..1d11b6eca0--- /dev/null+++ b/packages/smart-cells/src/parsers/sql-parser.ts@@ -0,0 +1,292 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { SyntaxNode, TreeCursor } from "@lezer/common";+import dedent from "string-dedent";+import type {+ FormatResult,+ LanguageParser,+ ParseResult,+ QuotePrefixKind,+} from "../types.js";+import {+ getPrefixLength,+ getStringContent,+ parseArgsKwargs,+ parsePythonAST,+ safeDedent,+} from "../utils/index.js";++export interface SQLMetadata {+ dataframeName: string;+ quotePrefix: QuotePrefixKind;+ commentLines: string[];+ showOutput: boolean;+ engine: string;+}++const DEFAULT_ENGINE = "__marimo_duckdb";++interface SQLParseInfo {+ dfName: string;+ sqlString: string;+ engine: string | undefined;+ output: boolean | undefined;+ startPosition: number;+}++/**+ * Parser for marimo SQL cells (mo.sql()).+ *+ * Converts between Python code like `_df = mo.sql(f"""SELECT * FROM users""")` and+ * plain SQL like `SELECT * FROM users`.+ */+export class SQLParser implements LanguageParser<SQLMetadata> {+ readonly type = "sql";++ readonly defaultMetadata: SQLMetadata = {+ dataframeName: "_df",+ quotePrefix: "f",+ commentLines: [],+ showOutput: true,+ engine: DEFAULT_ENGINE,+ };++ get defaultCode(): string {+ return `_df = mo.sql(f"""SELECT * FROM """)`;+ }++ /**+ * Create a SQL cell from a SQL query.+ */+ static fromQuery(query: string): string {+ return `_df = mo.sql(f"""${query.trim()}""")`;+ }++ transformIn(pythonCode: string): ParseResult<SQLMetadata> {+ pythonCode = pythonCode.trim();++ // Default metadata+ const metadata: SQLMetadata = {+ ...this.defaultMetadata,+ commentLines: this.extractCommentLines(pythonCode),+ };++ if (!this.isSupported(pythonCode)) {+ // Just return the original code if not supported+ return { code: pythonCode, offset: 0, metadata };+ }++ // Handle empty strings+ if (pythonCode === "") {+ return { code: "", offset: 0, metadata };+ }++ const sqlStatement = parseSQLStatement(pythonCode);+ if (sqlStatement) {+ metadata.dataframeName = sqlStatement.dfName;+ metadata.showOutput = sqlStatement.output ?? true;+ metadata.engine = sqlStatement.engine ?? DEFAULT_ENGINE;++ return {+ code: dedent(`\n${sqlStatement.sqlString}\n`).trim(),+ offset: sqlStatement.startPosition,+ metadata,+ };+ }++ return { code: pythonCode, offset: 0, metadata };+ }++ transformOut(code: string, metadata: SQLMetadata): FormatResult {+ const { quotePrefix, commentLines, showOutput, engine, dataframeName } =+ metadata;++ const start = `${dataframeName} = mo.sql(\n ${quotePrefix}"""\n`;+ const escapedCode = code.replaceAll('"""', String.raw`\"""`);++ const showOutputParam = showOutput ? "" : ",\n output=False";+ const engineParam =+ engine === DEFAULT_ENGINE ? "" : `,\n engine=${engine}`;+ const end = `\n """${showOutputParam}${engineParam}\n)`;++ return {+ code:+ [...commentLines, start].join("\n") + indentOneTab(escapedCode) + end,+ offset: start.length + 1,+ };+ }++ isSupported(pythonCode: string): boolean {+ if (pythonCode.trim() === "") {+ return true;+ }++ // Has at least one `mo.sql` call+ if (!pythonCode.includes("mo.sql")) {+ return false;+ }++ // Does not have 2 `mo.sql` calls+ if (pythonCode.split("mo.sql").length > 2) {+ return false;+ }++ return parseSQLStatement(pythonCode) !== null;+ }++ private extractCommentLines(pythonCode: string): string[] {+ const lines = pythonCode.split("\n");+ const commentLines: string[] = [];+ for (const line of lines) {+ if (line?.startsWith("#")) {+ commentLines.push(line);+ } else {+ break;+ }+ }+ return commentLines;+ }+}++/**+ * Find an assignment node that is preceded only by comments.+ */+function findAssignment(cursor: TreeCursor): SyntaxNode | null {+ do {+ if (cursor.name === "AssignStatement") {+ return cursor.node;+ }++ if (cursor.name !== "Comment") {+ return null;+ }+ } while (cursor.next());+ return null;+}++/**+ * Parse a SQL statement from a Python code string.+ *+ * @param code - The Python code string to parse+ * @returns The parsed SQL statement or null if parsing fails+ */+function parseSQLStatement(code: string): SQLParseInfo | null {+ try {+ const tree = parsePythonAST(code);+ const cursor = tree.cursor();++ // Trees start with a Script node+ if (cursor.name === "Script") {+ cursor.next();+ }+ const assignStmt = findAssignment(cursor);+ if (!assignStmt) {+ return null;+ }++ // Code after the assignment statement is not allowed+ if (code.slice(assignStmt.to).trim().length > 0) {+ return null;+ }++ let dfName: string | null = null;+ let sqlString: string | null = null;+ let engine: string | undefined;+ let output: boolean | undefined;+ let startPosition = 0;++ // Parse the assignment+ const assignCursor = assignStmt.cursor();+ assignCursor.firstChild(); // Move to first child of assignment++ // First child should be the variable name+ if (assignCursor.name === "VariableName") {+ dfName = code.slice(assignCursor.from, assignCursor.to);+ }++ if (!dfName) {+ return null;+ }++ // Move to the expression part (after the =)+ while (assignCursor.next()) {+ if (assignCursor.name === "CallExpression") {+ // Check if it's mo.sql call+ const callCursor = assignCursor.node.cursor();+ let isMoSql = false;++ callCursor.firstChild(); // Move to first child of call+ if (callCursor.name === "MemberExpression") {+ const memberText = code.slice(callCursor.from, callCursor.to);+ isMoSql = memberText === "mo.sql";+ }++ if (!isMoSql) {+ return null;+ }++ // Move to arguments+ while (callCursor.next()) {+ if (callCursor.name === "ArgList") {+ const argListCursor = callCursor.node.cursor();++ const { args, kwargs } = parseArgsKwargs(argListCursor, code);++ // Parse positional args (SQL query)+ if (args.length === 1) {+ sqlString = getStringContent(args[0], code);+ startPosition =+ args[0].from ++ getPrefixLength(code.slice(args[0].from, args[0].to));+ }++ // Parse kwargs (engine and output)+ for (const { key, value } of kwargs) {+ switch (key) {+ case "engine":+ engine = value;+ break;+ case "output":+ output = value === "True";+ break;+ }+ }++ // Check if sql string is empty+ if (sqlString === "") {+ return { dfName, sqlString: "", engine, output, startPosition };+ }++ break;+ }+ }+ }+ }++ if (!dfName || !sqlString) {+ return null;+ }++ return {+ dfName,+ sqlString: safeDedent(sqlString),+ engine,+ output,+ startPosition,+ };+ } catch (error) {+ // biome-ignore lint/suspicious/noConsole: warning ok+ console.warn("Failed to parse SQL statement", error);+ return null;+ }+}++/**+ * Indent code by one tab (4 spaces).+ */+function indentOneTab(code: string): string {+ return code+ .split("\n")+ .map((line) => (line?.trim() ? ` ${line}` : line))+ .join("\n");+}diff --git a/packages/smart-cells/src/types.ts b/packages/smart-cells/src/types.tsnew file mode 100644index 0000000000..1063d392c3--- /dev/null+++ b/packages/smart-cells/src/types.ts@@ -0,0 +1,76 @@+/* Copyright 2024 Marimo. All rights reserved. */++/**+ * Result of parsing Python code into a target language.+ */+export interface ParseResult<TMetadata = Record<string, unknown>> {+ /** The extracted code in the target language */+ code: string;+ /** Character offset from the start of the Python code where the extracted code begins */+ offset: number;+ /** Language-specific metadata extracted during parsing */+ metadata: TMetadata;+}++/**+ * Result of formatting target language code back into Python.+ */+export interface FormatResult {+ /** The formatted Python code */+ code: string;+ /** Character offset where the target language code begins (for cursor positioning) */+ offset: number;+}++/**+ * A language parser that can transform code between Python and a target language.+ * This is a stateless, framework-agnostic interface with no React or CodeMirror dependencies.+ *+ * @template TMetadata - The metadata type specific to this language+ */+export interface LanguageParser<TMetadata = Record<string, unknown>> {+ /** Unique identifier for this language parser */+ readonly type: string;++ /** Default Python code for this language */+ readonly defaultCode: string;++ /** Default metadata for this language */+ readonly defaultMetadata: Readonly<TMetadata>;++ /**+ * Transform Python code into the target language.+ *+ * @param pythonCode - The Python code to parse+ * @returns The extracted code, offset, and metadata+ */+ transformIn(pythonCode: string): ParseResult<TMetadata>;++ /**+ * Transform target language code back into Python.+ *+ * @param code - The target language code+ * @param metadata - The metadata from a previous transformIn call+ * @returns The formatted Python code and offset+ */+ transformOut(code: string, metadata: TMetadata): FormatResult;++ /**+ * Check if the given Python code is supported by this parser.+ *+ * @param pythonCode - The Python code to check+ * @returns true if this parser can handle the code+ */+ isSupported(pythonCode: string): boolean;+}++/**+ * Quote prefix types supported in Python strings.+ */+export const QUOTE_PREFIX_KINDS = ["", "f", "r", "fr", "rf"] as const;+export type QuotePrefixKind = (typeof QUOTE_PREFIX_KINDS)[number];++/**+ * Quote type (single, double, triple).+ */+export type QuoteType = '"' | "'" | '"""' | "'''";diff --git a/packages/smart-cells/src/utils/dedent.ts b/packages/smart-cells/src/utils/dedent.tsnew file mode 100644index 0000000000..c55798ab4d--- /dev/null+++ b/packages/smart-cells/src/utils/dedent.ts@@ -0,0 +1,22 @@+/* Copyright 2024 Marimo. All rights reserved. */++import dedent from "string-dedent";++/**+ * Safely dedent a string, handling cases where dedent might fail.+ *+ * The dedent library expects the first and last lines to be empty or contain+ * only whitespace, so we pad with newlines before dedenting.+ *+ * @param code - The code to dedent+ * @returns The dedented code, or the original code if dedenting fails+ */+export function safeDedent(code: string): string {+ try {+ // Dedent expects the first and last line to be empty / contain only whitespace,+ // so we pad with \n+ return dedent(`\n${code}\n`).trim();+ } catch {+ return code;+ }+}diff --git a/packages/smart-cells/src/utils/index.ts b/packages/smart-cells/src/utils/index.tsnew file mode 100644index 0000000000..971d48a4f2--- /dev/null+++ b/packages/smart-cells/src/utils/index.ts@@ -0,0 +1,6 @@+/* Copyright 2024 Marimo. All rights reserved. */++export * from "./dedent.js";+export * from "./python-ast.js";+export * from "./quote-parser.js";+export * from "./string-escaper.js";diff --git a/packages/smart-cells/src/utils/python-ast.ts b/packages/smart-cells/src/utils/python-ast.tsnew file mode 100644index 0000000000..0ec3a9704e--- /dev/null+++ b/packages/smart-cells/src/utils/python-ast.ts@@ -0,0 +1,208 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { SyntaxNode, TreeCursor } from "@lezer/common";+import { parser } from "@lezer/python";++/**+ * Parse Python code into an AST.+ */+export function parsePythonAST(code: string) {+ return parser.parse(code);+}++/**+ * Parsed function argument lists (positional and keyword arguments).+ */+export interface ParsedArgs {+ args: SyntaxNode[];+ kwargs: Array<{ key: string; value: string }>;+}++const SYNTAX_TOKENS = new Set(["(", "[", "{", ",", ")", "]", "}"]);++/**+ * Parse an ArgList node into positional and keyword arguments.+ *+ * @param argCursor - A cursor positioned at an ArgList node+ * @param code - The source code+ * @returns Parsed positional and keyword arguments+ */+export function parseArgsKwargs(+ argCursor: TreeCursor,+ code: string,+): ParsedArgs {+ if (argCursor.name !== "ArgList") {+ // biome-ignore lint/suspicious/noConsole: <explanation>+ console.warn(`Not an ArgList, name: ${argCursor.name}`);+ return { args: [], kwargs: [] };+ }++ return {+ args: parseArgs(argCursor),+ kwargs: parseKwargs(argCursor, code),+ };+}++function parseArgs(argCursor: TreeCursor): SyntaxNode[] {+ // Move to first argument+ argCursor.firstChild();++ const args: SyntaxNode[] = [];+ let name = argCursor.name;++ do {+ name = argCursor.name;+ if (SYNTAX_TOKENS.has(name)) {+ continue;+ }++ if (name === "VariableName") {+ // Stop if we hit a kwarg (next token is AssignOp)+ const peek = argCursor.node.nextSibling;+ if (peek?.name === "AssignOp") {+ argCursor.prev();+ break;+ }+ }++ args.push(argCursor.node);+ } while (argCursor.nextSibling());++ return args;+}++function parseKwargs(+ argCursor: TreeCursor,+ code: string,+): Array<{ key: string; value: string }> {+ const kwargs: Array<{ key: string; value: string }> = [];+ let name = argCursor.name;++ do {+ name = argCursor.name;+ if (name === "VariableName") {+ const key = code.slice(argCursor.from, argCursor.to);++ // Check for AssignOp+ const assignNode = argCursor.node.nextSibling;+ if (!assignNode || assignNode.name !== "AssignOp") {+ continue;+ }++ // Get the value node+ const valueNode = assignNode.nextSibling;+ if (!valueNode) {+ continue;+ }++ const value = code.slice(valueNode.from, valueNode.to).trim();+ kwargs.push({ key, value });+ }+ } while (argCursor.nextSibling());++ return kwargs;+}++/**+ * Get the content of a String or FormatString node.+ *+ * @param node - The String or FormatString node+ * @param code - The source code+ * @returns The string content, or null if not a string node+ */+export function getStringContent(+ node: SyntaxNode,+ code: string,+): string | null {+ // Handle triple quoted strings+ if (node.name === "String") {+ const content = code.slice(node.from, node.to);+ // Handle r-strings with triple quotes+ if (content.startsWith('r"""') || content.startsWith("r'''")) {+ return content.slice(4, -3);+ }+ // Handle regular triple quotes+ if (content.startsWith('"""') || content.startsWith("'''")) {+ return content.slice(3, -3);+ }+ // Handle r-strings with single quotes+ if (content.startsWith('r"') || content.startsWith("r'")) {+ return content.slice(2, -1);+ }+ // Handle single quoted strings+ return content.slice(1, -1);+ }++ // Handle f-strings+ if (node.name === "FormatString") {+ const content = code.slice(node.from, node.to);+ if (content.startsWith('f"""') || content.startsWith("f'''")) {+ return content.slice(4, -3);+ }+ if (content.startsWith('rf"""') || content.startsWith("rf'''")) {+ return content.slice(5, -3);+ }+ if (content.startsWith('fr"""') || content.startsWith("fr'''")) {+ return content.slice(5, -3);+ }+ if (content.startsWith('r"""') || content.startsWith("r'''")) {+ return content.slice(4, -3);+ }+ // Single quotes+ if (content.startsWith('f"') || content.startsWith("f'")) {+ return content.slice(2, -1);+ }+ if (content.startsWith('rf"') || content.startsWith("rf'")) {+ return content.slice(3, -1);+ }+ if (content.startsWith('fr"') || content.startsWith("fr'")) {+ return content.slice(3, -1);+ }+ if (content.startsWith('r"') || content.startsWith("r'")) {+ return content.slice(2, -1);+ }+ return content.slice(2, -1);+ }++ return null;+}++/**+ * Get the length of the quote prefix in a string (e.g., 'f"""' -> 4).+ */+export function getPrefixLength(code: string): number {+ if (code === "") {+ return 0;+ }+ if (code.startsWith('rf"""') || code.startsWith("rf'''")) {+ return 5;+ }+ if (code.startsWith('fr"""') || code.startsWith("fr'''")) {+ return 5;+ }+ if (code.startsWith('f"""') || code.startsWith("f'''")) {+ return 4;+ }+ if (code.startsWith('r"""') || code.startsWith("r'''")) {+ return 4;+ }+ if (code.startsWith('"""') || code.startsWith("'''")) {+ return 3;+ }+ if (code.startsWith("rf'") || code.startsWith('rf"')) {+ return 3;+ }+ if (code.startsWith("fr'") || code.startsWith('fr"')) {+ return 3;+ }+ if (code.startsWith("f'") || code.startsWith('f"')) {+ return 2;+ }+ if (code.startsWith("r'") || code.startsWith('r"')) {+ return 2;+ }+ if (code.startsWith("'") || code.startsWith('"')) {+ return 1;+ }+ return 0;+}diff --git a/packages/smart-cells/src/utils/quote-parser.ts b/packages/smart-cells/src/utils/quote-parser.tsnew file mode 100644index 0000000000..df3e1f5d0f--- /dev/null+++ b/packages/smart-cells/src/utils/quote-parser.ts@@ -0,0 +1,40 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { QuotePrefixKind, QuoteType } from "../types.js";+import { QUOTE_PREFIX_KINDS } from "../types.js";++/**+ * Split a quote string into its prefix (f/r/fr/rf) and the quote type.+ *+ * @param quote - The quote string (e.g., 'f"""', 'r"', '"""')+ * @returns A tuple of [prefix, quoteType]+ */+export function splitQuotePrefix(quote: string): [QuotePrefixKind, QuoteType] {+ // Start with the longest prefix to avoid partial matches+ const prefixKindsByLength = [...QUOTE_PREFIX_KINDS].sort(+ (a, b) => b.length - a.length,+ );++ for (const prefix of prefixKindsByLength) {+ if (quote.startsWith(prefix)) {+ return [prefix, quote.slice(prefix.length) as QuoteType];+ }+ }++ return ["", quote as QuoteType];+}++/**+ * Check if a quote type is a triple quote.+ */+export function isTripleQuote(quoteType: QuoteType): boolean {+ return quoteType === '"""' || quoteType === "'''";+}++/**+ * Get the matching closing quote for an opening quote.+ */+export function getClosingQuote(openingQuote: string): string {+ const [, quoteType] = splitQuotePrefix(openingQuote);+ return quoteType;+}diff --git a/packages/smart-cells/src/utils/string-escaper.ts b/packages/smart-cells/src/utils/string-escaper.tsnew file mode 100644index 0000000000..eda6289e26--- /dev/null+++ b/packages/smart-cells/src/utils/string-escaper.ts@@ -0,0 +1,37 @@+/* Copyright 2024 Marimo. All rights reserved. */++import type { QuoteType } from "../types.js";++/**+ * Escape quotes in a string for safe embedding in Python code.+ *+ * @param code - The code to escape+ * @param quoteType - The quote type to escape+ * @returns The escaped code+ */+export function escapeQuotes(code: string, quoteType: QuoteType): string {+ if (quoteType === '"""') {+ return code.replaceAll('"""', String.raw`\"""`);+ }+ if (quoteType === "'''") {+ return code.replaceAll("'''", String.raw`\'\'\'`);+ }+ if (quoteType === '"') {+ return code.replaceAll('"', String.raw`\"`);+ }+ if (quoteType === "'") {+ return code.replaceAll("'", String.raw`\'`);+ }+ return code;+}++/**+ * Unescape quotes in a string extracted from Python code.+ *+ * @param code - The code with escaped quotes+ * @param quoteType - The quote type that was escaped+ * @returns The unescaped code+ */+export function unescapeQuotes(code: string, quoteType: QuoteType): string {+ return code.replaceAll(`\\${quoteType}`, quoteType);+}diff --git a/packages/smart-cells/tsconfig.json b/packages/smart-cells/tsconfig.jsonnew file mode 100644index 0000000000..4024c4942e--- /dev/null+++ b/packages/smart-cells/tsconfig.json@@ -0,0 +1,10 @@+{+ "extends": "@marimo-team/tsconfig/library.json",+ "compilerOptions": {+ "lib": ["ESNext"],+ "outDir": "./dist",+ "rootDir": "./src",+ "allowImportingTsExtensions": true,+ "noEmit": true+ }+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 439aa50e2d..fd4df2ebf0 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -153,6 +153,9 @@ importers:'@marimo-team/react-slotz':specifier: ^0.2.0version: 0.2.0([email protected])+ '@marimo-team/smart-cells':+ specifier: workspace:*+ version: link:../packages/smart-cells'@mui/material':specifier: ^6.5.0version: 6.5.0(@emotion/[email protected](@types/[email protected])([email protected]))(@emotion/[email protected](@emotion/[email protected](@types/[email protected])([email protected]))(@types/[email protected])([email protected]))(@types/[email protected])([email protected]([email protected]))([email protected])@@ -718,6 +721,34 @@ importers:specifier: ^7.8.0version: 7.8.0([email protected])+ packages/smart-cells:+ dependencies:+ '@codemirror/lang-python':+ specifier: ^6.2.1+ version: 6.2.1+ '@lezer/common':+ specifier: ^1.2.1+ version: 1.2.3+ '@lezer/python':+ specifier: ^1.1.14+ version: 1.1.18+ string-dedent:+ specifier: ^3.0.1+ version: 3.0.2+ devDependencies:+ '@marimo-team/tsconfig':+ specifier: workspace:*+ version: link:../ts-config+ '@types/node':+ specifier: ^20.0.0+ version: 20.19.21+ typescript:+ specifier: ^5.3.3+ version: 5.9.2+ vitest:+ specifier: ^3.2.4+ version: 3.2.4(@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected]))([email protected])([email protected])+packages/ts-config: {}packages:@@ -4011,6 +4042,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}+ '@types/[email protected]':+ resolution: {integrity: sha512-CsGG2P3I5y48RPMfprQGfy4JPRZ6csfC3ltBZSRItG3ngggmNY/qs2uZKp4p9VbrpqNNSMzUZNFZKzgOGnd/VA==}+'@types/[email protected]':resolution: {integrity: sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g==}@@ -9530,6 +9564,9 @@ packages:resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}+resolution: {integrity: sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag==}@@ -11273,6 +11310,14 @@ snapshots:dependencies:img-comparison-slider: 8.0.6+ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/core': 10.1.14(@types/[email protected])+ '@inquirer/type': 3.0.7(@types/[email protected])+ optionalDependencies:+ '@types/node': 20.19.21+ optional: true+'@inquirer/[email protected](@types/[email protected])':dependencies:'@inquirer/core': 10.1.14(@types/[email protected])@@ -11280,6 +11325,20 @@ snapshots:optionalDependencies:'@types/node': 24.3.1+ '@inquirer/[email protected](@types/[email protected])':+ dependencies:+ '@inquirer/figures': 1.0.12+ '@inquirer/type': 3.0.7(@types/[email protected])+ ansi-escapes: 4.3.2+ cli-width: 4.1.0+ mute-stream: 2.0.0+ signal-exit: 4.1.0+ wrap-ansi: 6.2.0+ yoctocolors-cjs: 2.1.2+ optionalDependencies:+ '@types/node': 20.19.21+ optional: true+'@inquirer/[email protected](@types/[email protected])':dependencies:'@inquirer/figures': 1.0.12@@ -11295,6 +11354,11 @@ snapshots:'@inquirer/[email protected]': {}+ '@inquirer/[email protected](@types/[email protected])':+ optionalDependencies:+ '@types/node': 20.19.21+ optional: true+'@inquirer/[email protected](@types/[email protected])':optionalDependencies:'@types/node': 24.3.1@@ -14336,6 +14400,10 @@ snapshots:'@types/[email protected]': {}+ '@types/[email protected]':+ dependencies:+ undici-types: 6.21.0+'@types/[email protected]':dependencies:undici-types: 7.10.0@@ -14653,6 +14721,15 @@ snapshots:chai: 5.2.1tinyrainbow: 2.0.0+ '@vitest/[email protected]([email protected](@types/[email protected])([email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))':+ dependencies:+ '@vitest/spy': 3.2.4+ estree-walker: 3.0.3+ magic-string: 0.30.18+ optionalDependencies:+ msw: 2.11.1(@types/[email protected])([email protected])+ vite: [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])+'@vitest/[email protected]([email protected](@types/[email protected])([email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))':dependencies:'@vitest/spy': 3.2.4@@ -18457,6 +18534,32 @@ snapshots:+ dependencies:+ '@bundled-es-modules/cookie': 2.0.1+ '@bundled-es-modules/statuses': 1.0.1+ '@inquirer/confirm': 5.1.13(@types/[email protected])+ '@mswjs/interceptors': 0.39.2+ '@open-draft/deferred-promise': 2.2.0+ '@open-draft/until': 2.1.0+ '@types/cookie': 0.6.0+ '@types/statuses': 2.0.6+ graphql: 16.11.0+ headers-polyfill: 4.0.3+ is-node-process: 1.2.0+ outvariant: 1.4.3+ path-to-regexp: 6.3.0+ picocolors: 1.1.1+ strict-event-emitter: 0.5.1+ tough-cookie: 6.0.0+ type-fest: 4.41.0+ yargs: 17.7.2+ optionalDependencies:+ typescript: 5.9.2+ transitivePeerDependencies:+ - '@types/node'+ optional: true+dependencies:'@bundled-es-modules/cookie': 2.0.1@@ -19935,6 +20038,23 @@ snapshots:+ [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):+ dependencies:+ '@oxc-project/runtime': 0.92.0+ fdir: 6.5.0([email protected])+ lightningcss: 1.30.2+ picomatch: 4.0.3+ postcss: 8.5.6+ rolldown: 1.0.0-beta.43+ tinyglobby: 0.2.15+ optionalDependencies:+ '@types/node': 20.19.21+ esbuild: 0.25.9+ fsevents: 2.3.3+ jiti: 2.5.1+ terser: 5.44.0+ yaml: 2.8.1+[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:'@oxc-project/runtime': 0.92.0@@ -20987,6 +21107,8 @@ snapshots:has-symbols: 1.1.0which-boxed-primitive: 1.1.1+ [email protected]: {}+@@ -21472,6 +21594,27 @@ snapshots:'@types/unist': 3.0.3vfile-message: 4.0.2+ [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):+ dependencies:+ cac: 6.7.14+ debug: 4.4.1([email protected])+ es-module-lexer: 1.7.0+ pathe: 2.0.3+ vite: [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ transitivePeerDependencies:+ - '@types/node'+ - esbuild+ - jiti+ - less+ - sass+ - sass-embedded+ - stylus+ - sugarss+ - supports-color+ - terser+ - tsx+ - yaml+[email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]):dependencies:cac: 6.7.14@@ -21508,6 +21651,49 @@ snapshots:dependencies:vite: [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ [email protected](@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected]))([email protected])([email protected]):+ dependencies:+ '@types/chai': 5.2.2+ '@vitest/expect': 3.2.4+ '@vitest/mocker': 3.2.4([email protected](@types/[email protected])([email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected]))+ '@vitest/pretty-format': 3.2.4+ '@vitest/runner': 3.2.4+ '@vitest/snapshot': 3.2.4+ '@vitest/spy': 3.2.4+ '@vitest/utils': 3.2.4+ chai: 5.2.1+ debug: 4.4.1([email protected])+ expect-type: 1.2.2+ magic-string: 0.30.18+ pathe: 2.0.3+ picomatch: 4.0.3+ std-env: 3.9.0+ tinybench: 2.9.0+ tinyexec: 0.3.2+ tinyglobby: 0.2.15+ tinypool: 1.1.1+ tinyrainbow: 2.0.0+ vite: [email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ vite-node: 3.2.4(@types/[email protected])([email protected])([email protected])([email protected])([email protected])+ why-is-node-running: 2.3.0+ optionalDependencies:+ '@types/debug': 4.1.12+ '@types/node': 20.19.21+ jsdom: 24.1.3+ transitivePeerDependencies:+ - esbuild+ - jiti+ - less+ - msw+ - sass+ - sass-embedded+ - stylus+ - sugarss+ - supports-color+ - terser+ - tsx+ - yaml+[email protected](@types/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])([email protected](@types/[email protected])([email protected]))([email protected])([email protected]):dependencies:'@types/chai': 5.2.2