A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
add parse function in /validate endpoint for SQL (#6517)
Shahmir Varqhad88107067031
diff --git a/frontend/package.json b/frontend/package.jsonindex 70b955514b..6009c6a656 100644--- a/frontend/package.json+++ b/frontend/package.json@@ -52,7 +52,7 @@"@marimo-team/codemirror-ai": "^0.3.2","@marimo-team/codemirror-languageserver": "1.15.24","@marimo-team/codemirror-mcp": "^0.1.5",- "@marimo-team/codemirror-sql": "^0.2.1",+ "@marimo-team/codemirror-sql": "^0.2.3","@marimo-team/llm-info": "workspace:*","@marimo-team/marimo-api": "workspace:*","@marimo-team/react-slotz": "^0.2.0",diff --git a/frontend/src/components/editor/errors/sql-validation-errors.tsx b/frontend/src/components/editor/errors/sql-validation-errors.tsxindex d9e89e8fc0..eeb923eea5 100644--- a/frontend/src/components/editor/errors/sql-validation-errors.tsx+++ b/frontend/src/components/editor/errors/sql-validation-errors.tsx@@ -2,7 +2,7 @@import { AlertCircleIcon } from "lucide-react";import type { CellId } from "@/core/cells/ids";-import { useSqlValidationErrorsForCell } from "@/core/codemirror/language/languages/sql/validation-errors";+import { useSqlValidationErrorsForCell } from "@/core/codemirror/language/languages/sql/banner-validation-errors";export const SqlValidationErrorBanner = ({cellId,diff --git a/frontend/src/core/codemirror/language/__tests__/extension.test.ts b/frontend/src/core/codemirror/language/__tests__/extension.test.tsindex e6d04c1672..926989b674 100644--- a/frontend/src/core/codemirror/language/__tests__/extension.test.ts+++ b/frontend/src/core/codemirror/language/__tests__/extension.test.ts@@ -14,7 +14,7 @@ import {languageAdapterState,switchLanguage,} from "../extension";-import { exportedForTesting as sqlValidationErrorsForTesting } from "../languages/sql/validation-errors";+import { exportedForTesting as sqlValidationErrorsForTesting } from "../languages/sql/banner-validation-errors";import { languageMetadataField } from "../metadata";let view: EditorView | null = null;diff --git a/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts b/frontend/src/core/codemirror/language/__tests__/sql-validation.test.tsindex 9edd9c1e73..3d466a0e65 100644--- a/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts+++ b/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts@@ -1,7 +1,7 @@/* Copyright 2024 Marimo. All rights reserved. */import { describe, expect, it } from "vitest";-import { exportedForTesting } from "../languages/sql/validation-errors";+import { exportedForTesting } from "../languages/sql/banner-validation-errors";describe("Error Message Splitting", () => {it("should handle error message splitting correctly", () => {diff --git a/frontend/src/core/codemirror/language/languages/sql/validation-errors.ts b/frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.tssimilarity index 93%rename from frontend/src/core/codemirror/language/languages/sql/validation-errors.tsrename to frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.tsindex 052eb23f6f..2386fcba9d 100644--- a/frontend/src/core/codemirror/language/languages/sql/validation-errors.ts+++ b/frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.ts@@ -31,18 +31,20 @@ export function clearSqlValidationError(cellId: CellId) {export function setSqlValidationError({cellId,- error,+ errorMessage,dialect,}: {cellId: CellId;- error: string;+ errorMessage: string;dialect: SupportedDialects | null;}) {const sqlValidationErrors = store.get(sqlValidationErrorsAtom);const newErrors = new Map(sqlValidationErrors);const errorResult: SQLValidationError =- dialect === "DuckDB" ? handleDuckdbError(error) : splitErrorMessage(error);+ dialect === "DuckDB"+ ? handleDuckdbError(errorMessage)+ : splitErrorMessage(errorMessage);newErrors.set(cellId, errorResult);store.set(sqlValidationErrorsAtom, newErrors);diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.tsindex 3ef15ed407..fc47959dc6 100644--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts@@ -11,7 +11,9 @@ import { parser } from "@lezer/python";import {defaultSqlHoverTheme,NodeSqlParser,+ type NodeSqlParserResult,type SupportedDialects as ParserDialects,+ type SqlParseError,sqlExtension,} from "@marimo-team/codemirror-sql";import { DuckDBDialect } from "@marimo-team/codemirror-sql/dialects";@@ -28,7 +30,6 @@ import {INTERNAL_SQL_ENGINES,} from "@/core/datasets/engines";import { ValidateSQL } from "@/core/datasets/request-registry";-import type { ValidateSQLResult } from "@/core/kernel/messages";import { store } from "@/core/state/jotai";import { resolvedThemeAtom } from "@/theme/useTheme";import { Logger } from "@/utils/Logger";@@ -39,16 +40,16 @@ import { parseArgsKwargs } from "../../utils/ast";import { indentOneTab } from "../../utils/indentOneTab";import type { QuotePrefixKind } from "../../utils/quotes";import { MarkdownLanguageAdapter } from "../markdown";+import {+ clearSqlValidationError,+ setSqlValidationError,+} from "./banner-validation-errors";import {customKeywordCompletionSource,tablesCompletionSource,} from "./completion-sources";import { SCHEMA_CACHE } from "./completion-store";import { getSQLMode } from "./sql-mode";-import {- clearSqlValidationError,- setSqlValidationError,-} from "./validation-errors";const DEFAULT_DIALECT = DuckDBDialect;const DEFAULT_PARSER_DIALECT = "DuckDB";@@ -245,7 +246,7 @@ export class SQLLanguageAdapterif (this.sqlLinterEnabled) {const theme = store.get(resolvedThemeAtom);- const parser = new NodeSqlParser({+ const parser = new CustomSqlParser({getParserOptions: (state: EditorState) => {return {database: guessParserDialect(state) ?? DEFAULT_PARSER_DIALECT,@@ -280,14 +281,84 @@ export class SQLLanguageAdapter);}- if (this.sqlModeEnabled) {- extensions.push(sqlValidationExtension());- }+ // TODO: Re-enable after we optimize the endpoint+ // if (this.sqlModeEnabled) {+ // extensions.push(sqlValidationExtension());+ // }return extensions;}}+class CustomSqlParser extends NodeSqlParser {+ private validationTimeout: number | null = null;+ private readonly VALIDATION_DELAY_MS = 300; // Wait 300ms after user stops typing++ private async validateWithDelay(+ sql: string,+ engine: string,+ dialect: ParserDialects | null,+ ): Promise<SqlParseError[]> {+ // Clear any existing delay call+ if (this.validationTimeout) {+ window.clearTimeout(this.validationTimeout);+ }++ // Set up a new request to be called after the delay+ return new Promise((resolve) => {+ this.validationTimeout = window.setTimeout(async () => {+ try {+ const result = await ValidateSQL.request({+ onlyParse: true,+ engine,+ dialect,+ query: sql,+ });+ if (result.error) {+ Logger.error("Failed to validate SQL", { error: result.error });+ resolve([]);+ return;+ }+ resolve(result.parse_result?.errors ?? []);+ } catch (error) {+ Logger.error("Failed to validate SQL", { error });+ resolve([]);+ }+ }, this.VALIDATION_DELAY_MS);+ });+ }++ override async validateSql(+ sql: string,+ opts: { state: EditorState },+ ): Promise<SqlParseError[]> {+ const metadata = getSQLMetadata(opts.state);++ // Only perform custom validation for internal engines+ if (!INTERNAL_SQL_ENGINES.has(metadata.engine)) {+ return super.validateSql(sql, opts);+ }++ const dialect = guessParserDialect(opts.state);+ return this.validateWithDelay(sql, metadata.engine, dialect);+ }++ override async parse(+ sql: string,+ opts: { state: EditorState },+ ): Promise<NodeSqlParserResult> {+ const metadata = getSQLMetadata(opts.state);+ const engine = metadata.engine;++ // For now, always return success for DuckDB+ if (engine === DUCKDB_ENGINE) {+ return { success: true, errors: [] };+ }++ return super.parse(sql, opts);+ }+}+/*** Update the SQL dialect in the editor view.*/@@ -568,11 +639,18 @@ function safeDedent(code: string): string {}}-function sqlValidationExtension(): Extension {- let debounceTimeout: NodeJS.Timeout | null = null;+const SQL_VALIDATION_DEBOUNCE_MS = 300;++// @ts-expect-error: TODO: Re-enable after we optimize the endpoint+function _sqlValidationExtension(): Extension {+ let debounceTimeout: number | undefined;let lastValidationRequest: string | null = null;return EditorView.updateListener.of((update) => {+ if (!update.docChanged) {+ return;+ }+const sqlMode = getSQLMode();if (sqlMode !== "validate") {return;@@ -580,12 +658,8 @@ function sqlValidationExtension(): Extension {const metadata = getSQLMetadata(update.state);const connectionName = metadata.engine;+ // Currently only internal engines are supportedif (!INTERNAL_SQL_ENGINES.has(connectionName)) {- // Currently only internal engines are supported- return;- }-- if (!update.docChanged) {return;}@@ -594,11 +668,11 @@ function sqlValidationExtension(): Extension {// Clear existing timeoutif (debounceTimeout) {- clearTimeout(debounceTimeout);+ window.clearTimeout(debounceTimeout);}// Debounce the validation call- debounceTimeout = setTimeout(async () => {+ debounceTimeout = window.setTimeout(async () => {// Skip if content hasn't changed since last validationif (lastValidationRequest === sqlContent) {return;@@ -613,20 +687,32 @@ function sqlValidationExtension(): Extension {}try {- const result: ValidateSQLResult = await ValidateSQL.request({+ const result = await ValidateSQL.request({+ onlyParse: false,engine: connectionName,query: sqlContent,});if (result.error) {+ Logger.error("Failed to validate SQL", { error: result.error });+ return;+ }++ const validateResult = result.validate_result;++ if (validateResult?.error_message) {const dialect = connectionNameToParserDialect(connectionName);- setSqlValidationError({ cellId, error: result.error, dialect });+ setSqlValidationError({+ cellId,+ errorMessage: validateResult.error_message,+ dialect,+ });} else {clearSqlValidationError(cellId);}} catch (error) {Logger.warn("Failed to validate SQL", { error });}- }, 300);+ }, SQL_VALIDATION_DEBOUNCE_MS);});}diff --git a/marimo/_config/config.py b/marimo/_config/config.pyindex 79ba360ccc..aa52c7e039 100644--- a/marimo/_config/config.py+++ b/marimo/_config/config.py@@ -508,7 +508,7 @@ class ExperimentalConfig(TypedDict, total=False):performant_table_charts: boolmcp_docs: boolsql_linter: bool- sql_mode: bool+ sql_mode: bool # Not exposed for now# Internal featurescache: CacheConfigdiff --git a/marimo/_messaging/ops.py b/marimo/_messaging/ops.pyindex 4da2e836f3..217378f137 100644--- a/marimo/_messaging/ops.py+++ b/marimo/_messaging/ops.py@@ -53,6 +53,7 @@ from marimo._runtime.context.types import ContextNotInitializedErrorfrom marimo._runtime.context.utils import get_modefrom marimo._runtime.layout.layout import LayoutConfigfrom marimo._secrets.models import SecretKeysWithProvider+from marimo._sql.parse import SqlCatalogCheckResult, SqlParseResultfrom marimo._types.ids import CellId_t, RequestId, WidgetModelIdfrom marimo._utils.platform import is_pyodide, is_windows@@ -629,7 +630,8 @@ class DataSourceConnections(Op, tag="data-source-connections"):class ValidateSQLResult(Op, tag="validate-sql-result"):name: ClassVar[str] = "validate-sql-result"request_id: RequestId- result: Optional[Any] = None+ parse_result: Optional[SqlParseResult] = None+ validate_result: Optional[SqlCatalogCheckResult] = Noneerror: Optional[str] = Nonediff --git a/marimo/_runtime/requests.py b/marimo/_runtime/requests.pyindex 6b6ca0f0af..a2bc79148a 100644--- a/marimo/_runtime/requests.py+++ b/marimo/_runtime/requests.py@@ -339,10 +339,22 @@ class PreviewDataSourceConnectionRequest(msgspec.Struct, rename="camel"):class ValidateSQLRequest(msgspec.Struct, rename="camel"):- """Validate an SQL query"""+ """Validate an SQL query against the engine"""request_id: RequestId- engine: str+ query: str+ # Whether to only parse the query or validate against the database+ # Parsing is done without a DB connection and uses dialect, whereas validation requires a connection+ only_parse: bool+ engine: Optional[str] = None+ dialect: Optional[str] = None+++class ParseSQLRequest(msgspec.Struct, rename="camel"):+ """Parse an SQL query for linting"""++ request_id: RequestId+ dialect: strquery: strdiff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex 843fb6f193..090a5f4047 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -179,11 +179,16 @@ from marimo._secrets.secrets import get_secret_keysfrom marimo._server.model import SessionModefrom marimo._server.types import QueueTypefrom marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEngine-from marimo._sql.engines.types import EngineCatalog, QueryEngine+from marimo._sql.engines.types import (+ EngineCatalog,+ QueryEngine,+ SQLConnectionType,+)from marimo._sql.get_engines import (engine_to_data_source_connection,get_engines_from_variables,)+from marimo._sql.parse import SqlCatalogCheckResult, parse_sqlfrom marimo._tracer import kernel_tracerfrom marimo._types.ids import CellId_t, UIElementId, VariableNamefrom marimo._types.lifespan import Lifespan@@ -2252,25 +2257,28 @@ class Kernel:await self.request_handler.handle(request)LOGGER.debug("Handled control request: %s", request)- def get_engine_catalog(+ def get_sql_connection(self, variable_name: str- ) -> tuple[Optional[EngineCatalog[Any]], Optional[str]]:- """Fetch the catalog-capable engine associated with the given variable name.-- Returns the engine if it supports catalog operations, or an error message if not."""+ ) -> tuple[Optional[SQLConnectionType], Optional[str]]:+ """+ Fetch the SQL connection associated with the given variable name.+ Returns the connection if it supports query or catalog operations, or an error message if not.+ """variable_name = cast(VariableName, variable_name)try:- # Should we find the existing engine instead?engine_val = self.globals.get(variable_name)engines = get_engines_from_variables([(variable_name, engine_val)])if engines is None or len(engines) == 0:return None, "Engine not found"engine = engines[0][1]- if isinstance(engine, EngineCatalog):+ if isinstance(engine, (QueryEngine, EngineCatalog)):return engine, Noneelse:- return None, "Connection does not support catalog operations"+ return (+ None,+ "Connection does not support query or catalog operations",+ )except Exception as e:LOGGER.warning("Failed to get engine %s", variable_name, exc_info=e@@ -2282,6 +2290,21 @@ class DatasetCallbacks:def __init__(self, kernel: Kernel):self._kernel = kernel+ def get_engine_catalog(+ self, variable_name: str+ ) -> tuple[Optional[EngineCatalog[Any]], Optional[str]]:+ """Get engines that support catalog operations.+ Returns an error if the connection does not support catalog operations."""+ variable_name = cast(VariableName, variable_name)+ connection, error = self._kernel.get_sql_connection(variable_name)+ if error is not None or connection is None:+ return None, error++ if isinstance(connection, EngineCatalog):+ return connection, None+ else:+ return None, "Connection does not support catalog operations"+@kernel_tracer.start_as_current_span("preview_dataset_column")async def preview_dataset_column(self, request: PreviewDatasetColumnRequest@@ -2367,7 +2390,7 @@ class DatasetCallbacks:schema_name = request.schematable_name = request.table_name- engine, error = self._kernel.get_engine_catalog(variable_name)+ engine, error = self.get_engine_catalog(variable_name)if error is not None or engine is None:SQLTablePreview(request_id=request.request_id, table=None, error=error@@ -2412,7 +2435,7 @@ class DatasetCallbacks:database_name = request.databaseschema_name = request.schema- engine, error = self._kernel.get_engine_catalog(variable_name)+ engine, error = self.get_engine_catalog(variable_name)if error is not None or engine is None:SQLTableListPreview(request_id=request.request_id, tables=[], error=error@@ -2444,7 +2467,7 @@ class DatasetCallbacks:) -> None:"""Broadcasts a datasource connection for a given engine"""variable_name = cast(VariableName, request.engine)- engine, error = self._kernel.get_engine_catalog(variable_name)+ engine, error = self.get_engine_catalog(variable_name)if error is not None or engine is None:LOGGER.error("Failed to get engine %s", variable_name)return@@ -2467,34 +2490,82 @@ class SqlCallbacks:@kernel_tracer.start_as_current_span("validate_sql_query")async def validate_sql(self, request: ValidateSQLRequest) -> None:- """Validate an SQL query"""- # TODO: Place request in queue+ """Validate an SQL query++ This will validate:+ - the syntax (parsing)+ - the catalog (table and column names)+ """+ request_id = request.request_id++ if request.only_parse:+ if request.dialect is None:+ ValidateSQLResult(+ request_id=request_id,+ error="Dialect is required when only parsing",+ ).broadcast()+ return++ # Just parse the query (no DB connection required)+ parse_result, error = parse_sql(request.query, request.dialect)+ ValidateSQLResult(+ request_id=request_id,+ parse_result=parse_result,+ error=error,+ ).broadcast()+ return++ # Validate against the database+ # This can be cheap for in-memory engines (duckdb, sqlite)+ # But potentially expensive and requires an active connection for remote engines+ # For failed connections, we should not raise an error++ if request.engine is None:+ ValidateSQLResult(+ request_id=request_id,+ error="Engine is required for validating catalog",+ ).broadcast()+ return+variable_name = cast(VariableName, request.engine)- engine: Optional[EngineCatalog[Any]] = None+ engine: Optional[SQLConnectionType] = Noneif variable_name == INTERNAL_DUCKDB_ENGINE:engine = DuckDBEngine(connection=None)error = Noneelse:- engine, error = self._kernel.get_engine_catalog(variable_name)+ engine, error = self._kernel.get_sql_connection(variable_name)if error is not None or engine is None:- LOGGER.error("Failed to get engine %s", variable_name)ValidateSQLResult(- request_id=request.request_id,- result=None,- error="Engine not found",+ request_id=request_id,+ error="Failed to get engine " + variable_name,).broadcast()return- if isinstance(engine, QueryEngine):- result, error = engine.execute_in_explain_mode(request.query) # type: ignore- else:- error = "Engine does not support explain mode"+ # Get the parse error for linting+ parse_result, parse_error = parse_sql(request.query, engine.dialect)+ if parse_error is not None:+ # We don't want to fail the validation if there is a parse error+ LOGGER.debug("Parse error: %s", parse_error)+ if not isinstance(engine, QueryEngine):+ ValidateSQLResult(+ request_id=request_id,+ error=f"Engine {variable_name} does not support catalog validation.",+ parse_result=parse_result,+ ).broadcast()+ return++ _, error_message = engine.execute_in_explain_mode(request.query) # type: ignore+ validate_result = SqlCatalogCheckResult(+ success=True if error_message is None else False,+ error_message=error_message,+ )ValidateSQLResult(- request_id=request.request_id,- result=None, # We aren't using the result yet- error=error,+ request_id=request_id,+ validate_result=validate_result,+ parse_result=parse_result,+ error=None,).broadcast()diff --git a/marimo/_server/api/endpoints/sql.py b/marimo/_server/api/endpoints/sql.pyindex e153b33670..e84a696659 100644--- a/marimo/_server/api/endpoints/sql.py+++ b/marimo/_server/api/endpoints/sql.py@@ -5,9 +5,7 @@ from typing import TYPE_CHECKINGfrom starlette.authentication import requires-from marimo._runtime.requests import (- ValidateSQLRequest,-)+from marimo._runtime.requests import ValidateSQLRequestfrom marimo._server.api.deps import AppStatefrom marimo._server.api.utils import parse_requestfrom marimo._server.models.models import BaseResponse, SuccessResponsediff --git a/marimo/_smoke_tests/sql/parse_errors.py b/marimo/_smoke_tests/sql/parse_errors.pynew file mode 100644index 0000000000..27dd2d5043--- /dev/null+++ b/marimo/_smoke_tests/sql/parse_errors.py@@ -0,0 +1,54 @@+import marimo++__generated_with = "0.16.2"+app = marimo.App(width="medium")+++def _():+ import duckdb+ import marimo as mo+ import json++ JSON_SERIALIZE_QUERY = "SELECT JSON_SERIALIZE_SQL(?, skip_null := true, skip_empty := true, skip_default := true, format := true)"+++ def parse(query):+ return json.loads(+ duckdb.execute(JSON_SERIALIZE_QUERY, [query]).fetchone()[0]+ )+ return mo, parse+++def _():+ queries = [+ "SELECT",+ "SELECT * FROM (SELECT 1 AS a, 2 AS b) AS subquery",+ "SELECT a, b FROM (SELECT 1 AS a, 2 AS b) AS subquery WHERE a = 1",+ "FROM subquery WHERE a = ",+ """+ SELECT a, b+ FROM foo+ WHERE a = 1;+ SELECT * FROM""",+ ]+ return (queries,)+++def _(mo, queries):+ table = mo.ui.table(queries, selection="single")+ table+ return (table,)+++def _(mo, parse, table):+ mo.stop(not table.value)+ parse(table.value[0])+ return+++if __name__ == "__main__":+ app.run()diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.pyindex 38b28cf409..6520537ba4 100644--- a/marimo/_sql/engines/duckdb.py+++ b/marimo/_sql/engines/duckdb.py@@ -38,6 +38,17 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):def dialect(self) -> str:return "duckdb"+ @staticmethod+ def execute_and_return_relation(+ query: str, params: Optional[list[Any]] = None+ ) -> duckdb.DuckDBPyRelation:+ """Execute a query and return a relation. Supports parameters."""+ DependencyManager.duckdb.require("to execute sql")++ import duckdb++ return duckdb.sql(query, params=params)+def execute(self, query: str) -> Any:relation = wrapped_sql(query, self._connection)diff --git a/marimo/_sql/engines/types.py b/marimo/_sql/engines/types.pyindex a0eb89b9a8..9902ea70ea 100644--- a/marimo/_sql/engines/types.py+++ b/marimo/_sql/engines/types.py@@ -154,3 +154,6 @@ class SQLConnection(EngineCatalog[CONN], QueryEngine[CONN]):"""Combines the catalog and query interfaces for an SQL engine."""pass+++SQLConnectionType = Union[EngineCatalog[Any], QueryEngine[Any]]diff --git a/marimo/_sql/error_utils.py b/marimo/_sql/error_utils.pyindex 4a5b46de2e..94bac1bad5 100644--- a/marimo/_sql/error_utils.py+++ b/marimo/_sql/error_utils.py@@ -98,7 +98,7 @@ def is_sql_parse_error(exception: BaseException) -> bool:return isinstance(exception, MarimoSQLException)-def extract_sql_position(+def _extract_sql_position(exception_msg: str,) -> tuple[Optional[int], Optional[int]]:"""Extract line and column position from SQL exception message."""@@ -144,7 +144,7 @@ def create_sql_error_metadata(This is the single source of truth for parsing SQL exceptions into metadata."""exception_msg = str(exception)- sql_line, sql_col = extract_sql_position(exception_msg)+ sql_line, sql_col = _extract_sql_position(exception_msg)# Truncate long SQL contenttruncated_sql = sql_contentdiff --git a/marimo/_sql/parse.py b/marimo/_sql/parse.pynew file mode 100644index 0000000000..9208e9bb16--- /dev/null+++ b/marimo/_sql/parse.py@@ -0,0 +1,137 @@+# Copyright 2025 Marimo. All rights reserved.++from typing import Literal, Optional, Union++import msgspec++from marimo import _loggers+from marimo._dependencies.dependencies import DependencyManager++LOGGER = _loggers.marimo_logger()+++class SqlParseError(msgspec.Struct):+ """+ Represents a single SQL parse error.++ Attributes:+ message (str): Description of the error.+ line (int): Line number where the error occurred (1-based).+ column (int): Column number where the error occurred (1-based).+ severity (Literal["error", "warning"]): Severity of the error.+ """++ message: str+ line: int+ column: int+ severity: Literal["error", "warning"]+++class SqlParseResult(msgspec.Struct):+ """+ Result of parsing an SQL query.++ Attributes:+ success (bool): True if parsing succeeded without errors.+ errors (list[SqlParseError]): List of parse errors (empty if success is True).+ """++ success: bool+ errors: list[SqlParseError]+++class SqlCatalogCheckResult(msgspec.Struct):+ """+ Result of running validation against the database.+ """++ success: bool+ error_message: Optional[str]+++def parse_sql(+ query: str, dialect: str+) -> tuple[Optional[SqlParseResult], Optional[str]]:+ """Parses an SQL query. Returns syntax errors.+ Does not check for catalog errors (incorrect table names, etc).+ Currently only supports DuckDB.++ Args:+ query (str): The SQL query to parse.+ dialect (str): The dialect of the SQL query.++ Returns:+ tuple[SqlParseResult, str]: SqlParseResult and unexpected errors+ """+ dialect = dialect.strip().lower()++ # Handle DuckDB+ if "duckdb" in dialect:+ return _parse_sql_duckdb(query)++ return None, "Unsupported dialect: " + dialect+++# DuckDB+++class DuckDBParseError(msgspec.Struct):+ error: bool+ error_type: Optional[str] = None+ error_message: Optional[str] = None+ error_subtype: Optional[str] = None+ position: Optional[Union[int, str]] = None+++# skip to reduce the response size+# the response doesn't matter too much, we are interested in the errors+JSON_SERIALIZE_QUERY = "SELECT JSON_SERIALIZE_SQL(CAST(? AS VARCHAR), skip_null := true, skip_empty := true, skip_default := true)"+++def _parse_sql_duckdb(+ query: str,+) -> tuple[Optional[SqlParseResult], Optional[str]]:+ """Parse an SQL query using DuckDB. Returns parse result and unexpected errors.++ Note:+ - Only SELECT statements support json_serialize_sql+ - Invalid function names do not throw errors+ - Some syntax errors do not throw errors since they are not errors in the AST parser+ """+ DependencyManager.duckdb.require("to parse sql")++ import duckdb++ relation = duckdb.execute(JSON_SERIALIZE_QUERY, [query])+ fetch_result = relation.fetchone()+ if fetch_result is None:+ return None, "No result from DuckDB parse query"++ parse_response = fetch_result[0]+ parsed_error = msgspec.json.decode(parse_response, type=DuckDBParseError)++ if not parsed_error.error:+ return SqlParseResult(success=True, errors=[]), None++ if parsed_error.error_type == "not implemented":+ # This is a valid query, but not supported by DuckDB+ # Only SELECT statements support json_serialize_sql+ return SqlParseResult(success=True, errors=[]), None++ position = int(parsed_error.position or 0)+ subquery = query[:position]+ line_number = subquery.count("\n") + 1+ column_number = position - subquery.rfind("\n") - 1++ sql_parse_result = SqlParseResult(+ success=False,+ errors=[+ SqlParseError(+ message=parsed_error.error_message or "Syntax error in query",+ line=line_number,+ column=column_number,+ severity="error",+ )+ ],+ )+ return sql_parse_result, Nonediff --git a/packages/openapi/api.yaml b/packages/openapi/api.yamlindex 1ae97b810b..5b196a082e 100644--- a/packages/openapi/api.yaml+++ b/packages/openapi/api.yaml@@ -3147,6 +3147,60 @@ components:required: []title: SnippetsConfigtype: object+ SqlCatalogCheckResult:+ description: Result of running validation against the database.+ properties:+ error_message:+ anyOf:+ - type: string+ - type: 'null'+ success:+ type: boolean+ required:+ - success+ - error_message+ title: SqlCatalogCheckResult+ type: object+ SqlParseError:+ description: "Represents a single SQL parse error.\n\nAttributes:\n message\+ \ (str): Description of the error.\n line (int): Line number where the\+ \ error occurred (1-based).\n column (int): Column number where the error\+ \ occurred (1-based).\n severity (Literal[\"error\", \"warning\"]): Severity\+ \ of the error."+ properties:+ column:+ type: integer+ line:+ type: integer+ message:+ type: string+ severity:+ enum:+ - error+ - warning+ required:+ - message+ - line+ - column+ - severity+ title: SqlParseError+ type: object+ SqlParseResult:+ description: "Result of parsing an SQL query.\n\nAttributes:\n success (bool):\+ \ True if parsing succeeded without errors.\n errors (list[SqlParseError]):\+ \ List of parse errors (empty if success is True)."+ properties:+ errors:+ items:+ $ref: '#/components/schemas/SqlParseError'+ type: array+ success:+ type: boolean+ required:+ - success+ - errors+ title: SqlParseResult+ type: objectStartupLogs:properties:content:@@ -3286,18 +3340,28 @@ components:title: UpdateComponentValuesRequesttype: objectValidateSQLRequest:- description: Validate an SQL query+ description: Validate an SQL query against the engineproperties:+ dialect:+ anyOf:+ - type: string+ - type: 'null'+ default: nullengine:- type: string+ anyOf:+ - type: string+ - type: 'null'+ default: null+ onlyParse:+ type: booleanquery:type: stringrequestId:type: stringrequired:- requestId- - engine- query+ - onlyParsetitle: ValidateSQLRequesttype: objectValidateSQLResult:@@ -3310,12 +3374,17 @@ components:op:enum:- validate-sql-result+ parse_result:+ anyOf:+ - type: 'null'+ - $ref: '#/components/schemas/SqlParseResult'+ default: nullrequest_id:type: string- result:+ validate_result:anyOf:- - {}- type: 'null'+ - $ref: '#/components/schemas/SqlCatalogCheckResult'default: nullrequired:- opdiff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.tsindex 9428c4c8b6..51badc1dd9 100644--- a/packages/openapi/src/api.ts+++ b/packages/openapi/src/api.ts@@ -4322,6 +4322,43 @@ export interface components {custom_paths?: string[];include_default_snippets?: boolean;};+ /**+ * SqlCatalogCheckResult+ * @description Result of running validation against the database.+ */+ SqlCatalogCheckResult: {+ error_message: string | null;+ success: boolean;+ };+ /**+ * SqlParseError+ * @description Represents a single SQL parse error.+ *+ * Attributes:+ * message (str): Description of the error.+ * line (int): Line number where the error occurred (1-based).+ * column (int): Column number where the error occurred (1-based).+ * severity (Literal["error", "warning"]): Severity of the error.+ */+ SqlParseError: {+ column: number;+ line: number;+ message: string;+ /** @enum {unknown} */+ severity: "error" | "warning";+ };+ /**+ * SqlParseResult+ * @description Result of parsing an SQL query.+ *+ * Attributes:+ * success (bool): True if parsing succeeded without errors.+ * errors (list[SqlParseError]): List of parse errors (empty if success is True).+ */+ SqlParseResult: {+ errors: components["schemas"]["SqlParseError"][];+ success: boolean;+ };/** StartupLogs */StartupLogs: {content: string;@@ -4395,10 +4432,14 @@ export interface components {};/*** ValidateSQLRequest- * @description Validate an SQL query+ * @description Validate an SQL query against the engine*/ValidateSQLRequest: {- engine: string;+ /** @default null */+ dialect?: string | null;+ /** @default null */+ engine?: string | null;+ onlyParse: boolean;query: string;requestId: string;};@@ -4408,9 +4449,11 @@ export interface components {error?: string | null;/** @enum {unknown} */op: "validate-sql-result";+ /** @default null */+ parse_result?: null | components["schemas"]["SqlParseResult"];request_id: string;/** @default null */- result?: unknown | null;+ validate_result?: null | components["schemas"]["SqlCatalogCheckResult"];};/** VariableContext */VariableContext: {diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex a9b093a2f1..1f6dfa7237 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -137,8 +137,8 @@ importers:specifier: ^0.1.5version: 0.1.5(@codemirror/[email protected])(@codemirror/[email protected])(@codemirror/[email protected])(@modelcontextprotocol/[email protected])'@marimo-team/codemirror-sql':- specifier: ^0.2.1- version: 0.2.1(@codemirror/[email protected])(@codemirror/[email protected])+ specifier: ^0.2.3+ version: 0.2.3(@codemirror/[email protected])(@codemirror/[email protected])'@marimo-team/llm-info':specifier: workspace:*version: link:../packages/llm-info@@ -1804,8 +1804,8 @@ packages:'@codemirror/view': ^6'@modelcontextprotocol/sdk': ^1- '@marimo-team/[email protected]':- resolution: {integrity: sha512-k/5IwEOIdSUB7wVoazF5qqhogOwaCW63Wpfk0uQ0oc1r5pNolfC0h6ZdZuf+pIHzkWun2u2dg0rMc9UOFtzIHQ==}+ '@marimo-team/[email protected]':+ resolution: {integrity: sha512-8Zwalm/MND9aWQATrnnMsXsATowDdbknE/Y1j+D7mwO3+kIbJnkQcY5OO5Y6hiB1AoEJKkQ6qpVBwIu76Z8G7g==}peerDependencies:'@codemirror/state': ^6'@codemirror/view': ^6@@ -7383,8 +7383,8 @@ packages:resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==}- resolution: {integrity: sha512-1ZcXu2qEFZlMkkaYPpMoTkypTunYTvtnInzVjWDhjc1+FS5h/WqT1rbgP2pP42/nXBWfPVN66mEfidWSpYVs1Q==}+ resolution: {integrity: sha512-GQBwA2e44qjbK0MzFwh5bNYefniV6cKT4KfjNDpuh/2EWipUEK1BCMc//moSidp8EF6fHB/EqJwUH9GCh9MAPg==}engines: {node: '>=8'}@@ -11394,13 +11394,13 @@ snapshots:'@codemirror/view': 6.38.2'@modelcontextprotocol/sdk': 1.17.2dependencies:'@codemirror/autocomplete': 6.18.7'@codemirror/lint': 6.8.5'@codemirror/state': 6.5.2'@codemirror/view': 6.38.2- node-sql-parser: 5.3.11+ node-sql-parser: 5.3.12'@marimo-team/[email protected]([email protected])':dependencies:@@ -18268,7 +18268,7 @@ snapshots:dependencies:'@types/pegjs': 0.10.6big-integer: 1.6.52diff --git a/pyproject.toml b/pyproject.tomlindex 638a9907c2..293a0cc28d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -545,7 +545,7 @@ format_on_save = truemulti_column = trueperformant_table_charts = truesql_linter = true-sql_mode = true+sql_mode = false[tool.marimo.display]dataframes = "rich"diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.pynew file mode 100644index 0000000000..89e127b477--- /dev/null+++ b/tests/_runtime/test_runtime_datasets.py@@ -0,0 +1,524 @@+# Copyright 2025 Marimo. All rights reserved.++from __future__ import annotations++import pytest++from marimo._dependencies.dependencies import DependencyManager+from marimo._messaging.ops import (+ DataSourceConnections,+ SQLTableListPreview,+ SQLTablePreview,+ ValidateSQLResult,+)+from marimo._runtime.requests import (+ ExecutionRequest,+ PreviewDataSourceConnectionRequest,+ PreviewSQLTableListRequest,+ PreviewSQLTableRequest,+ ValidateSQLRequest,+)+from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE+from marimo._sql.parse import SqlCatalogCheckResult, SqlParseResult+from marimo._types.ids import CellId_t, RequestId+from tests.conftest import MockedKernel++HAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()+++DUCKDB_CONN = "duckdb_conn"+SQLITE_CONN = "sqlite_conn"+++async def connection_requests() -> list[ExecutionRequest]:+ return [+ ExecutionRequest(cell_id=CellId_t("0"), code="import duckdb"),+ ExecutionRequest(+ cell_id=CellId_t("1"),+ code=f"{DUCKDB_CONN} = duckdb.connect(':memory:')",+ ),+ ExecutionRequest(cell_id=CellId_t("2"), code="import sqlite3"),+ ExecutionRequest(+ cell_id=CellId_t("3"),+ code=f"{SQLITE_CONN} = sqlite3.connect(':memory:')",+ ),+ ]+++# @pytest.mark.skipif(not HAS_SQL, reason="SQL deps not available")+# class TestGetSQLConnection:+# async def test_non_existent_engine(+# self, mocked_kernel: MockedKernel+# ) -> None:+# k = mocked_kernel.k++# # Non-existent engine+# k.get_sql_connection(DUCKDB_CONN)+# assert k.get_sql_connection(DUCKDB_CONN) == (None, "Engine not found")++# async def test_created_engine(+# self,+# mocked_kernel: MockedKernel,+# connection_requests: list[ExecutionRequest],+# ) -> None:+# k = mocked_kernel.k++# # Create duckdb and sqlite connections+# await k.run(connection_requests)++# connection, error = k.get_sql_connection("duckdb_conn")+# assert connection is not None+# assert error is None++# # Test with SQLite engine (which is a QueryEngine, but not a EngineCatalog)+# connection, error = k.get_sql_connection(SQLITE_CONN)+# assert connection is not None+# assert error is None++[email protected](not HAS_SQL, reason="SQL deps not available")+class TestPreviewSQLTable:+ async def test_non_existent_engine(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ preview_sql_table_request = PreviewSQLTableRequest(+ request_id=RequestId("0"),+ engine=DUCKDB_CONN,+ database="test",+ schema="test",+ table_name="t1",+ )+ await k.handle_message(preview_sql_table_request)++ preview_sql_table_results = [+ op for op in stream.operations if isinstance(op, SQLTablePreview)+ ]+ assert preview_sql_table_results == [+ SQLTablePreview(+ request_id=RequestId("0"), table=None, error="Engine not found"+ )+ ]++ async def test_catalog_engine(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ preview_sql_table_request = PreviewSQLTableRequest(+ request_id=RequestId("0"),+ engine=DUCKDB_CONN,+ database="test",+ schema="test",+ table_name="t1",+ )+ await k.handle_message(preview_sql_table_request)++ preview_sql_table_results = [+ op for op in stream.operations if isinstance(op, SQLTablePreview)+ ]+ assert preview_sql_table_results == [+ SQLTablePreview(request_id=RequestId("0"), table=None, error=None)+ ]++ async def test_query_engine(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ preview_sql_table_request = PreviewSQLTableRequest(+ request_id=RequestId("0"),+ engine=SQLITE_CONN,+ database="test",+ schema="test",+ table_name="t1",+ )+ await k.handle_message(preview_sql_table_request)++ preview_sql_table_results = [+ op for op in stream.operations if isinstance(op, SQLTablePreview)+ ]+ assert preview_sql_table_results == [+ SQLTablePreview(+ request_id=RequestId("0"),+ table=None,+ error="Connection does not support catalog operations",+ )+ ]++[email protected](not HAS_SQL, reason="SQL deps not available")+class TestPreviewSQLTableList:+ async def test_non_existent_engine(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ preview_sql_table_list_request = PreviewSQLTableListRequest(+ request_id=RequestId("0"),+ engine=DUCKDB_CONN,+ database="test",+ schema="test",+ )+ await k.handle_message(preview_sql_table_list_request)+ preview_sql_table_list_results = [+ op+ for op in stream.operations+ if isinstance(op, SQLTableListPreview)+ ]+ assert preview_sql_table_list_results == [+ SQLTableListPreview(+ request_id=RequestId("0"), tables=[], error="Engine not found"+ )+ ]++ async def test_catalog_engine(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ preview_sql_table_list_request = PreviewSQLTableListRequest(+ request_id=RequestId("0"),+ engine=DUCKDB_CONN,+ database="test",+ schema="test",+ )+ await k.handle_message(preview_sql_table_list_request)++ preview_sql_table_list_results = [+ op+ for op in stream.operations+ if isinstance(op, SQLTableListPreview)+ ]+ assert preview_sql_table_list_results == [+ SQLTableListPreview(+ request_id=RequestId("0"), tables=[], error=None+ )+ ]++ async def test_query_engine(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ preview_sql_table_list_request = PreviewSQLTableListRequest(+ request_id=RequestId("0"),+ engine=SQLITE_CONN,+ database="test",+ schema="test",+ )+ await k.handle_message(preview_sql_table_list_request)++ preview_sql_table_list_results = [+ op+ for op in stream.operations+ if isinstance(op, SQLTableListPreview)+ ]+ assert preview_sql_table_list_results == [+ SQLTableListPreview(+ request_id=RequestId("0"),+ tables=[],+ error="Connection does not support catalog operations",+ )+ ]+++class TestPreviewDatasourceConnection:+ async def test_non_existent_engine(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ preview_datasource_connection_request = (+ PreviewDataSourceConnectionRequest(engine=DUCKDB_CONN)+ )+ await k.handle_message(preview_datasource_connection_request)+ preview_datasource_connection_results = [+ op+ for op in stream.operations+ if isinstance(op, DataSourceConnections)+ ]+ assert preview_datasource_connection_results == []++ @pytest.mark.xfail(+ reason="Should have only 2 connections (duckdb and sqlite)"+ )+ async def test_engines(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ preview_datasource_connection_request = (+ PreviewDataSourceConnectionRequest(engine=DUCKDB_CONN)+ )+ await k.handle_message(preview_datasource_connection_request)++ preview_datasource_connection_results = [+ op+ for op in stream.operations+ if isinstance(op, DataSourceConnections)+ ]+ assert len(preview_datasource_connection_results) == 2++[email protected](not HAS_SQL, reason="SQL deps not available")+class TestSQLValidate:+ async def test_non_existent_engine(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ # Non-existent engine+ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("0"),+ engine=DUCKDB_CONN,+ query="SELECT * from t1",+ only_parse=False,+ )+ await k.handle_message(validate_sql_request)+ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results == [+ ValidateSQLResult(+ request_id=RequestId("0"),+ parse_result=None,+ validate_result=None,+ error="Failed to get engine duckdb_conn",+ )+ ]++ async def test_internal_engine_and_valid_query(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ # Internal engine and valid query+ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("1"),+ engine=INTERNAL_DUCKDB_ENGINE,+ query="SELECT 1, 2",+ only_parse=False,+ )+ await k.handle_message(validate_sql_request)+ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results[-1] == ValidateSQLResult(+ request_id=RequestId("1"),+ parse_result=SqlParseResult(success=True, errors=[]),+ validate_result=SqlCatalogCheckResult(+ success=True, error_message=None+ ),+ error=None,+ )++ async def test_internal_engine_and_invalid_query(+ self, mocked_kernel: MockedKernel+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ # Internal engine and invalid query+ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("2"),+ engine=INTERNAL_DUCKDB_ENGINE,+ query="SELECT * FROM t1",+ only_parse=False,+ )+ await k.handle_message(validate_sql_request)+ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ latest_validate_sql_result = validate_sql_results[-1]+ assert latest_validate_sql_result.request_id == RequestId("2")++ assert latest_validate_sql_result.parse_result is not None+ # query is syntactically valid+ assert latest_validate_sql_result.parse_result.success is True+ assert len(latest_validate_sql_result.parse_result.errors) == 0++ assert latest_validate_sql_result.validate_result is not None+ assert latest_validate_sql_result.validate_result.success is False+ assert (+ latest_validate_sql_result.validate_result.error_message+ is not None+ )+ assert latest_validate_sql_result.error is None++ stream.operations.clear()++ async def test_other_engine_and_valid_query(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ # Handle other engines+ await k.run(connection_requests)+ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("3"),+ engine=SQLITE_CONN,+ query="SELECT 1, 2",+ only_parse=False,+ )+ await k.handle_message(validate_sql_request)+ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert (+ validate_sql_results[-1]+ == ValidateSQLResult(+ request_id=RequestId("3"),+ parse_result=None, # Currently does not support parse errors for non-duckdb engines+ validate_result=SqlCatalogCheckResult(+ success=True, error_message=None+ ),+ error=None,+ )+ )++ async def test_only_parse_with_no_dialect(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("4"),+ engine=SQLITE_CONN,+ query="SELECT 1, 2",+ only_parse=True,+ )+ await k.handle_message(validate_sql_request)++ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results[-1] == ValidateSQLResult(+ request_id=RequestId("4"),+ parse_result=None,+ validate_result=None,+ error="Dialect is required when only parsing",+ )++ async def test_only_parse_unsupported_dialect(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("5"),+ dialect="sqlite",+ query="SELECT 1, 2",+ only_parse=True,+ )+ await k.handle_message(validate_sql_request)++ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results[-1] == ValidateSQLResult(+ request_id=RequestId("5"),+ parse_result=None,+ validate_result=None,+ error="Unsupported dialect: sqlite",+ )++ async def test_only_parse_duckdb(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("6"),+ dialect="duckdb",+ query="SELECT 1, 2",+ only_parse=True,+ )+ await k.handle_message(validate_sql_request)++ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results[-1] == ValidateSQLResult(+ request_id=RequestId("6"),+ parse_result=SqlParseResult(success=True, errors=[]),+ validate_result=None,+ error=None,+ )++ async def test_validate_but_no_engine(+ self,+ mocked_kernel: MockedKernel,+ connection_requests: list[ExecutionRequest],+ ) -> None:+ k = mocked_kernel.k+ stream = mocked_kernel.stream++ await k.run(connection_requests)++ validate_sql_request = ValidateSQLRequest(+ request_id=RequestId("7"),+ query="SELECT 1, 2",+ only_parse=False,+ )+ await k.handle_message(validate_sql_request)++ validate_sql_results = [+ op for op in stream.operations if isinstance(op, ValidateSQLResult)+ ]+ assert validate_sql_results[-1] == ValidateSQLResult(+ request_id=RequestId("7"),+ parse_result=None,+ validate_result=None,+ error="Engine is required for validating catalog",+ )diff --git a/tests/_server/api/endpoints/test_sql_endpoints.py b/tests/_server/api/endpoints/test_sql_endpoints.pyindex e69509b79a..ac29b61279 100644--- a/tests/_server/api/endpoints/test_sql_endpoints.py+++ b/tests/_server/api/endpoints/test_sql_endpoints.py@@ -24,6 +24,7 @@ def test_validate_sql(client: TestClient) -> None:"requestId": "test_request_id","engine": "test_engine","query": "SELECT * FROM test",+ "onlyParse": False,},)assert response.status_code == 200, response.textdiff --git a/tests/_sql/test_engines.py b/tests/_sql/test_engines.pyindex dfca9277cb..808712162b 100644--- a/tests/_sql/test_engines.py+++ b/tests/_sql/test_engines.py@@ -316,32 +316,36 @@ def test_duckdb_execute_in_explain_mode():not HAS_SQLALCHEMY or not (HAS_PANDAS or HAS_POLARS),reason="SQLAlchemy and either pandas or polars not installed",)-def test_sqlalchemy_execute() -> None:- """Test SQLAlchemyEngine execute method."""- import sqlalchemy as sa+class TestExecuteExplainModeSQLAlchemy:+ @pytest.fixture+ def engine(self) -> SQLAlchemyEngine:+ import sqlalchemy as sa- sqlite_engine = sa.create_engine("sqlite:///:memory:")- engine = SQLAlchemyEngine(sqlite_engine)+ sqlite_engine = sa.create_engine("sqlite:///:memory:")+ engine = SQLAlchemyEngine(sqlite_engine)+ return engine- # Create a test table- engine.execute("CREATE TABLE test (id INTEGER, name TEXT)")- engine.execute("INSERT INTO test VALUES (1, 'test1'), (2, 'test2')")+ def test_sqlalchemy_execute(self, engine: SQLAlchemyEngine) -> None:+ """Test SQLAlchemyEngine execute method."""+ # Create a test table+ engine.execute("CREATE TABLE test (id INTEGER, name TEXT)")+ engine.execute("INSERT INTO test VALUES (1, 'test1'), (2, 'test2')")- # Query the table- result = engine.execute("SELECT * FROM test ORDER BY id")+ # Query the table+ result = engine.execute("SELECT * FROM test ORDER BY id")- # Check result type based on available libraries- if HAS_POLARS:- import polars as pl+ # Check result type based on available libraries+ if HAS_POLARS:+ import polars as pl- assert isinstance(result, pl.DataFrame)- elif HAS_PANDAS:- import pandas as pd+ assert isinstance(result, pl.DataFrame)+ elif HAS_PANDAS:+ import pandas as pd- assert isinstance(result, pd.DataFrame)+ assert isinstance(result, pd.DataFrame)- # Test with a query that doesn't return a result set- assert engine.execute("PRAGMA journal_mode=WAL") is not None+ # Test with a query that doesn't return a result set+ assert engine.execute("PRAGMA journal_mode=WAL") is not None@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")diff --git a/tests/_sql/test_sql_error_handling.py b/tests/_sql/test_sql_error_handling.pyindex 24589616cb..046fd9912f 100644--- a/tests/_sql/test_sql_error_handling.py+++ b/tests/_sql/test_sql_error_handling.py@@ -8,8 +8,8 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._messaging.errors import MarimoSQLErrorfrom marimo._sql.error_utils import (MarimoSQLException,+ _extract_sql_position,create_sql_error_from_exception,- extract_sql_position,is_sql_parse_error,)from marimo._sql.sql import sql@@ -245,7 +245,7 @@ class TestErrorMessageQuality:"""Test position extraction from DuckDB error messages."""# DuckDB format: "Line 1, Col: 15"duckdb_msg = "Parser Error: syntax error at Line 1, Col: 15"- line, col = extract_sql_position(duckdb_msg)+ line, col = _extract_sql_position(duckdb_msg)assert line == 0 # 0-basedassert col == 14 # 0-based@@ -253,14 +253,14 @@ class TestErrorMessageQuality:"""Test position extraction from SQLGlot error messages."""# SQLGlot format variationssqlglot_msg = "Parse error at line 2, col 10"- line, col = extract_sql_position(sqlglot_msg)+ line, col = _extract_sql_position(sqlglot_msg)assert line == 1 # 0-basedassert col == 9 # 0-baseddef test_extract_sql_position_no_position(self):"""Test position extraction when no position info available."""no_position_msg = "Some generic SQL error without position"- line, col = extract_sql_position(no_position_msg)+ line, col = _extract_sql_position(no_position_msg)assert line is Noneassert col is Nonediff --git a/tests/_sql/test_sql_parse.py b/tests/_sql/test_sql_parse.pynew file mode 100644index 0000000000..6a397d61d8--- /dev/null+++ b/tests/_sql/test_sql_parse.py@@ -0,0 +1,423 @@+# Copyright 2025 Marimo. All rights reserved.++from __future__ import annotations++import pytest++from marimo._dependencies.dependencies import DependencyManager+from marimo._sql.parse import SqlParseError, SqlParseResult, parse_sql++HAS_DUCKDB = DependencyManager.duckdb.has()+++def test_sql_parse_result_with_errors():+ """Test SqlParseResult with errors."""+ error = SqlParseError(+ message="Syntax error", line=1, column=5, severity="error"+ )+ result = SqlParseResult(success=False, errors=[error])++ assert result.success is False+ assert len(result.errors) == 1+ assert result.errors[0].message == "Syntax error"+++class TestUnsupportedDialects:+ """Test handling of unsupported SQL dialects."""++ @pytest.mark.parametrize(+ "dialect",+ [+ "postgresql",+ "mysql",+ "sqlite",+ "oracle",+ "sqlserver",+ "bigquery",+ "snowflake",+ "redshift",+ "",+ "unknown_dialect",+ ],+ )+ def test_unsupported_dialects_return_none(self, dialect: str):+ """Test that unsupported dialects return none."""+ result, error = parse_sql("SELECT * FROM table", dialect)+ assert result is None+ assert error == "Unsupported dialect: " + dialect++ def test_dialect_with_whitespace(self):+ """Test dialects with leading/trailing whitespace."""+ result, error = parse_sql("SELECT 1", " postgresql ")++ assert result is None+ assert error == "Unsupported dialect: postgresql"++ result, error = parse_sql("SELECT 1", " duckdb ")+ assert result is not None+ assert error is None++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+class TestDuckDBValidQueries:+ """Test parsing of valid SQL queries with DuckDB."""++ @pytest.mark.parametrize(+ "query",+ [+ "SELECT 1",+ "SELECT 1 as value",+ "SELECT 'hello' as greeting",+ "SELECT 1, 2, 3",+ "SELECT 1 as a, 2 as b, 3 as c",+ "SELECT 1 UNION SELECT 2",+ "SELECT CASE WHEN 1 > 0 THEN 'positive' ELSE 'zero or negative' END",+ "SELECT COUNT(*)",+ "SELECT SUM(1), AVG(1), MAX(1), MIN(1)",+ "SELECT 1 ORDER BY 1",+ "SELECT 1 LIMIT 10",+ ],+ )+ def test_valid_queries_return_success(self, query: str):+ """Test that valid SQL queries return successful parse results."""+ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert isinstance(result, SqlParseResult)+ assert result.success is True+ assert result.errors == []++ @pytest.mark.parametrize(+ "dialect",+ [+ "duckdb",+ "DUCKDB",+ "DuckDB",+ " duckdb ",+ ],+ )+ def test_duckdb_dialect_variations(self, dialect: str):+ """Test that various DuckDB dialect strings work."""+ result, error = parse_sql("SELECT * FROM", dialect)+ assert result is not None+ assert error is None++ assert result.success is False++ def test_multiline_valid_query(self):+ """Test parsing multiline valid queries."""+ query = """+ SELECT+ 1 as id,+ 'test' as name,+ true as active+ ORDER BY id DESC+ """++ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert result.success is True+ assert result.errors == []++ def test_query_with_comments(self):+ """Test parsing queries with SQL comments."""+ query = """+ -- This is a comment+ SELECT 1 as test_column; -- End line comment+ """++ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert result.success is True+ assert result.errors == []++ def test_complex_valid_query(self):+ """Test parsing a complex but valid query."""+ query = """+ WITH summary AS (+ SELECT+ 1 as id,+ 100 as total_quantity,+ 50.5 as avg_price+ )+ SELECT+ id,+ total_quantity,+ avg_price,+ CASE+ WHEN total_quantity > 100 THEN 'High Volume'+ WHEN total_quantity > 50 THEN 'Medium Volume'+ ELSE 'Low Volume'+ END as volume_category+ FROM summary+ ORDER BY total_quantity DESC+ """++ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert result.success is True+ assert result.errors == []++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+class TestDuckDBInvalidQueries:+ """Test parsing of invalid SQL queries with DuckDB."""++ @pytest.mark.parametrize(+ ("query", "expected_error_keywords"),+ [+ ("SELECT * FRM table", ["syntax", "error"]),+ ("SELECT (", ["syntax", "error"]),+ ("SELECT * FROM", ["syntax", "error"]),+ ("SELECT ,", ["syntax", "error"]),+ ("SELEC * FROM table", ["syntax", "error"]),+ ("SELECT * FROM table WHERE", ["syntax", "error"]),+ ("SELECT * FROM table ORDER", ["syntax", "error"]),+ ("INSERT INTO", ["syntax", "error"]),+ ("UPDATE", ["syntax", "error"]),+ ("DELETE", ["syntax", "error"]),+ # Unclosed parentheses+ ("SELECT COUNT(*", ["syntax", "error"]),+ # Unclosed quotes+ # ("SELECT 'unclosed string", ["syntax", "error"]), # DuckDB does not raise errors for this case+ # # Invalid function call+ ("SELECT COUNT(DISTINCT)", ["syntax", "error"]),+ # # Malformed CASE statement+ ("SELECT CASE WHEN FROM table", ["syntax", "error"]),+ # # Incomplete JOIN+ ("SELECT * FROM table1 JOIN", ["syntax", "error"]),+ ],+ )+ def test_invalid_queries_return_errors(+ self, query: str, expected_error_keywords: list[str]+ ):+ """Test that invalid SQL queries return error results."""+ result, error = parse_sql(query, "duckdb")++ assert isinstance(result, SqlParseResult)+ assert result.success is False+ assert len(result.errors) > 0++ error = result.errors[0]+ assert isinstance(error, SqlParseError)+ assert error.severity == "error"+ assert error.line > 0+ assert error.column >= 0++ # Check that error message contains expected keywords+ error_msg_lower = error.message.lower()+ assert any(+ keyword in error_msg_lower for keyword in expected_error_keywords+ )++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+class TestErrorPositionCalculation:+ """Test that error positions (line and column) are calculated correctly."""++ def test_single_line_error_position(self):+ """Test position calculation for single-line queries."""+ query = "SELECT * FRM table" # Error at position of "FRM"++ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert result.success is False+ assert len(result.errors) == 1++ error = result.errors[0]+ assert error.line == 1+ assert error.column == 13++ @pytest.mark.xfail(reason="DuckDB does not raise errors for this case")+ def test_multiline_error_position(self):+ """Test position calculation for multiline queries."""+ query = """SELECT name,+ email,+ FRM users""" # Error on line 3++ result, error = parse_sql(query, "duckdb")+ assert result is not None++ assert result.success is False+ assert len(result.errors) == 1++ error = result.errors[0]+ assert error.line == 3+ assert error.column == 0++ def test_error_at_beginning_of_line(self):+ """Test position calculation when error is at beginning of line."""+ query = """SELECT *+FRM table"""++ result, error = parse_sql(query, "duckdb")+ assert result is not None++ assert result.success is False+ assert len(result.errors) == 1++ error = result.errors[0]+ assert error.line == 2+ assert error.column == 4++ def test_error_with_leading_whitespace(self):+ """Test position calculation with leading whitespace."""+ query = """ SELECT *+ FRM table"""++ result, error = parse_sql(query, "duckdb")+ assert result is not None++ assert result.success is False+ assert len(result.errors) == 1++ error = result.errors[0]+ assert error.line == 2++ @pytest.mark.xfail(+ reason="DuckDB does not raise errors for invalid syntax"+ )+ def test_error_position_after_newlines(self):+ """Test position calculation with multiple newlines."""+ query = """+ SELECT name+ FROM users+ WHERE invalid_syntax"""++ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert error is None++ assert result.success is False+ assert len(result.errors) == 1++ error = result.errors[0]+ assert error.line == 3+ assert error.column >= 0++[email protected](not HAS_DUCKDB, reason="DuckDB not installed")+class TestEdgeCases:+ """Test edge cases and boundary conditions."""++ def test_empty_query(self):+ """Test parsing empty query."""+ result, error = parse_sql("", "duckdb")+ assert result is not None+ assert error is None++ assert isinstance(result, SqlParseResult)+ assert result.success is True++ def test_whitespace_only_query(self):+ """Test parsing query with only whitespace."""+ result, error = parse_sql(" \n \t ", "duckdb")+ assert result is not None+ assert error is None++ assert isinstance(result, SqlParseResult)+ assert result.success is True+ assert result.errors == []++ def test_query_with_semicolon(self):+ """Test parsing query with trailing semicolon."""+ result, error = parse_sql("SELECT 1;", "duckdb")+ assert result is not None++ assert result.success is True+ assert result.errors == []++ def test_multiple_statements(self):+ """Test parsing multiple SQL statements."""+ query = "SELECT 1; SELECT 2;"++ result, error = parse_sql(query, "duckdb")++ # DuckDB might handle this differently, just ensure we get a result+ assert isinstance(result, SqlParseResult)++ def test_very_long_query(self):+ """Test parsing a very long query."""+ # Create a query with many columns+ columns = ", ".join([f"{i} as col_{i}" for i in range(100)])+ query = f"SELECT {columns}"++ result, error = parse_sql(query, "duckdb")+ assert error is None++ assert result is not None+ assert result.success is True+ assert result.errors == []++ def test_query_with_unicode(self):+ """Test parsing query with Unicode characters."""+ query = "SELECT 'Hello 世界' as greeting"++ result, error = parse_sql(query, "duckdb")+ assert error is None++ assert result is not None+ assert result.success is True+ assert result.errors == []++ def test_query_with_special_characters(self):+ """Test parsing query with various special characters."""+ query = "SELECT 'test@#$%^&*()' as special_chars"++ result, error = parse_sql(query, "duckdb")+ assert error is None++ assert result is not None+ assert result.success is True+ assert result.errors == []++ def test_deeply_nested_query(self):+ """Test parsing deeply nested subqueries."""+ query = """+ SELECT * FROM (+ SELECT * FROM (+ SELECT * FROM (+ SELECT 1 as nested_value+ ) inner_query+ ) middle_query+ ) outer_query+ """++ result, error = parse_sql(query, "duckdb")+ assert error is None++ assert result is not None+ assert result.success is True+ assert result.errors == []++ def test_just_comments(self):+ """Test parsing query with just comments."""+ for query in [+ """+ -- This is a comment+ /* This is a comment */+ """,+ """-- SELECT 1""",+ """/* SELECT 1 */""",+ """+ -- This is a comment+ SELECT 1 as test_column; -- End line comment+ """,+ """+ /* This is a comment */+ SELECT 1 as test_column; /* End line comment */+ """,+ ]:+ result, error = parse_sql(query, "duckdb")+ assert result is not None+ assert result.success is True+ assert result.errors == []