Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

add parse function in /validate endpoint for SQL (#6517)

Shahmir Varqhad88107067031
diff --git a/frontend/package.json b/frontend/package.json
index 70b955514b..6009c6a656 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -52,7 +52,7 @@
"@marimo-team/codemirror-ai": "^0.3.2",
"@marimo-team/codemirror-languageserver": "1.15.24",
"@marimo-team/codemirror-mcp": "^0.1.5",
- "@marimo-team/codemirror-sql": "^0.2.1",
+ "@marimo-team/codemirror-sql": "^0.2.3",
"@marimo-team/llm-info": "workspace:*",
"@marimo-team/marimo-api": "workspace:*",
"@marimo-team/react-slotz": "^0.2.0",
diff --git a/frontend/src/components/editor/errors/sql-validation-errors.tsx b/frontend/src/components/editor/errors/sql-validation-errors.tsx
index d9e89e8fc0..eeb923eea5 100644
--- a/frontend/src/components/editor/errors/sql-validation-errors.tsx
+++ b/frontend/src/components/editor/errors/sql-validation-errors.tsx
@@ -2,7 +2,7 @@
import { AlertCircleIcon } from "lucide-react";
import type { CellId } from "@/core/cells/ids";
-import { useSqlValidationErrorsForCell } from "@/core/codemirror/language/languages/sql/validation-errors";
+import { useSqlValidationErrorsForCell } from "@/core/codemirror/language/languages/sql/banner-validation-errors";
export const SqlValidationErrorBanner = ({
cellId,
diff --git a/frontend/src/core/codemirror/language/__tests__/extension.test.ts b/frontend/src/core/codemirror/language/__tests__/extension.test.ts
index e6d04c1672..926989b674 100644
--- a/frontend/src/core/codemirror/language/__tests__/extension.test.ts
+++ b/frontend/src/core/codemirror/language/__tests__/extension.test.ts
@@ -14,7 +14,7 @@ import {
languageAdapterState,
switchLanguage,
} from "../extension";
-import { exportedForTesting as sqlValidationErrorsForTesting } from "../languages/sql/validation-errors";
+import { exportedForTesting as sqlValidationErrorsForTesting } from "../languages/sql/banner-validation-errors";
import { languageMetadataField } from "../metadata";
let view: EditorView | null = null;
diff --git a/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts b/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts
index 9edd9c1e73..3d466a0e65 100644
--- a/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts
+++ b/frontend/src/core/codemirror/language/__tests__/sql-validation.test.ts
@@ -1,7 +1,7 @@
/* Copyright 2024 Marimo. All rights reserved. */
import { describe, expect, it } from "vitest";
-import { exportedForTesting } from "../languages/sql/validation-errors";
+import { exportedForTesting } from "../languages/sql/banner-validation-errors";
describe("Error Message Splitting", () => {
it("should handle error message splitting correctly", () => {
diff --git a/frontend/src/core/codemirror/language/languages/sql/validation-errors.ts b/frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.ts
similarity index 93%
rename from frontend/src/core/codemirror/language/languages/sql/validation-errors.ts
rename to frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.ts
index 052eb23f6f..2386fcba9d 100644
--- a/frontend/src/core/codemirror/language/languages/sql/validation-errors.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/banner-validation-errors.ts
@@ -31,18 +31,20 @@ export function clearSqlValidationError(cellId: CellId) {
export function setSqlValidationError({
cellId,
- error,
+ errorMessage,
dialect,
}: {
cellId: CellId;
- error: string;
+ errorMessage: string;
dialect: SupportedDialects | null;
}) {
const sqlValidationErrors = store.get(sqlValidationErrorsAtom);
const newErrors = new Map(sqlValidationErrors);
const errorResult: SQLValidationError =
- dialect === "DuckDB" ? handleDuckdbError(error) : splitErrorMessage(error);
+ dialect === "DuckDB"
+ ? handleDuckdbError(errorMessage)
+ : splitErrorMessage(errorMessage);
newErrors.set(cellId, errorResult);
store.set(sqlValidationErrorsAtom, newErrors);
diff --git a/frontend/src/core/codemirror/language/languages/sql/sql.ts b/frontend/src/core/codemirror/language/languages/sql/sql.ts
index 3ef15ed407..fc47959dc6 100644
--- a/frontend/src/core/codemirror/language/languages/sql/sql.ts
+++ b/frontend/src/core/codemirror/language/languages/sql/sql.ts
@@ -11,7 +11,9 @@ import { parser } from "@lezer/python";
import {
defaultSqlHoverTheme,
NodeSqlParser,
+ type NodeSqlParserResult,
type SupportedDialects as ParserDialects,
+ type SqlParseError,
sqlExtension,
} from "@marimo-team/codemirror-sql";
import { DuckDBDialect } from "@marimo-team/codemirror-sql/dialects";
@@ -28,7 +30,6 @@ import {
INTERNAL_SQL_ENGINES,
} from "@/core/datasets/engines";
import { ValidateSQL } from "@/core/datasets/request-registry";
-import type { ValidateSQLResult } from "@/core/kernel/messages";
import { store } from "@/core/state/jotai";
import { resolvedThemeAtom } from "@/theme/useTheme";
import { Logger } from "@/utils/Logger";
@@ -39,16 +40,16 @@ import { parseArgsKwargs } from "../../utils/ast";
import { indentOneTab } from "../../utils/indentOneTab";
import type { QuotePrefixKind } from "../../utils/quotes";
import { MarkdownLanguageAdapter } from "../markdown";
+import {
+ clearSqlValidationError,
+ setSqlValidationError,
+} from "./banner-validation-errors";
import {
customKeywordCompletionSource,
tablesCompletionSource,
} from "./completion-sources";
import { SCHEMA_CACHE } from "./completion-store";
import { getSQLMode } from "./sql-mode";
-import {
- clearSqlValidationError,
- setSqlValidationError,
-} from "./validation-errors";
const DEFAULT_DIALECT = DuckDBDialect;
const DEFAULT_PARSER_DIALECT = "DuckDB";
@@ -245,7 +246,7 @@ export class SQLLanguageAdapter
if (this.sqlLinterEnabled) {
const theme = store.get(resolvedThemeAtom);
- const parser = new NodeSqlParser({
+ const parser = new CustomSqlParser({
getParserOptions: (state: EditorState) => {
return {
database: guessParserDialect(state) ?? DEFAULT_PARSER_DIALECT,
@@ -280,14 +281,84 @@ export class SQLLanguageAdapter
);
}
- if (this.sqlModeEnabled) {
- extensions.push(sqlValidationExtension());
- }
+ // TODO: Re-enable after we optimize the endpoint
+ // if (this.sqlModeEnabled) {
+ // extensions.push(sqlValidationExtension());
+ // }
return extensions;
}
}
+class CustomSqlParser extends NodeSqlParser {
+ private validationTimeout: number | null = null;
+ private readonly VALIDATION_DELAY_MS = 300; // Wait 300ms after user stops typing
+
+ private async validateWithDelay(
+ sql: string,
+ engine: string,
+ dialect: ParserDialects | null,
+ ): Promise<SqlParseError[]> {
+ // Clear any existing delay call
+ if (this.validationTimeout) {
+ window.clearTimeout(this.validationTimeout);
+ }
+
+ // Set up a new request to be called after the delay
+ return new Promise((resolve) => {
+ this.validationTimeout = window.setTimeout(async () => {
+ try {
+ const result = await ValidateSQL.request({
+ onlyParse: true,
+ engine,
+ dialect,
+ query: sql,
+ });
+ if (result.error) {
+ Logger.error("Failed to validate SQL", { error: result.error });
+ resolve([]);
+ return;
+ }
+ resolve(result.parse_result?.errors ?? []);
+ } catch (error) {
+ Logger.error("Failed to validate SQL", { error });
+ resolve([]);
+ }
+ }, this.VALIDATION_DELAY_MS);
+ });
+ }
+
+ override async validateSql(
+ sql: string,
+ opts: { state: EditorState },
+ ): Promise<SqlParseError[]> {
+ const metadata = getSQLMetadata(opts.state);
+
+ // Only perform custom validation for internal engines
+ if (!INTERNAL_SQL_ENGINES.has(metadata.engine)) {
+ return super.validateSql(sql, opts);
+ }
+
+ const dialect = guessParserDialect(opts.state);
+ return this.validateWithDelay(sql, metadata.engine, dialect);
+ }
+
+ override async parse(
+ sql: string,
+ opts: { state: EditorState },
+ ): Promise<NodeSqlParserResult> {
+ const metadata = getSQLMetadata(opts.state);
+ const engine = metadata.engine;
+
+ // For now, always return success for DuckDB
+ if (engine === DUCKDB_ENGINE) {
+ return { success: true, errors: [] };
+ }
+
+ return super.parse(sql, opts);
+ }
+}
+
/**
* Update the SQL dialect in the editor view.
*/
@@ -568,11 +639,18 @@ function safeDedent(code: string): string {
}
}
-function sqlValidationExtension(): Extension {
- let debounceTimeout: NodeJS.Timeout | null = null;
+const SQL_VALIDATION_DEBOUNCE_MS = 300;
+
+// @ts-expect-error: TODO: Re-enable after we optimize the endpoint
+function _sqlValidationExtension(): Extension {
+ let debounceTimeout: number | undefined;
let lastValidationRequest: string | null = null;
return EditorView.updateListener.of((update) => {
+ if (!update.docChanged) {
+ return;
+ }
+
const sqlMode = getSQLMode();
if (sqlMode !== "validate") {
return;
@@ -580,12 +658,8 @@ function sqlValidationExtension(): Extension {
const metadata = getSQLMetadata(update.state);
const connectionName = metadata.engine;
+ // Currently only internal engines are supported
if (!INTERNAL_SQL_ENGINES.has(connectionName)) {
- // Currently only internal engines are supported
- return;
- }
-
- if (!update.docChanged) {
return;
}
@@ -594,11 +668,11 @@ function sqlValidationExtension(): Extension {
// Clear existing timeout
if (debounceTimeout) {
- clearTimeout(debounceTimeout);
+ window.clearTimeout(debounceTimeout);
}
// Debounce the validation call
- debounceTimeout = setTimeout(async () => {
+ debounceTimeout = window.setTimeout(async () => {
// Skip if content hasn't changed since last validation
if (lastValidationRequest === sqlContent) {
return;
@@ -613,20 +687,32 @@ function sqlValidationExtension(): Extension {
}
try {
- const result: ValidateSQLResult = await ValidateSQL.request({
+ const result = await ValidateSQL.request({
+ onlyParse: false,
engine: connectionName,
query: sqlContent,
});
if (result.error) {
+ Logger.error("Failed to validate SQL", { error: result.error });
+ return;
+ }
+
+ const validateResult = result.validate_result;
+
+ if (validateResult?.error_message) {
const dialect = connectionNameToParserDialect(connectionName);
- setSqlValidationError({ cellId, error: result.error, dialect });
+ setSqlValidationError({
+ cellId,
+ errorMessage: validateResult.error_message,
+ dialect,
+ });
} else {
clearSqlValidationError(cellId);
}
} catch (error) {
Logger.warn("Failed to validate SQL", { error });
}
- }, 300);
+ }, SQL_VALIDATION_DEBOUNCE_MS);
});
}
diff --git a/marimo/_config/config.py b/marimo/_config/config.py
index 79ba360ccc..aa52c7e039 100644
--- a/marimo/_config/config.py
+++ b/marimo/_config/config.py
@@ -508,7 +508,7 @@ class ExperimentalConfig(TypedDict, total=False):
performant_table_charts: bool
mcp_docs: bool
sql_linter: bool
- sql_mode: bool
+ sql_mode: bool # Not exposed for now
# Internal features
cache: CacheConfig
diff --git a/marimo/_messaging/ops.py b/marimo/_messaging/ops.py
index 4da2e836f3..217378f137 100644
--- a/marimo/_messaging/ops.py
+++ b/marimo/_messaging/ops.py
@@ -53,6 +53,7 @@ from marimo._runtime.context.types import ContextNotInitializedError
from marimo._runtime.context.utils import get_mode
from marimo._runtime.layout.layout import LayoutConfig
from marimo._secrets.models import SecretKeysWithProvider
+from marimo._sql.parse import SqlCatalogCheckResult, SqlParseResult
from marimo._types.ids import CellId_t, RequestId, WidgetModelId
from marimo._utils.platform import is_pyodide, is_windows
@@ -629,7 +630,8 @@ class DataSourceConnections(Op, tag="data-source-connections"):
class ValidateSQLResult(Op, tag="validate-sql-result"):
name: ClassVar[str] = "validate-sql-result"
request_id: RequestId
- result: Optional[Any] = None
+ parse_result: Optional[SqlParseResult] = None
+ validate_result: Optional[SqlCatalogCheckResult] = None
error: Optional[str] = None
diff --git a/marimo/_runtime/requests.py b/marimo/_runtime/requests.py
index 6b6ca0f0af..a2bc79148a 100644
--- a/marimo/_runtime/requests.py
+++ b/marimo/_runtime/requests.py
@@ -339,10 +339,22 @@ class PreviewDataSourceConnectionRequest(msgspec.Struct, rename="camel"):
class ValidateSQLRequest(msgspec.Struct, rename="camel"):
- """Validate an SQL query"""
+ """Validate an SQL query against the engine"""
request_id: RequestId
- engine: str
+ query: str
+ # Whether to only parse the query or validate against the database
+ # Parsing is done without a DB connection and uses dialect, whereas validation requires a connection
+ only_parse: bool
+ engine: Optional[str] = None
+ dialect: Optional[str] = None
+
+
+class ParseSQLRequest(msgspec.Struct, rename="camel"):
+ """Parse an SQL query for linting"""
+
+ request_id: RequestId
+ dialect: str
query: str
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index 843fb6f193..090a5f4047 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -179,11 +179,16 @@ from marimo._secrets.secrets import get_secret_keys
from marimo._server.model import SessionMode
from marimo._server.types import QueueType
from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE, DuckDBEngine
-from marimo._sql.engines.types import EngineCatalog, QueryEngine
+from marimo._sql.engines.types import (
+ EngineCatalog,
+ QueryEngine,
+ SQLConnectionType,
+)
from marimo._sql.get_engines import (
engine_to_data_source_connection,
get_engines_from_variables,
)
+from marimo._sql.parse import SqlCatalogCheckResult, parse_sql
from marimo._tracer import kernel_tracer
from marimo._types.ids import CellId_t, UIElementId, VariableName
from marimo._types.lifespan import Lifespan
@@ -2252,25 +2257,28 @@ class Kernel:
await self.request_handler.handle(request)
LOGGER.debug("Handled control request: %s", request)
- def get_engine_catalog(
+ def get_sql_connection(
self, variable_name: str
- ) -> tuple[Optional[EngineCatalog[Any]], Optional[str]]:
- """Fetch the catalog-capable engine associated with the given variable name.
-
- Returns the engine if it supports catalog operations, or an error message if not."""
+ ) -> tuple[Optional[SQLConnectionType], Optional[str]]:
+ """
+ Fetch the SQL connection associated with the given variable name.
+ Returns the connection if it supports query or catalog operations, or an error message if not.
+ """
variable_name = cast(VariableName, variable_name)
try:
- # Should we find the existing engine instead?
engine_val = self.globals.get(variable_name)
engines = get_engines_from_variables([(variable_name, engine_val)])
if engines is None or len(engines) == 0:
return None, "Engine not found"
engine = engines[0][1]
- if isinstance(engine, EngineCatalog):
+ if isinstance(engine, (QueryEngine, EngineCatalog)):
return engine, None
else:
- return None, "Connection does not support catalog operations"
+ return (
+ None,
+ "Connection does not support query or catalog operations",
+ )
except Exception as e:
LOGGER.warning(
"Failed to get engine %s", variable_name, exc_info=e
@@ -2282,6 +2290,21 @@ class DatasetCallbacks:
def __init__(self, kernel: Kernel):
self._kernel = kernel
+ def get_engine_catalog(
+ self, variable_name: str
+ ) -> tuple[Optional[EngineCatalog[Any]], Optional[str]]:
+ """Get engines that support catalog operations.
+ Returns an error if the connection does not support catalog operations."""
+ variable_name = cast(VariableName, variable_name)
+ connection, error = self._kernel.get_sql_connection(variable_name)
+ if error is not None or connection is None:
+ return None, error
+
+ if isinstance(connection, EngineCatalog):
+ return connection, None
+ else:
+ return None, "Connection does not support catalog operations"
+
@kernel_tracer.start_as_current_span("preview_dataset_column")
async def preview_dataset_column(
self, request: PreviewDatasetColumnRequest
@@ -2367,7 +2390,7 @@ class DatasetCallbacks:
schema_name = request.schema
table_name = request.table_name
- engine, error = self._kernel.get_engine_catalog(variable_name)
+ engine, error = self.get_engine_catalog(variable_name)
if error is not None or engine is None:
SQLTablePreview(
request_id=request.request_id, table=None, error=error
@@ -2412,7 +2435,7 @@ class DatasetCallbacks:
database_name = request.database
schema_name = request.schema
- engine, error = self._kernel.get_engine_catalog(variable_name)
+ engine, error = self.get_engine_catalog(variable_name)
if error is not None or engine is None:
SQLTableListPreview(
request_id=request.request_id, tables=[], error=error
@@ -2444,7 +2467,7 @@ class DatasetCallbacks:
) -> None:
"""Broadcasts a datasource connection for a given engine"""
variable_name = cast(VariableName, request.engine)
- engine, error = self._kernel.get_engine_catalog(variable_name)
+ engine, error = self.get_engine_catalog(variable_name)
if error is not None or engine is None:
LOGGER.error("Failed to get engine %s", variable_name)
return
@@ -2467,34 +2490,82 @@ class SqlCallbacks:
@kernel_tracer.start_as_current_span("validate_sql_query")
async def validate_sql(self, request: ValidateSQLRequest) -> None:
- """Validate an SQL query"""
- # TODO: Place request in queue
+ """Validate an SQL query
+
+ This will validate:
+ - the syntax (parsing)
+ - the catalog (table and column names)
+ """
+ request_id = request.request_id
+
+ if request.only_parse:
+ if request.dialect is None:
+ ValidateSQLResult(
+ request_id=request_id,
+ error="Dialect is required when only parsing",
+ ).broadcast()
+ return
+
+ # Just parse the query (no DB connection required)
+ parse_result, error = parse_sql(request.query, request.dialect)
+ ValidateSQLResult(
+ request_id=request_id,
+ parse_result=parse_result,
+ error=error,
+ ).broadcast()
+ return
+
+ # Validate against the database
+ # This can be cheap for in-memory engines (duckdb, sqlite)
+ # But potentially expensive and requires an active connection for remote engines
+ # For failed connections, we should not raise an error
+
+ if request.engine is None:
+ ValidateSQLResult(
+ request_id=request_id,
+ error="Engine is required for validating catalog",
+ ).broadcast()
+ return
+
variable_name = cast(VariableName, request.engine)
- engine: Optional[EngineCatalog[Any]] = None
+ engine: Optional[SQLConnectionType] = None
if variable_name == INTERNAL_DUCKDB_ENGINE:
engine = DuckDBEngine(connection=None)
error = None
else:
- engine, error = self._kernel.get_engine_catalog(variable_name)
+ engine, error = self._kernel.get_sql_connection(variable_name)
if error is not None or engine is None:
- LOGGER.error("Failed to get engine %s", variable_name)
ValidateSQLResult(
- request_id=request.request_id,
- result=None,
- error="Engine not found",
+ request_id=request_id,
+ error="Failed to get engine " + variable_name,
).broadcast()
return
- if isinstance(engine, QueryEngine):
- result, error = engine.execute_in_explain_mode(request.query) # type: ignore
- else:
- error = "Engine does not support explain mode"
+ # Get the parse error for linting
+ parse_result, parse_error = parse_sql(request.query, engine.dialect)
+ if parse_error is not None:
+ # We don't want to fail the validation if there is a parse error
+ LOGGER.debug("Parse error: %s", parse_error)
+ if not isinstance(engine, QueryEngine):
+ ValidateSQLResult(
+ request_id=request_id,
+ error=f"Engine {variable_name} does not support catalog validation.",
+ parse_result=parse_result,
+ ).broadcast()
+ return
+
+ _, error_message = engine.execute_in_explain_mode(request.query) # type: ignore
+ validate_result = SqlCatalogCheckResult(
+ success=True if error_message is None else False,
+ error_message=error_message,
+ )
ValidateSQLResult(
- request_id=request.request_id,
- result=None, # We aren't using the result yet
- error=error,
+ request_id=request_id,
+ validate_result=validate_result,
+ parse_result=parse_result,
+ error=None,
).broadcast()
diff --git a/marimo/_server/api/endpoints/sql.py b/marimo/_server/api/endpoints/sql.py
index e153b33670..e84a696659 100644
--- a/marimo/_server/api/endpoints/sql.py
+++ b/marimo/_server/api/endpoints/sql.py
@@ -5,9 +5,7 @@ from typing import TYPE_CHECKING
from starlette.authentication import requires
-from marimo._runtime.requests import (
- ValidateSQLRequest,
-)
+from marimo._runtime.requests import ValidateSQLRequest
from marimo._server.api.deps import AppState
from marimo._server.api.utils import parse_request
from marimo._server.models.models import BaseResponse, SuccessResponse
diff --git a/marimo/_smoke_tests/sql/parse_errors.py b/marimo/_smoke_tests/sql/parse_errors.py
new file mode 100644
index 0000000000..27dd2d5043
--- /dev/null
+++ b/marimo/_smoke_tests/sql/parse_errors.py
@@ -0,0 +1,54 @@
+import marimo
+
+__generated_with = "0.16.2"
+app = marimo.App(width="medium")
+
+
+def _():
+ import duckdb
+ import marimo as mo
+ import json
+
+ JSON_SERIALIZE_QUERY = "SELECT JSON_SERIALIZE_SQL(?, skip_null := true, skip_empty := true, skip_default := true, format := true)"
+
+
+ def parse(query):
+ return json.loads(
+ duckdb.execute(JSON_SERIALIZE_QUERY, [query]).fetchone()[0]
+ )
+ return mo, parse
+
+
+def _():
+ queries = [
+ "SELECT",
+ "SELECT * FROM (SELECT 1 AS a, 2 AS b) AS subquery",
+ "SELECT a, b FROM (SELECT 1 AS a, 2 AS b) AS subquery WHERE a = 1",
+ "FROM subquery WHERE a = ",
+ """
+ SELECT a, b
+ FROM foo
+ WHERE a = 1;
+ SELECT * FROM""",
+ ]
+ return (queries,)
+
+
+def _(mo, queries):
+ table = mo.ui.table(queries, selection="single")
+ table
+ return (table,)
+
+
+def _(mo, parse, table):
+ mo.stop(not table.value)
+ parse(table.value[0])
+ return
+
+
+if __name__ == "__main__":
+ app.run()
diff --git a/marimo/_sql/engines/duckdb.py b/marimo/_sql/engines/duckdb.py
index 38b28cf409..6520537ba4 100644
--- a/marimo/_sql/engines/duckdb.py
+++ b/marimo/_sql/engines/duckdb.py
@@ -38,6 +38,17 @@ class DuckDBEngine(SQLConnection[Optional["duckdb.DuckDBPyConnection"]]):
def dialect(self) -> str:
return "duckdb"
+ @staticmethod
+ def execute_and_return_relation(
+ query: str, params: Optional[list[Any]] = None
+ ) -> duckdb.DuckDBPyRelation:
+ """Execute a query and return a relation. Supports parameters."""
+ DependencyManager.duckdb.require("to execute sql")
+
+ import duckdb
+
+ return duckdb.sql(query, params=params)
+
def execute(self, query: str) -> Any:
relation = wrapped_sql(query, self._connection)
diff --git a/marimo/_sql/engines/types.py b/marimo/_sql/engines/types.py
index a0eb89b9a8..9902ea70ea 100644
--- a/marimo/_sql/engines/types.py
+++ b/marimo/_sql/engines/types.py
@@ -154,3 +154,6 @@ class SQLConnection(EngineCatalog[CONN], QueryEngine[CONN]):
"""Combines the catalog and query interfaces for an SQL engine."""
pass
+
+
+SQLConnectionType = Union[EngineCatalog[Any], QueryEngine[Any]]
diff --git a/marimo/_sql/error_utils.py b/marimo/_sql/error_utils.py
index 4a5b46de2e..94bac1bad5 100644
--- a/marimo/_sql/error_utils.py
+++ b/marimo/_sql/error_utils.py
@@ -98,7 +98,7 @@ def is_sql_parse_error(exception: BaseException) -> bool:
return isinstance(exception, MarimoSQLException)
-def extract_sql_position(
+def _extract_sql_position(
exception_msg: str,
) -> tuple[Optional[int], Optional[int]]:
"""Extract line and column position from SQL exception message."""
@@ -144,7 +144,7 @@ def create_sql_error_metadata(
This is the single source of truth for parsing SQL exceptions into metadata.
"""
exception_msg = str(exception)
- sql_line, sql_col = extract_sql_position(exception_msg)
+ sql_line, sql_col = _extract_sql_position(exception_msg)
# Truncate long SQL content
truncated_sql = sql_content
diff --git a/marimo/_sql/parse.py b/marimo/_sql/parse.py
new file mode 100644
index 0000000000..9208e9bb16
--- /dev/null
+++ b/marimo/_sql/parse.py
@@ -0,0 +1,137 @@
+# Copyright 2025 Marimo. All rights reserved.
+
+from typing import Literal, Optional, Union
+
+import msgspec
+
+from marimo import _loggers
+from marimo._dependencies.dependencies import DependencyManager
+
+LOGGER = _loggers.marimo_logger()
+
+
+class SqlParseError(msgspec.Struct):
+ """
+ Represents a single SQL parse error.
+
+ Attributes:
+ message (str): Description of the error.
+ line (int): Line number where the error occurred (1-based).
+ column (int): Column number where the error occurred (1-based).
+ severity (Literal["error", "warning"]): Severity of the error.
+ """
+
+ message: str
+ line: int
+ column: int
+ severity: Literal["error", "warning"]
+
+
+class SqlParseResult(msgspec.Struct):
+ """
+ Result of parsing an SQL query.
+
+ Attributes:
+ success (bool): True if parsing succeeded without errors.
+ errors (list[SqlParseError]): List of parse errors (empty if success is True).
+ """
+
+ success: bool
+ errors: list[SqlParseError]
+
+
+class SqlCatalogCheckResult(msgspec.Struct):
+ """
+ Result of running validation against the database.
+ """
+
+ success: bool
+ error_message: Optional[str]
+
+
+def parse_sql(
+ query: str, dialect: str
+) -> tuple[Optional[SqlParseResult], Optional[str]]:
+ """Parses an SQL query. Returns syntax errors.
+ Does not check for catalog errors (incorrect table names, etc).
+ Currently only supports DuckDB.
+
+ Args:
+ query (str): The SQL query to parse.
+ dialect (str): The dialect of the SQL query.
+
+ Returns:
+ tuple[SqlParseResult, str]: SqlParseResult and unexpected errors
+ """
+ dialect = dialect.strip().lower()
+
+ # Handle DuckDB
+ if "duckdb" in dialect:
+ return _parse_sql_duckdb(query)
+
+ return None, "Unsupported dialect: " + dialect
+
+
+# DuckDB
+
+
+class DuckDBParseError(msgspec.Struct):
+ error: bool
+ error_type: Optional[str] = None
+ error_message: Optional[str] = None
+ error_subtype: Optional[str] = None
+ position: Optional[Union[int, str]] = None
+
+
+# skip to reduce the response size
+# the response doesn't matter too much, we are interested in the errors
+JSON_SERIALIZE_QUERY = "SELECT JSON_SERIALIZE_SQL(CAST(? AS VARCHAR), skip_null := true, skip_empty := true, skip_default := true)"
+
+
+def _parse_sql_duckdb(
+ query: str,
+) -> tuple[Optional[SqlParseResult], Optional[str]]:
+ """Parse an SQL query using DuckDB. Returns parse result and unexpected errors.
+
+ Note:
+ - Only SELECT statements support json_serialize_sql
+ - Invalid function names do not throw errors
+ - Some syntax errors do not throw errors since they are not errors in the AST parser
+ """
+ DependencyManager.duckdb.require("to parse sql")
+
+ import duckdb
+
+ relation = duckdb.execute(JSON_SERIALIZE_QUERY, [query])
+ fetch_result = relation.fetchone()
+ if fetch_result is None:
+ return None, "No result from DuckDB parse query"
+
+ parse_response = fetch_result[0]
+ parsed_error = msgspec.json.decode(parse_response, type=DuckDBParseError)
+
+ if not parsed_error.error:
+ return SqlParseResult(success=True, errors=[]), None
+
+ if parsed_error.error_type == "not implemented":
+ # This is a valid query, but not supported by DuckDB
+ # Only SELECT statements support json_serialize_sql
+ return SqlParseResult(success=True, errors=[]), None
+
+ position = int(parsed_error.position or 0)
+ subquery = query[:position]
+ line_number = subquery.count("\n") + 1
+ column_number = position - subquery.rfind("\n") - 1
+
+ sql_parse_result = SqlParseResult(
+ success=False,
+ errors=[
+ SqlParseError(
+ message=parsed_error.error_message or "Syntax error in query",
+ line=line_number,
+ column=column_number,
+ severity="error",
+ )
+ ],
+ )
+ return sql_parse_result, None
diff --git a/packages/openapi/api.yaml b/packages/openapi/api.yaml
index 1ae97b810b..5b196a082e 100644
--- a/packages/openapi/api.yaml
+++ b/packages/openapi/api.yaml
@@ -3147,6 +3147,60 @@ components:
required: []
title: SnippetsConfig
type: object
+ SqlCatalogCheckResult:
+ description: Result of running validation against the database.
+ properties:
+ error_message:
+ anyOf:
+ - type: string
+ - type: 'null'
+ success:
+ type: boolean
+ required:
+ - success
+ - error_message
+ title: SqlCatalogCheckResult
+ type: object
+ SqlParseError:
+ description: "Represents a single SQL parse error.\n\nAttributes:\n message\
+ \ (str): Description of the error.\n line (int): Line number where the\
+ \ error occurred (1-based).\n column (int): Column number where the error\
+ \ occurred (1-based).\n severity (Literal[\"error\", \"warning\"]): Severity\
+ \ of the error."
+ properties:
+ column:
+ type: integer
+ line:
+ type: integer
+ message:
+ type: string
+ severity:
+ enum:
+ - error
+ - warning
+ required:
+ - message
+ - line
+ - column
+ - severity
+ title: SqlParseError
+ type: object
+ SqlParseResult:
+ description: "Result of parsing an SQL query.\n\nAttributes:\n success (bool):\
+ \ True if parsing succeeded without errors.\n errors (list[SqlParseError]):\
+ \ List of parse errors (empty if success is True)."
+ properties:
+ errors:
+ items:
+ $ref: '#/components/schemas/SqlParseError'
+ type: array
+ success:
+ type: boolean
+ required:
+ - success
+ - errors
+ title: SqlParseResult
+ type: object
StartupLogs:
properties:
content:
@@ -3286,18 +3340,28 @@ components:
title: UpdateComponentValuesRequest
type: object
ValidateSQLRequest:
- description: Validate an SQL query
+ description: Validate an SQL query against the engine
properties:
+ dialect:
+ anyOf:
+ - type: string
+ - type: 'null'
+ default: null
engine:
- type: string
+ anyOf:
+ - type: string
+ - type: 'null'
+ default: null
+ onlyParse:
+ type: boolean
query:
type: string
requestId:
type: string
required:
- requestId
- - engine
- query
+ - onlyParse
title: ValidateSQLRequest
type: object
ValidateSQLResult:
@@ -3310,12 +3374,17 @@ components:
op:
enum:
- validate-sql-result
+ parse_result:
+ anyOf:
+ - type: 'null'
+ - $ref: '#/components/schemas/SqlParseResult'
+ default: null
request_id:
type: string
- result:
+ validate_result:
anyOf:
- - {}
- type: 'null'
+ - $ref: '#/components/schemas/SqlCatalogCheckResult'
default: null
required:
- op
diff --git a/packages/openapi/src/api.ts b/packages/openapi/src/api.ts
index 9428c4c8b6..51badc1dd9 100644
--- a/packages/openapi/src/api.ts
+++ b/packages/openapi/src/api.ts
@@ -4322,6 +4322,43 @@ export interface components {
custom_paths?: string[];
include_default_snippets?: boolean;
};
+ /**
+ * SqlCatalogCheckResult
+ * @description Result of running validation against the database.
+ */
+ SqlCatalogCheckResult: {
+ error_message: string | null;
+ success: boolean;
+ };
+ /**
+ * SqlParseError
+ * @description Represents a single SQL parse error.
+ *
+ * Attributes:
+ * message (str): Description of the error.
+ * line (int): Line number where the error occurred (1-based).
+ * column (int): Column number where the error occurred (1-based).
+ * severity (Literal["error", "warning"]): Severity of the error.
+ */
+ SqlParseError: {
+ column: number;
+ line: number;
+ message: string;
+ /** @enum {unknown} */
+ severity: "error" | "warning";
+ };
+ /**
+ * SqlParseResult
+ * @description Result of parsing an SQL query.
+ *
+ * Attributes:
+ * success (bool): True if parsing succeeded without errors.
+ * errors (list[SqlParseError]): List of parse errors (empty if success is True).
+ */
+ SqlParseResult: {
+ errors: components["schemas"]["SqlParseError"][];
+ success: boolean;
+ };
/** StartupLogs */
StartupLogs: {
content: string;
@@ -4395,10 +4432,14 @@ export interface components {
};
/**
* ValidateSQLRequest
- * @description Validate an SQL query
+ * @description Validate an SQL query against the engine
*/
ValidateSQLRequest: {
- engine: string;
+ /** @default null */
+ dialect?: string | null;
+ /** @default null */
+ engine?: string | null;
+ onlyParse: boolean;
query: string;
requestId: string;
};
@@ -4408,9 +4449,11 @@ export interface components {
error?: string | null;
/** @enum {unknown} */
op: "validate-sql-result";
+ /** @default null */
+ parse_result?: null | components["schemas"]["SqlParseResult"];
request_id: string;
/** @default null */
- result?: unknown | null;
+ validate_result?: null | components["schemas"]["SqlCatalogCheckResult"];
};
/** VariableContext */
VariableContext: {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a9b093a2f1..1f6dfa7237 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -137,8 +137,8 @@ importers:
specifier: ^0.1.5
version: 0.1.5(@codemirror/[email protected])(@codemirror/[email protected])(@codemirror/[email protected])(@modelcontextprotocol/[email protected])
'@marimo-team/codemirror-sql':
- specifier: ^0.2.1
- version: 0.2.1(@codemirror/[email protected])(@codemirror/[email protected])
+ specifier: ^0.2.3
+ version: 0.2.3(@codemirror/[email protected])(@codemirror/[email protected])
'@marimo-team/llm-info':
specifier: workspace:*
version: link:../packages/llm-info
@@ -1804,8 +1804,8 @@ packages:
'@codemirror/view': ^6
'@modelcontextprotocol/sdk': ^1
- '@marimo-team/[email protected]':
- resolution: {integrity: sha512-k/5IwEOIdSUB7wVoazF5qqhogOwaCW63Wpfk0uQ0oc1r5pNolfC0h6ZdZuf+pIHzkWun2u2dg0rMc9UOFtzIHQ==}
+ '@marimo-team/[email protected]':
+ resolution: {integrity: sha512-8Zwalm/MND9aWQATrnnMsXsATowDdbknE/Y1j+D7mwO3+kIbJnkQcY5OO5Y6hiB1AoEJKkQ6qpVBwIu76Z8G7g==}
peerDependencies:
'@codemirror/state': ^6
'@codemirror/view': ^6
@@ -7383,8 +7383,8 @@ packages:
resolution: {integrity: sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==}
- resolution: {integrity: sha512-1ZcXu2qEFZlMkkaYPpMoTkypTunYTvtnInzVjWDhjc1+FS5h/WqT1rbgP2pP42/nXBWfPVN66mEfidWSpYVs1Q==}
+ resolution: {integrity: sha512-GQBwA2e44qjbK0MzFwh5bNYefniV6cKT4KfjNDpuh/2EWipUEK1BCMc//moSidp8EF6fHB/EqJwUH9GCh9MAPg==}
engines: {node: '>=8'}
@@ -11394,13 +11394,13 @@ snapshots:
'@codemirror/view': 6.38.2
'@modelcontextprotocol/sdk': 1.17.2
- '@marimo-team/[email protected](@codemirror/[email protected])(@codemirror/[email protected])':
+ '@marimo-team/[email protected](@codemirror/[email protected])(@codemirror/[email protected])':
dependencies:
'@codemirror/autocomplete': 6.18.7
'@codemirror/lint': 6.8.5
'@codemirror/state': 6.5.2
'@codemirror/view': 6.38.2
- node-sql-parser: 5.3.11
+ node-sql-parser: 5.3.12
dependencies:
@@ -18268,7 +18268,7 @@ snapshots:
dependencies:
'@types/pegjs': 0.10.6
big-integer: 1.6.52
diff --git a/pyproject.toml b/pyproject.toml
index 638a9907c2..293a0cc28d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -545,7 +545,7 @@ format_on_save = true
multi_column = true
performant_table_charts = true
sql_linter = true
-sql_mode = true
+sql_mode = false
[tool.marimo.display]
dataframes = "rich"
diff --git a/tests/_runtime/test_runtime_datasets.py b/tests/_runtime/test_runtime_datasets.py
new file mode 100644
index 0000000000..89e127b477
--- /dev/null
+++ b/tests/_runtime/test_runtime_datasets.py
@@ -0,0 +1,524 @@
+# Copyright 2025 Marimo. All rights reserved.
+
+from __future__ import annotations
+
+import pytest
+
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._messaging.ops import (
+ DataSourceConnections,
+ SQLTableListPreview,
+ SQLTablePreview,
+ ValidateSQLResult,
+)
+from marimo._runtime.requests import (
+ ExecutionRequest,
+ PreviewDataSourceConnectionRequest,
+ PreviewSQLTableListRequest,
+ PreviewSQLTableRequest,
+ ValidateSQLRequest,
+)
+from marimo._sql.engines.duckdb import INTERNAL_DUCKDB_ENGINE
+from marimo._sql.parse import SqlCatalogCheckResult, SqlParseResult
+from marimo._types.ids import CellId_t, RequestId
+from tests.conftest import MockedKernel
+
+HAS_SQL = DependencyManager.duckdb.has() and DependencyManager.polars.has()
+
+
+DUCKDB_CONN = "duckdb_conn"
+SQLITE_CONN = "sqlite_conn"
+
+
+async def connection_requests() -> list[ExecutionRequest]:
+ return [
+ ExecutionRequest(cell_id=CellId_t("0"), code="import duckdb"),
+ ExecutionRequest(
+ cell_id=CellId_t("1"),
+ code=f"{DUCKDB_CONN} = duckdb.connect(':memory:')",
+ ),
+ ExecutionRequest(cell_id=CellId_t("2"), code="import sqlite3"),
+ ExecutionRequest(
+ cell_id=CellId_t("3"),
+ code=f"{SQLITE_CONN} = sqlite3.connect(':memory:')",
+ ),
+ ]
+
+
+# @pytest.mark.skipif(not HAS_SQL, reason="SQL deps not available")
+# class TestGetSQLConnection:
+# async def test_non_existent_engine(
+# self, mocked_kernel: MockedKernel
+# ) -> None:
+# k = mocked_kernel.k
+
+# # Non-existent engine
+# k.get_sql_connection(DUCKDB_CONN)
+# assert k.get_sql_connection(DUCKDB_CONN) == (None, "Engine not found")
+
+# async def test_created_engine(
+# self,
+# mocked_kernel: MockedKernel,
+# connection_requests: list[ExecutionRequest],
+# ) -> None:
+# k = mocked_kernel.k
+
+# # Create duckdb and sqlite connections
+# await k.run(connection_requests)
+
+# connection, error = k.get_sql_connection("duckdb_conn")
+# assert connection is not None
+# assert error is None
+
+# # Test with SQLite engine (which is a QueryEngine, but not a EngineCatalog)
+# connection, error = k.get_sql_connection(SQLITE_CONN)
+# assert connection is not None
+# assert error is None
+
+
[email protected](not HAS_SQL, reason="SQL deps not available")
+class TestPreviewSQLTable:
+ async def test_non_existent_engine(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ preview_sql_table_request = PreviewSQLTableRequest(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ database="test",
+ schema="test",
+ table_name="t1",
+ )
+ await k.handle_message(preview_sql_table_request)
+
+ preview_sql_table_results = [
+ op for op in stream.operations if isinstance(op, SQLTablePreview)
+ ]
+ assert preview_sql_table_results == [
+ SQLTablePreview(
+ request_id=RequestId("0"), table=None, error="Engine not found"
+ )
+ ]
+
+ async def test_catalog_engine(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_sql_table_request = PreviewSQLTableRequest(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ database="test",
+ schema="test",
+ table_name="t1",
+ )
+ await k.handle_message(preview_sql_table_request)
+
+ preview_sql_table_results = [
+ op for op in stream.operations if isinstance(op, SQLTablePreview)
+ ]
+ assert preview_sql_table_results == [
+ SQLTablePreview(request_id=RequestId("0"), table=None, error=None)
+ ]
+
+ async def test_query_engine(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_sql_table_request = PreviewSQLTableRequest(
+ request_id=RequestId("0"),
+ engine=SQLITE_CONN,
+ database="test",
+ schema="test",
+ table_name="t1",
+ )
+ await k.handle_message(preview_sql_table_request)
+
+ preview_sql_table_results = [
+ op for op in stream.operations if isinstance(op, SQLTablePreview)
+ ]
+ assert preview_sql_table_results == [
+ SQLTablePreview(
+ request_id=RequestId("0"),
+ table=None,
+ error="Connection does not support catalog operations",
+ )
+ ]
+
+
[email protected](not HAS_SQL, reason="SQL deps not available")
+class TestPreviewSQLTableList:
+ async def test_non_existent_engine(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ preview_sql_table_list_request = PreviewSQLTableListRequest(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ database="test",
+ schema="test",
+ )
+ await k.handle_message(preview_sql_table_list_request)
+ preview_sql_table_list_results = [
+ op
+ for op in stream.operations
+ if isinstance(op, SQLTableListPreview)
+ ]
+ assert preview_sql_table_list_results == [
+ SQLTableListPreview(
+ request_id=RequestId("0"), tables=[], error="Engine not found"
+ )
+ ]
+
+ async def test_catalog_engine(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_sql_table_list_request = PreviewSQLTableListRequest(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ database="test",
+ schema="test",
+ )
+ await k.handle_message(preview_sql_table_list_request)
+
+ preview_sql_table_list_results = [
+ op
+ for op in stream.operations
+ if isinstance(op, SQLTableListPreview)
+ ]
+ assert preview_sql_table_list_results == [
+ SQLTableListPreview(
+ request_id=RequestId("0"), tables=[], error=None
+ )
+ ]
+
+ async def test_query_engine(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_sql_table_list_request = PreviewSQLTableListRequest(
+ request_id=RequestId("0"),
+ engine=SQLITE_CONN,
+ database="test",
+ schema="test",
+ )
+ await k.handle_message(preview_sql_table_list_request)
+
+ preview_sql_table_list_results = [
+ op
+ for op in stream.operations
+ if isinstance(op, SQLTableListPreview)
+ ]
+ assert preview_sql_table_list_results == [
+ SQLTableListPreview(
+ request_id=RequestId("0"),
+ tables=[],
+ error="Connection does not support catalog operations",
+ )
+ ]
+
+
+class TestPreviewDatasourceConnection:
+ async def test_non_existent_engine(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ preview_datasource_connection_request = (
+ PreviewDataSourceConnectionRequest(engine=DUCKDB_CONN)
+ )
+ await k.handle_message(preview_datasource_connection_request)
+ preview_datasource_connection_results = [
+ op
+ for op in stream.operations
+ if isinstance(op, DataSourceConnections)
+ ]
+ assert preview_datasource_connection_results == []
+
+ @pytest.mark.xfail(
+ reason="Should have only 2 connections (duckdb and sqlite)"
+ )
+ async def test_engines(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ preview_datasource_connection_request = (
+ PreviewDataSourceConnectionRequest(engine=DUCKDB_CONN)
+ )
+ await k.handle_message(preview_datasource_connection_request)
+
+ preview_datasource_connection_results = [
+ op
+ for op in stream.operations
+ if isinstance(op, DataSourceConnections)
+ ]
+ assert len(preview_datasource_connection_results) == 2
+
+
[email protected](not HAS_SQL, reason="SQL deps not available")
+class TestSQLValidate:
+ async def test_non_existent_engine(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ # Non-existent engine
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("0"),
+ engine=DUCKDB_CONN,
+ query="SELECT * from t1",
+ only_parse=False,
+ )
+ await k.handle_message(validate_sql_request)
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results == [
+ ValidateSQLResult(
+ request_id=RequestId("0"),
+ parse_result=None,
+ validate_result=None,
+ error="Failed to get engine duckdb_conn",
+ )
+ ]
+
+ async def test_internal_engine_and_valid_query(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ # Internal engine and valid query
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("1"),
+ engine=INTERNAL_DUCKDB_ENGINE,
+ query="SELECT 1, 2",
+ only_parse=False,
+ )
+ await k.handle_message(validate_sql_request)
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results[-1] == ValidateSQLResult(
+ request_id=RequestId("1"),
+ parse_result=SqlParseResult(success=True, errors=[]),
+ validate_result=SqlCatalogCheckResult(
+ success=True, error_message=None
+ ),
+ error=None,
+ )
+
+ async def test_internal_engine_and_invalid_query(
+ self, mocked_kernel: MockedKernel
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ # Internal engine and invalid query
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("2"),
+ engine=INTERNAL_DUCKDB_ENGINE,
+ query="SELECT * FROM t1",
+ only_parse=False,
+ )
+ await k.handle_message(validate_sql_request)
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ latest_validate_sql_result = validate_sql_results[-1]
+ assert latest_validate_sql_result.request_id == RequestId("2")
+
+ assert latest_validate_sql_result.parse_result is not None
+ # query is syntactically valid
+ assert latest_validate_sql_result.parse_result.success is True
+ assert len(latest_validate_sql_result.parse_result.errors) == 0
+
+ assert latest_validate_sql_result.validate_result is not None
+ assert latest_validate_sql_result.validate_result.success is False
+ assert (
+ latest_validate_sql_result.validate_result.error_message
+ is not None
+ )
+ assert latest_validate_sql_result.error is None
+
+ stream.operations.clear()
+
+ async def test_other_engine_and_valid_query(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ # Handle other engines
+ await k.run(connection_requests)
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("3"),
+ engine=SQLITE_CONN,
+ query="SELECT 1, 2",
+ only_parse=False,
+ )
+ await k.handle_message(validate_sql_request)
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert (
+ validate_sql_results[-1]
+ == ValidateSQLResult(
+ request_id=RequestId("3"),
+ parse_result=None, # Currently does not support parse errors for non-duckdb engines
+ validate_result=SqlCatalogCheckResult(
+ success=True, error_message=None
+ ),
+ error=None,
+ )
+ )
+
+ async def test_only_parse_with_no_dialect(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("4"),
+ engine=SQLITE_CONN,
+ query="SELECT 1, 2",
+ only_parse=True,
+ )
+ await k.handle_message(validate_sql_request)
+
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results[-1] == ValidateSQLResult(
+ request_id=RequestId("4"),
+ parse_result=None,
+ validate_result=None,
+ error="Dialect is required when only parsing",
+ )
+
+ async def test_only_parse_unsupported_dialect(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("5"),
+ dialect="sqlite",
+ query="SELECT 1, 2",
+ only_parse=True,
+ )
+ await k.handle_message(validate_sql_request)
+
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results[-1] == ValidateSQLResult(
+ request_id=RequestId("5"),
+ parse_result=None,
+ validate_result=None,
+ error="Unsupported dialect: sqlite",
+ )
+
+ async def test_only_parse_duckdb(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("6"),
+ dialect="duckdb",
+ query="SELECT 1, 2",
+ only_parse=True,
+ )
+ await k.handle_message(validate_sql_request)
+
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results[-1] == ValidateSQLResult(
+ request_id=RequestId("6"),
+ parse_result=SqlParseResult(success=True, errors=[]),
+ validate_result=None,
+ error=None,
+ )
+
+ async def test_validate_but_no_engine(
+ self,
+ mocked_kernel: MockedKernel,
+ connection_requests: list[ExecutionRequest],
+ ) -> None:
+ k = mocked_kernel.k
+ stream = mocked_kernel.stream
+
+ await k.run(connection_requests)
+
+ validate_sql_request = ValidateSQLRequest(
+ request_id=RequestId("7"),
+ query="SELECT 1, 2",
+ only_parse=False,
+ )
+ await k.handle_message(validate_sql_request)
+
+ validate_sql_results = [
+ op for op in stream.operations if isinstance(op, ValidateSQLResult)
+ ]
+ assert validate_sql_results[-1] == ValidateSQLResult(
+ request_id=RequestId("7"),
+ parse_result=None,
+ validate_result=None,
+ error="Engine is required for validating catalog",
+ )
diff --git a/tests/_server/api/endpoints/test_sql_endpoints.py b/tests/_server/api/endpoints/test_sql_endpoints.py
index e69509b79a..ac29b61279 100644
--- a/tests/_server/api/endpoints/test_sql_endpoints.py
+++ b/tests/_server/api/endpoints/test_sql_endpoints.py
@@ -24,6 +24,7 @@ def test_validate_sql(client: TestClient) -> None:
"requestId": "test_request_id",
"engine": "test_engine",
"query": "SELECT * FROM test",
+ "onlyParse": False,
},
)
assert response.status_code == 200, response.text
diff --git a/tests/_sql/test_engines.py b/tests/_sql/test_engines.py
index dfca9277cb..808712162b 100644
--- a/tests/_sql/test_engines.py
+++ b/tests/_sql/test_engines.py
@@ -316,32 +316,36 @@ def test_duckdb_execute_in_explain_mode():
not HAS_SQLALCHEMY or not (HAS_PANDAS or HAS_POLARS),
reason="SQLAlchemy and either pandas or polars not installed",
)
-def test_sqlalchemy_execute() -> None:
- """Test SQLAlchemyEngine execute method."""
- import sqlalchemy as sa
+class TestExecuteExplainModeSQLAlchemy:
+ @pytest.fixture
+ def engine(self) -> SQLAlchemyEngine:
+ import sqlalchemy as sa
- sqlite_engine = sa.create_engine("sqlite:///:memory:")
- engine = SQLAlchemyEngine(sqlite_engine)
+ sqlite_engine = sa.create_engine("sqlite:///:memory:")
+ engine = SQLAlchemyEngine(sqlite_engine)
+ return engine
- # Create a test table
- engine.execute("CREATE TABLE test (id INTEGER, name TEXT)")
- engine.execute("INSERT INTO test VALUES (1, 'test1'), (2, 'test2')")
+ def test_sqlalchemy_execute(self, engine: SQLAlchemyEngine) -> None:
+ """Test SQLAlchemyEngine execute method."""
+ # Create a test table
+ engine.execute("CREATE TABLE test (id INTEGER, name TEXT)")
+ engine.execute("INSERT INTO test VALUES (1, 'test1'), (2, 'test2')")
- # Query the table
- result = engine.execute("SELECT * FROM test ORDER BY id")
+ # Query the table
+ result = engine.execute("SELECT * FROM test ORDER BY id")
- # Check result type based on available libraries
- if HAS_POLARS:
- import polars as pl
+ # Check result type based on available libraries
+ if HAS_POLARS:
+ import polars as pl
- assert isinstance(result, pl.DataFrame)
- elif HAS_PANDAS:
- import pandas as pd
+ assert isinstance(result, pl.DataFrame)
+ elif HAS_PANDAS:
+ import pandas as pd
- assert isinstance(result, pd.DataFrame)
+ assert isinstance(result, pd.DataFrame)
- # Test with a query that doesn't return a result set
- assert engine.execute("PRAGMA journal_mode=WAL") is not None
+ # Test with a query that doesn't return a result set
+ assert engine.execute("PRAGMA journal_mode=WAL") is not None
@pytest.mark.skipif(not HAS_SQLALCHEMY, reason="SQLAlchemy not installed")
diff --git a/tests/_sql/test_sql_error_handling.py b/tests/_sql/test_sql_error_handling.py
index 24589616cb..046fd9912f 100644
--- a/tests/_sql/test_sql_error_handling.py
+++ b/tests/_sql/test_sql_error_handling.py
@@ -8,8 +8,8 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._messaging.errors import MarimoSQLError
from marimo._sql.error_utils import (
MarimoSQLException,
+ _extract_sql_position,
create_sql_error_from_exception,
- extract_sql_position,
is_sql_parse_error,
)
from marimo._sql.sql import sql
@@ -245,7 +245,7 @@ class TestErrorMessageQuality:
"""Test position extraction from DuckDB error messages."""
# DuckDB format: "Line 1, Col: 15"
duckdb_msg = "Parser Error: syntax error at Line 1, Col: 15"
- line, col = extract_sql_position(duckdb_msg)
+ line, col = _extract_sql_position(duckdb_msg)
assert line == 0 # 0-based
assert col == 14 # 0-based
@@ -253,14 +253,14 @@ class TestErrorMessageQuality:
"""Test position extraction from SQLGlot error messages."""
# SQLGlot format variations
sqlglot_msg = "Parse error at line 2, col 10"
- line, col = extract_sql_position(sqlglot_msg)
+ line, col = _extract_sql_position(sqlglot_msg)
assert line == 1 # 0-based
assert col == 9 # 0-based
def test_extract_sql_position_no_position(self):
"""Test position extraction when no position info available."""
no_position_msg = "Some generic SQL error without position"
- line, col = extract_sql_position(no_position_msg)
+ line, col = _extract_sql_position(no_position_msg)
assert line is None
assert col is None
diff --git a/tests/_sql/test_sql_parse.py b/tests/_sql/test_sql_parse.py
new file mode 100644
index 0000000000..6a397d61d8
--- /dev/null
+++ b/tests/_sql/test_sql_parse.py
@@ -0,0 +1,423 @@
+# Copyright 2025 Marimo. All rights reserved.
+
+from __future__ import annotations
+
+import pytest
+
+from marimo._dependencies.dependencies import DependencyManager
+from marimo._sql.parse import SqlParseError, SqlParseResult, parse_sql
+
+HAS_DUCKDB = DependencyManager.duckdb.has()
+
+
+def test_sql_parse_result_with_errors():
+ """Test SqlParseResult with errors."""
+ error = SqlParseError(
+ message="Syntax error", line=1, column=5, severity="error"
+ )
+ result = SqlParseResult(success=False, errors=[error])
+
+ assert result.success is False
+ assert len(result.errors) == 1
+ assert result.errors[0].message == "Syntax error"
+
+
+class TestUnsupportedDialects:
+ """Test handling of unsupported SQL dialects."""
+
+ @pytest.mark.parametrize(
+ "dialect",
+ [
+ "postgresql",
+ "mysql",
+ "sqlite",
+ "oracle",
+ "sqlserver",
+ "bigquery",
+ "snowflake",
+ "redshift",
+ "",
+ "unknown_dialect",
+ ],
+ )
+ def test_unsupported_dialects_return_none(self, dialect: str):
+ """Test that unsupported dialects return none."""
+ result, error = parse_sql("SELECT * FROM table", dialect)
+ assert result is None
+ assert error == "Unsupported dialect: " + dialect
+
+ def test_dialect_with_whitespace(self):
+ """Test dialects with leading/trailing whitespace."""
+ result, error = parse_sql("SELECT 1", " postgresql ")
+
+ assert result is None
+ assert error == "Unsupported dialect: postgresql"
+
+ result, error = parse_sql("SELECT 1", " duckdb ")
+ assert result is not None
+ assert error is None
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+class TestDuckDBValidQueries:
+ """Test parsing of valid SQL queries with DuckDB."""
+
+ @pytest.mark.parametrize(
+ "query",
+ [
+ "SELECT 1",
+ "SELECT 1 as value",
+ "SELECT 'hello' as greeting",
+ "SELECT 1, 2, 3",
+ "SELECT 1 as a, 2 as b, 3 as c",
+ "SELECT 1 UNION SELECT 2",
+ "SELECT CASE WHEN 1 > 0 THEN 'positive' ELSE 'zero or negative' END",
+ "SELECT COUNT(*)",
+ "SELECT SUM(1), AVG(1), MAX(1), MIN(1)",
+ "SELECT 1 ORDER BY 1",
+ "SELECT 1 LIMIT 10",
+ ],
+ )
+ def test_valid_queries_return_success(self, query: str):
+ """Test that valid SQL queries return successful parse results."""
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert isinstance(result, SqlParseResult)
+ assert result.success is True
+ assert result.errors == []
+
+ @pytest.mark.parametrize(
+ "dialect",
+ [
+ "duckdb",
+ "DUCKDB",
+ "DuckDB",
+ " duckdb ",
+ ],
+ )
+ def test_duckdb_dialect_variations(self, dialect: str):
+ """Test that various DuckDB dialect strings work."""
+ result, error = parse_sql("SELECT * FROM", dialect)
+ assert result is not None
+ assert error is None
+
+ assert result.success is False
+
+ def test_multiline_valid_query(self):
+ """Test parsing multiline valid queries."""
+ query = """
+ SELECT
+ 1 as id,
+ 'test' as name,
+ true as active
+ ORDER BY id DESC
+ """
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert result.success is True
+ assert result.errors == []
+
+ def test_query_with_comments(self):
+ """Test parsing queries with SQL comments."""
+ query = """
+ -- This is a comment
+ SELECT 1 as test_column; -- End line comment
+ """
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert result.success is True
+ assert result.errors == []
+
+ def test_complex_valid_query(self):
+ """Test parsing a complex but valid query."""
+ query = """
+ WITH summary AS (
+ SELECT
+ 1 as id,
+ 100 as total_quantity,
+ 50.5 as avg_price
+ )
+ SELECT
+ id,
+ total_quantity,
+ avg_price,
+ CASE
+ WHEN total_quantity > 100 THEN 'High Volume'
+ WHEN total_quantity > 50 THEN 'Medium Volume'
+ ELSE 'Low Volume'
+ END as volume_category
+ FROM summary
+ ORDER BY total_quantity DESC
+ """
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert result.success is True
+ assert result.errors == []
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+class TestDuckDBInvalidQueries:
+ """Test parsing of invalid SQL queries with DuckDB."""
+
+ @pytest.mark.parametrize(
+ ("query", "expected_error_keywords"),
+ [
+ ("SELECT * FRM table", ["syntax", "error"]),
+ ("SELECT (", ["syntax", "error"]),
+ ("SELECT * FROM", ["syntax", "error"]),
+ ("SELECT ,", ["syntax", "error"]),
+ ("SELEC * FROM table", ["syntax", "error"]),
+ ("SELECT * FROM table WHERE", ["syntax", "error"]),
+ ("SELECT * FROM table ORDER", ["syntax", "error"]),
+ ("INSERT INTO", ["syntax", "error"]),
+ ("UPDATE", ["syntax", "error"]),
+ ("DELETE", ["syntax", "error"]),
+ # Unclosed parentheses
+ ("SELECT COUNT(*", ["syntax", "error"]),
+ # Unclosed quotes
+ # ("SELECT 'unclosed string", ["syntax", "error"]), # DuckDB does not raise errors for this case
+ # # Invalid function call
+ ("SELECT COUNT(DISTINCT)", ["syntax", "error"]),
+ # # Malformed CASE statement
+ ("SELECT CASE WHEN FROM table", ["syntax", "error"]),
+ # # Incomplete JOIN
+ ("SELECT * FROM table1 JOIN", ["syntax", "error"]),
+ ],
+ )
+ def test_invalid_queries_return_errors(
+ self, query: str, expected_error_keywords: list[str]
+ ):
+ """Test that invalid SQL queries return error results."""
+ result, error = parse_sql(query, "duckdb")
+
+ assert isinstance(result, SqlParseResult)
+ assert result.success is False
+ assert len(result.errors) > 0
+
+ error = result.errors[0]
+ assert isinstance(error, SqlParseError)
+ assert error.severity == "error"
+ assert error.line > 0
+ assert error.column >= 0
+
+ # Check that error message contains expected keywords
+ error_msg_lower = error.message.lower()
+ assert any(
+ keyword in error_msg_lower for keyword in expected_error_keywords
+ )
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+class TestErrorPositionCalculation:
+ """Test that error positions (line and column) are calculated correctly."""
+
+ def test_single_line_error_position(self):
+ """Test position calculation for single-line queries."""
+ query = "SELECT * FRM table" # Error at position of "FRM"
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert result.success is False
+ assert len(result.errors) == 1
+
+ error = result.errors[0]
+ assert error.line == 1
+ assert error.column == 13
+
+ @pytest.mark.xfail(reason="DuckDB does not raise errors for this case")
+ def test_multiline_error_position(self):
+ """Test position calculation for multiline queries."""
+ query = """SELECT name,
+ email,
+ FRM users""" # Error on line 3
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+
+ assert result.success is False
+ assert len(result.errors) == 1
+
+ error = result.errors[0]
+ assert error.line == 3
+ assert error.column == 0
+
+ def test_error_at_beginning_of_line(self):
+ """Test position calculation when error is at beginning of line."""
+ query = """SELECT *
+FRM table"""
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+
+ assert result.success is False
+ assert len(result.errors) == 1
+
+ error = result.errors[0]
+ assert error.line == 2
+ assert error.column == 4
+
+ def test_error_with_leading_whitespace(self):
+ """Test position calculation with leading whitespace."""
+ query = """ SELECT *
+ FRM table"""
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+
+ assert result.success is False
+ assert len(result.errors) == 1
+
+ error = result.errors[0]
+ assert error.line == 2
+
+ @pytest.mark.xfail(
+ reason="DuckDB does not raise errors for invalid syntax"
+ )
+ def test_error_position_after_newlines(self):
+ """Test position calculation with multiple newlines."""
+ query = """
+ SELECT name
+ FROM users
+ WHERE invalid_syntax"""
+
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert result.success is False
+ assert len(result.errors) == 1
+
+ error = result.errors[0]
+ assert error.line == 3
+ assert error.column >= 0
+
+
[email protected](not HAS_DUCKDB, reason="DuckDB not installed")
+class TestEdgeCases:
+ """Test edge cases and boundary conditions."""
+
+ def test_empty_query(self):
+ """Test parsing empty query."""
+ result, error = parse_sql("", "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert isinstance(result, SqlParseResult)
+ assert result.success is True
+
+ def test_whitespace_only_query(self):
+ """Test parsing query with only whitespace."""
+ result, error = parse_sql(" \n \t ", "duckdb")
+ assert result is not None
+ assert error is None
+
+ assert isinstance(result, SqlParseResult)
+ assert result.success is True
+ assert result.errors == []
+
+ def test_query_with_semicolon(self):
+ """Test parsing query with trailing semicolon."""
+ result, error = parse_sql("SELECT 1;", "duckdb")
+ assert result is not None
+
+ assert result.success is True
+ assert result.errors == []
+
+ def test_multiple_statements(self):
+ """Test parsing multiple SQL statements."""
+ query = "SELECT 1; SELECT 2;"
+
+ result, error = parse_sql(query, "duckdb")
+
+ # DuckDB might handle this differently, just ensure we get a result
+ assert isinstance(result, SqlParseResult)
+
+ def test_very_long_query(self):
+ """Test parsing a very long query."""
+ # Create a query with many columns
+ columns = ", ".join([f"{i} as col_{i}" for i in range(100)])
+ query = f"SELECT {columns}"
+
+ result, error = parse_sql(query, "duckdb")
+ assert error is None
+
+ assert result is not None
+ assert result.success is True
+ assert result.errors == []
+
+ def test_query_with_unicode(self):
+ """Test parsing query with Unicode characters."""
+ query = "SELECT 'Hello 世界' as greeting"
+
+ result, error = parse_sql(query, "duckdb")
+ assert error is None
+
+ assert result is not None
+ assert result.success is True
+ assert result.errors == []
+
+ def test_query_with_special_characters(self):
+ """Test parsing query with various special characters."""
+ query = "SELECT 'test@#$%^&*()' as special_chars"
+
+ result, error = parse_sql(query, "duckdb")
+ assert error is None
+
+ assert result is not None
+ assert result.success is True
+ assert result.errors == []
+
+ def test_deeply_nested_query(self):
+ """Test parsing deeply nested subqueries."""
+ query = """
+ SELECT * FROM (
+ SELECT * FROM (
+ SELECT * FROM (
+ SELECT 1 as nested_value
+ ) inner_query
+ ) middle_query
+ ) outer_query
+ """
+
+ result, error = parse_sql(query, "duckdb")
+ assert error is None
+
+ assert result is not None
+ assert result.success is True
+ assert result.errors == []
+
+ def test_just_comments(self):
+ """Test parsing query with just comments."""
+ for query in [
+ """
+ -- This is a comment
+ /* This is a comment */
+ """,
+ """-- SELECT 1""",
+ """/* SELECT 1 */""",
+ """
+ -- This is a comment
+ SELECT 1 as test_column; -- End line comment
+ """,
+ """
+ /* This is a comment */
+ SELECT 1 as test_column; /* End line comment */
+ """,
+ ]:
+ result, error = parse_sql(query, "duckdb")
+ assert result is not None
+ assert result.success is True
+ assert result.errors == []