A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
Fix SQL cell deps lost when duckdb fails to parse normalized SQL (#8437)
Trevor Manz7d658febaafd
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.pyindex f4ac96f37b..ec92a89c3a 100644--- a/marimo/_ast/sql_visitor.py+++ b/marimo/_ast/sql_visitor.py@@ -526,9 +526,13 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:table=table_name, schema=schema_name, catalog=catalog_name)- # May raise a ParseError- with _loggers.suppress_warnings_logs("sqlglot"):- expression_list = parse(sql_statement, dialect="duckdb")+ from sqlglot.errors import ParseError++ try:+ with _loggers.suppress_warnings_logs("sqlglot"):+ expression_list = parse(sql_statement, dialect="duckdb")+ except ParseError:+ return set()refs: set[SQLRef] = set()diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.pyindex aff41cbded..a70d17fb31 100644--- a/marimo/_ast/visitor.py+++ b/marimo/_ast/visitor.py@@ -655,17 +655,6 @@ class ScopedVisitor(ast.NodeVisitor):):import duckdb # type: ignore[import-not-found,import-untyped,unused-ignore] # noqa: E501- # Import ParseError outside try block so we can except it- # Use a Union approach to handle both cases- ParseErrorType: type[Exception]- if DependencyManager.sqlglot.has():- from sqlglot.errors import ParseError as SQLGLOTParseError-- ParseErrorType = SQLGLOTParseError- else:- # Fallback when sqlglot is not available- ParseErrorType = Exception-# TODO: Handle other SQL languages# TODO: Get the engine so we can differentiate tables in diff engines@@ -681,10 +670,13 @@ class ScopedVisitor(ast.NodeVisitor):statements = duckdb.extract_statements(sql)except (duckdb.ProgrammingError, duckdb.IOException):# The user's sql query may have a syntax error,- # or duckdb failed for an unknown reason; don't- # break marimo.- self.generic_visit(node)- return node+ # or duckdb failed for an unknown reason.+ #+ # Don't bail out, and instead fall through to use sqlglot+ # for ref extraction on the full SQL string. This handles+ # cases where f-string placeholders produce SQL that+ # duckdb rejects but sqlglot can still parse.+ statements = []except BaseException as e:# We catch base exceptions because we don't want to# fail due to bugs in duckdb -- users code should@@ -697,8 +689,7 @@ class ScopedVisitor(ast.NodeVisitor):rule_code="MF005",sql_content=sql,)- self.generic_visit(node)- return node+ statements = []# Try to process each statement individually# For some SQL types (e.g., PIVOT with certain clauses),@@ -768,25 +759,7 @@ class ScopedVisitor(ast.NodeVisitor):self._define(None, _catalog, VariableData("catalog"))defined_names.add(_catalog)- sql_refs: set[SQLRef] = set()- try:- # Take results- sql_refs = find_sql_refs_cached(statement_sql)- except (- duckdb.ProgrammingError,- duckdb.IOException,- ParseErrorType,- BaseException,- ) as e:- # Use first_arg (SQL string node) for accurate positioning- log_sql_error(- LOGGER.error,- message=f"Error parsing SQL statement: {e}",- exception=e,- node=first_arg,- rule_code="MF005",- sql_content=statement_sql,- )+ sql_refs = find_sql_refs_cached(statement_sql)for ref in sql_refs:name = ref.qualified_namediff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.pyindex 33d2374626..bebd4f4eae 100644--- a/tests/_ast/test_sql_visitor.py+++ b/tests/_ast/test_sql_visitor.py@@ -788,10 +788,7 @@ class TestFindSQLRefs:def test_invalid_sql(self) -> None:sql = "SELECT * FROM"- from sqlglot import ParseError-- with pytest.raises(ParseError):- find_sql_refs(sql)+ assert find_sql_refs(sql) == set()def test_dml_with_subquery(self) -> None:# firstdiff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.pyindex 3c42b7a5f6..d69abd2bcb 100644--- a/tests/_ast/test_visitor.py+++ b/tests/_ast/test_visitor.py@@ -1485,7 +1485,9 @@ def test_sql_statement_with_function() -> None:def test_unparsable_sql_doesnt_fail() -> None:code = "\n".join([- # duckdb will raise a BinderError, but codegen shouldnt fail+ # duckdb will raise a BinderError, but codegen shouldnt fail.+ # sqlglot can still extract table refs even when duckdb can't+ # parse the SQL."df = mo.sql('select * from cars where cars.foo = ANY({bar})')",])@@ -1493,7 +1495,7 @@ def test_unparsable_sql_doesnt_fail() -> None:mod = ast.parse(code)v.visit(mod)assert v.defs == set(["df"])- assert v.refs == set(["mo"])+ assert v.refs == set(["mo", "cars"])@pytest.mark.skipif(not HAS_DEPS, reason="Requires duckdb")@@ -1564,6 +1566,22 @@ def test_sql_table_f_string() -> None:assert v.refs == set(["mo", "my_table", "lim"])[email protected](not HAS_DEPS, reason="Requires duckdb")+def test_sql_join_using_f_string() -> None:+ # Regression test for https://github.com/marimo-team/marimo/issues/8436+ # When an f-string expression appears in a USING clause, the placeholder+ # "1" makes duckdb.extract_statements fail, which used to cause all SQL+ # table references to be lost.+ code = (+ "mo.sql(f'SELECT {col}, colC, colD FROM df1 JOIN df2 USING ({col})')"+ )+ v = visitor.ScopedVisitor()+ mod = ast.parse(code)+ v.visit(mod)+ assert not v.defs+ assert v.refs == set(["mo", "df1", "df2", "col"])++@pytest.mark.skipif(not HAS_DEPS, reason="Requires duckdb")@pytest.mark.parametrize(("description", "sql_statement", "expected_defs", "expected_refs"),@@ -1686,11 +1704,14 @@ def test_sql_defined_in_same_statement(reason="We still set the schema as defined even if it is deleted in the same statement"),),- (+ pytest.param("attach and detach catalog in same statement",'attach "my_catalog.db" as my_catalog; detach my_catalog;',set(),{"mo"},+ marks=pytest.mark.xfail(+ reason="We still set the catalog as defined even if it is detached in the same statement"+ ),),],)