Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

Fix SQL cell deps lost when duckdb fails to parse normalized SQL (#8437)

Trevor Manz7d658febaafd
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.py
index f4ac96f37b..ec92a89c3a 100644
--- a/marimo/_ast/sql_visitor.py
+++ b/marimo/_ast/sql_visitor.py
@@ -526,9 +526,13 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:
table=table_name, schema=schema_name, catalog=catalog_name
)
- # May raise a ParseError
- with _loggers.suppress_warnings_logs("sqlglot"):
- expression_list = parse(sql_statement, dialect="duckdb")
+ from sqlglot.errors import ParseError
+
+ try:
+ with _loggers.suppress_warnings_logs("sqlglot"):
+ expression_list = parse(sql_statement, dialect="duckdb")
+ except ParseError:
+ return set()
refs: set[SQLRef] = set()
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.py
index aff41cbded..a70d17fb31 100644
--- a/marimo/_ast/visitor.py
+++ b/marimo/_ast/visitor.py
@@ -655,17 +655,6 @@ class ScopedVisitor(ast.NodeVisitor):
):
import duckdb # type: ignore[import-not-found,import-untyped,unused-ignore] # noqa: E501
- # Import ParseError outside try block so we can except it
- # Use a Union approach to handle both cases
- ParseErrorType: type[Exception]
- if DependencyManager.sqlglot.has():
- from sqlglot.errors import ParseError as SQLGLOTParseError
-
- ParseErrorType = SQLGLOTParseError
- else:
- # Fallback when sqlglot is not available
- ParseErrorType = Exception
-
# TODO: Handle other SQL languages
# TODO: Get the engine so we can differentiate tables in diff engines
@@ -681,10 +670,13 @@ class ScopedVisitor(ast.NodeVisitor):
statements = duckdb.extract_statements(sql)
except (duckdb.ProgrammingError, duckdb.IOException):
# The user's sql query may have a syntax error,
- # or duckdb failed for an unknown reason; don't
- # break marimo.
- self.generic_visit(node)
- return node
+ # or duckdb failed for an unknown reason.
+ #
+ # Don't bail out, and instead fall through to use sqlglot
+ # for ref extraction on the full SQL string. This handles
+ # cases where f-string placeholders produce SQL that
+ # duckdb rejects but sqlglot can still parse.
+ statements = []
except BaseException as e:
# We catch base exceptions because we don't want to
# fail due to bugs in duckdb -- users code should
@@ -697,8 +689,7 @@ class ScopedVisitor(ast.NodeVisitor):
rule_code="MF005",
sql_content=sql,
)
- self.generic_visit(node)
- return node
+ statements = []
# Try to process each statement individually
# For some SQL types (e.g., PIVOT with certain clauses),
@@ -768,25 +759,7 @@ class ScopedVisitor(ast.NodeVisitor):
self._define(None, _catalog, VariableData("catalog"))
defined_names.add(_catalog)
- sql_refs: set[SQLRef] = set()
- try:
- # Take results
- sql_refs = find_sql_refs_cached(statement_sql)
- except (
- duckdb.ProgrammingError,
- duckdb.IOException,
- ParseErrorType,
- BaseException,
- ) as e:
- # Use first_arg (SQL string node) for accurate positioning
- log_sql_error(
- LOGGER.error,
- message=f"Error parsing SQL statement: {e}",
- exception=e,
- node=first_arg,
- rule_code="MF005",
- sql_content=statement_sql,
- )
+ sql_refs = find_sql_refs_cached(statement_sql)
for ref in sql_refs:
name = ref.qualified_name
diff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.py
index 33d2374626..bebd4f4eae 100644
--- a/tests/_ast/test_sql_visitor.py
+++ b/tests/_ast/test_sql_visitor.py
@@ -788,10 +788,7 @@ class TestFindSQLRefs:
def test_invalid_sql(self) -> None:
sql = "SELECT * FROM"
- from sqlglot import ParseError
-
- with pytest.raises(ParseError):
- find_sql_refs(sql)
+ assert find_sql_refs(sql) == set()
def test_dml_with_subquery(self) -> None:
# first
diff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.py
index 3c42b7a5f6..d69abd2bcb 100644
--- a/tests/_ast/test_visitor.py
+++ b/tests/_ast/test_visitor.py
@@ -1485,7 +1485,9 @@ def test_sql_statement_with_function() -> None:
def test_unparsable_sql_doesnt_fail() -> None:
code = "\n".join(
[
- # duckdb will raise a BinderError, but codegen shouldnt fail
+ # duckdb will raise a BinderError, but codegen shouldnt fail.
+ # sqlglot can still extract table refs even when duckdb can't
+ # parse the SQL.
"df = mo.sql('select * from cars where cars.foo = ANY({bar})')",
]
)
@@ -1493,7 +1495,7 @@ def test_unparsable_sql_doesnt_fail() -> None:
mod = ast.parse(code)
v.visit(mod)
assert v.defs == set(["df"])
- assert v.refs == set(["mo"])
+ assert v.refs == set(["mo", "cars"])
@pytest.mark.skipif(not HAS_DEPS, reason="Requires duckdb")
@@ -1564,6 +1566,22 @@ def test_sql_table_f_string() -> None:
assert v.refs == set(["mo", "my_table", "lim"])
[email protected](not HAS_DEPS, reason="Requires duckdb")
+def test_sql_join_using_f_string() -> None:
+ # Regression test for https://github.com/marimo-team/marimo/issues/8436
+ # When an f-string expression appears in a USING clause, the placeholder
+ # "1" makes duckdb.extract_statements fail, which used to cause all SQL
+ # table references to be lost.
+ code = (
+ "mo.sql(f'SELECT {col}, colC, colD FROM df1 JOIN df2 USING ({col})')"
+ )
+ v = visitor.ScopedVisitor()
+ mod = ast.parse(code)
+ v.visit(mod)
+ assert not v.defs
+ assert v.refs == set(["mo", "df1", "df2", "col"])
+
+
@pytest.mark.skipif(not HAS_DEPS, reason="Requires duckdb")
@pytest.mark.parametrize(
("description", "sql_statement", "expected_defs", "expected_refs"),
@@ -1686,11 +1704,14 @@ def test_sql_defined_in_same_statement(
reason="We still set the schema as defined even if it is deleted in the same statement"
),
),
- (
+ pytest.param(
"attach and detach catalog in same statement",
'attach "my_catalog.db" as my_catalog; detach my_catalog;',
set(),
{"mo"},
+ marks=pytest.mark.xfail(
+ reason="We still set the catalog as defined even if it is detached in the same statement"
+ ),
),
],
)