Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: exclude DML CTE aliases from SQL dependencies (#11347)

Shahmir Varqhad0b73b931fc3
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.py
index e84a19e40f..b3ffacc2f3 100644
--- a/marimo/_ast/sql_visitor.py
+++ b/marimo/_ast/sql_visitor.py
@@ -545,42 +545,49 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:
refs: set[SQLRef] = set()
+ def is_cte_reference(table: exp.Table) -> bool:
+ if table.db or table.catalog:
+ return False
+
+ target = (
+ table.parent if isinstance(table.parent, exp.Schema) else table
+ )
+ if (
+ isinstance(target.parent, (exp.Insert, exp.Update, exp.Delete))
+ and target.parent.this is target
+ ):
+ return False
+
+ table_name = table.name.lower()
+ node: exp.Expr = table
+ while parent := node.parent:
+ with_clause = (
+ parent
+ if isinstance(parent, exp.With)
+ else parent.args.get("with_")
+ )
+ if isinstance(with_clause, exp.With) and node is not with_clause:
+ for cte in with_clause.expressions:
+ if cte is node and not with_clause.args.get("recursive"):
+ break
+ if cte.alias.lower() == table_name:
+ return True
+ if cte is node:
+ break
+ node = parent
+ return False
+
def _collect_table_refs_excluding_ctes(expression: exp.Expression) -> None:
- """Walk all Table nodes, filtering out unqualified CTE references.
-
- find_all(exp.Table) doesn't understand CTE scope, so bare
- references to CTE names would be misidentified as real tables.
-
- We only collect CTEs from the statement-level WITH clause
- rather than nested subqueries, because a subquery's CTE is
- scoped to that subquery and must not mask a real table with the
- same name in the outer query. We identify statement-level CTEs
- by checking that the CTE's grandparent (With -> Expression) is
- the top-level expression. Schema-qualified refs (e.g. schema.foo)
- are always real tables even if a CTE shares the same base name.
- """
- cte_names: set[str] = set()
- for cte in expression.find_all(exp.CTE):
- with_node = cte.parent
- if with_node and with_node.parent is expression:
- alias = cte.alias
- if alias:
- cte_names.add(alias.lower())
for table in expression.find_all(exp.Table):
- if ref := get_ref_from_table(table):
- is_unqualified_cte = (
- ref.table.lower() in cte_names
- and ref.schema is None
- and ref.catalog is None
- )
- if not is_unqualified_cte:
+ if not is_cte_reference(table):
+ if ref := get_ref_from_table(table):
refs.add(ref)
for expression in expression_list:
if expression is None:
continue
- if bool(
+ requires_table_scan = bool(
expression.find(
exp.Update,
exp.Insert,
@@ -593,7 +600,8 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:
exp.TruncateTable,
exp.Copy,
)
- ):
+ )
+ if requires_table_scan:
_collect_table_refs_excluding_ctes(expression) # type: ignore[arg-type]
# build_scope only works for select statements.
@@ -604,7 +612,12 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:
if root := build_scope(expression):
for scope in root.traverse(): # type: ignore
for _node, source in scope.selected_sources.values():
+ # SQLGlot can expose DML CTE references as plain tables.
if isinstance(source, exp.Table):
+ if requires_table_scan and is_cte_reference(
+ source
+ ):
+ continue
if ref := get_ref_from_table(source):
refs.add(ref)
except OptimizeError:
diff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.py
index 48a465ece1..192e11a7b1 100644
--- a/tests/_ast/test_sql_visitor.py
+++ b/tests/_ast/test_sql_visitor.py
@@ -1168,6 +1168,89 @@ class TestFindSQLRefs:
SQLRef(table="target_table"),
}
+ @pytest.mark.parametrize(
+ "statement",
+ [
+ "INSERT INTO items VALUES (1) RETURNING *",
+ "UPDATE items SET id = 2 RETURNING *",
+ "DELETE FROM items RETURNING *",
+ ],
+ )
+ def test_dml_cte_reference(self, statement: str) -> None:
+ sql = f"WITH changed AS ({statement}) SELECT * FROM changed"
+ assert find_sql_refs(sql) == {SQLRef(table="items")}
+
+ @pytest.mark.parametrize(
+ "statement",
+ [
+ "INSERT INTO changed (id) VALUES (1) RETURNING *",
+ "UPDATE changed SET id = 2 RETURNING *",
+ "DELETE FROM changed RETURNING *",
+ ],
+ )
+ def test_dml_cte_name_matches_target(self, statement: str) -> None:
+ sql = f"WITH changed AS ({statement}) SELECT * FROM changed"
+ assert find_sql_refs(sql) == {SQLRef(table="changed")}
+
+ def test_nested_dml_cte_scope(self) -> None:
+ sql = """
+ SELECT * FROM changed
+ JOIN (
+ WITH changed AS (INSERT INTO items VALUES (1) RETURNING *),
+ local_rows AS (SELECT * FROM changed)
+ SELECT * FROM local_rows
+ ) sub ON changed.id = sub.id
+ """
+ assert find_sql_refs(sql) == {
+ SQLRef(table="changed"),
+ SQLRef(table="items"),
+ }
+
+ def test_dml_cte_schema_qualified_reference(self) -> None:
+ sql = """
+ WITH changed AS (DELETE FROM items RETURNING *)
+ SELECT * FROM archive.changed a JOIN changed c ON a.id = c.id
+ """
+ assert find_sql_refs(sql) == {
+ SQLRef(table="items"),
+ SQLRef(table="changed", schema="archive"),
+ }
+
+ def test_dml_cte_reads_table_with_same_name(self) -> None:
+ sql = """
+ WITH changed AS (
+ INSERT INTO items SELECT * FROM changed RETURNING *
+ )
+ SELECT * FROM changed
+ """
+ assert find_sql_refs(sql) == {
+ SQLRef(table="items"),
+ SQLRef(table="changed"),
+ }
+
+ def test_dml_cte_with_recursive_source(self) -> None:
+ sql = """
+ WITH RECURSIVE nums AS (
+ SELECT 1 AS id
+ UNION ALL
+ SELECT id + 1 FROM nums WHERE id < 3
+ ), changed AS (INSERT INTO items SELECT * FROM nums RETURNING *)
+ SELECT * FROM changed
+ """
+ assert find_sql_refs(sql) == {SQLRef(table="items")}
+
+ def test_dml_cte_cannot_reference_later_recursive_cte(self) -> None:
+ sql = """
+ WITH RECURSIVE changed AS (
+ INSERT INTO items SELECT * FROM later RETURNING *
+ ), later AS (SELECT 1 AS id)
+ SELECT * FROM changed
+ """
+ assert find_sql_refs(sql) == {
+ SQLRef(table="items"),
+ SQLRef(table="later"),
+ }
+
def test_multiple_statements_with_optimize_error(self) -> None:
# Verify that OptimizeError in one statement doesn't affect others.
# The try/except is inside the loop, so each statement is independent.