A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix: exclude DML CTE aliases from SQL dependencies (#11347)
Shahmir Varqhad0b73b931fc3
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.pyindex e84a19e40f..b3ffacc2f3 100644--- a/marimo/_ast/sql_visitor.py+++ b/marimo/_ast/sql_visitor.py@@ -545,42 +545,49 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:refs: set[SQLRef] = set()+ def is_cte_reference(table: exp.Table) -> bool:+ if table.db or table.catalog:+ return False++ target = (+ table.parent if isinstance(table.parent, exp.Schema) else table+ )+ if (+ isinstance(target.parent, (exp.Insert, exp.Update, exp.Delete))+ and target.parent.this is target+ ):+ return False++ table_name = table.name.lower()+ node: exp.Expr = table+ while parent := node.parent:+ with_clause = (+ parent+ if isinstance(parent, exp.With)+ else parent.args.get("with_")+ )+ if isinstance(with_clause, exp.With) and node is not with_clause:+ for cte in with_clause.expressions:+ if cte is node and not with_clause.args.get("recursive"):+ break+ if cte.alias.lower() == table_name:+ return True+ if cte is node:+ break+ node = parent+ return False+def _collect_table_refs_excluding_ctes(expression: exp.Expression) -> None:- """Walk all Table nodes, filtering out unqualified CTE references.-- find_all(exp.Table) doesn't understand CTE scope, so bare- references to CTE names would be misidentified as real tables.-- We only collect CTEs from the statement-level WITH clause- rather than nested subqueries, because a subquery's CTE is- scoped to that subquery and must not mask a real table with the- same name in the outer query. We identify statement-level CTEs- by checking that the CTE's grandparent (With -> Expression) is- the top-level expression. Schema-qualified refs (e.g. schema.foo)- are always real tables even if a CTE shares the same base name.- """- cte_names: set[str] = set()- for cte in expression.find_all(exp.CTE):- with_node = cte.parent- if with_node and with_node.parent is expression:- alias = cte.alias- if alias:- cte_names.add(alias.lower())for table in expression.find_all(exp.Table):- if ref := get_ref_from_table(table):- is_unqualified_cte = (- ref.table.lower() in cte_names- and ref.schema is None- and ref.catalog is None- )- if not is_unqualified_cte:+ if not is_cte_reference(table):+ if ref := get_ref_from_table(table):refs.add(ref)for expression in expression_list:if expression is None:continue- if bool(+ requires_table_scan = bool(expression.find(exp.Update,exp.Insert,@@ -593,7 +600,8 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:exp.TruncateTable,exp.Copy,)- ):+ )+ if requires_table_scan:_collect_table_refs_excluding_ctes(expression) # type: ignore[arg-type]# build_scope only works for select statements.@@ -604,7 +612,12 @@ def find_sql_refs(sql_statement: str) -> set[SQLRef]:if root := build_scope(expression):for scope in root.traverse(): # type: ignorefor _node, source in scope.selected_sources.values():+ # SQLGlot can expose DML CTE references as plain tables.if isinstance(source, exp.Table):+ if requires_table_scan and is_cte_reference(+ source+ ):+ continueif ref := get_ref_from_table(source):refs.add(ref)except OptimizeError:diff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.pyindex 48a465ece1..192e11a7b1 100644--- a/tests/_ast/test_sql_visitor.py+++ b/tests/_ast/test_sql_visitor.py@@ -1168,6 +1168,89 @@ class TestFindSQLRefs:SQLRef(table="target_table"),}+ @pytest.mark.parametrize(+ "statement",+ [+ "INSERT INTO items VALUES (1) RETURNING *",+ "UPDATE items SET id = 2 RETURNING *",+ "DELETE FROM items RETURNING *",+ ],+ )+ def test_dml_cte_reference(self, statement: str) -> None:+ sql = f"WITH changed AS ({statement}) SELECT * FROM changed"+ assert find_sql_refs(sql) == {SQLRef(table="items")}++ @pytest.mark.parametrize(+ "statement",+ [+ "INSERT INTO changed (id) VALUES (1) RETURNING *",+ "UPDATE changed SET id = 2 RETURNING *",+ "DELETE FROM changed RETURNING *",+ ],+ )+ def test_dml_cte_name_matches_target(self, statement: str) -> None:+ sql = f"WITH changed AS ({statement}) SELECT * FROM changed"+ assert find_sql_refs(sql) == {SQLRef(table="changed")}++ def test_nested_dml_cte_scope(self) -> None:+ sql = """+ SELECT * FROM changed+ JOIN (+ WITH changed AS (INSERT INTO items VALUES (1) RETURNING *),+ local_rows AS (SELECT * FROM changed)+ SELECT * FROM local_rows+ ) sub ON changed.id = sub.id+ """+ assert find_sql_refs(sql) == {+ SQLRef(table="changed"),+ SQLRef(table="items"),+ }++ def test_dml_cte_schema_qualified_reference(self) -> None:+ sql = """+ WITH changed AS (DELETE FROM items RETURNING *)+ SELECT * FROM archive.changed a JOIN changed c ON a.id = c.id+ """+ assert find_sql_refs(sql) == {+ SQLRef(table="items"),+ SQLRef(table="changed", schema="archive"),+ }++ def test_dml_cte_reads_table_with_same_name(self) -> None:+ sql = """+ WITH changed AS (+ INSERT INTO items SELECT * FROM changed RETURNING *+ )+ SELECT * FROM changed+ """+ assert find_sql_refs(sql) == {+ SQLRef(table="items"),+ SQLRef(table="changed"),+ }++ def test_dml_cte_with_recursive_source(self) -> None:+ sql = """+ WITH RECURSIVE nums AS (+ SELECT 1 AS id+ UNION ALL+ SELECT id + 1 FROM nums WHERE id < 3+ ), changed AS (INSERT INTO items SELECT * FROM nums RETURNING *)+ SELECT * FROM changed+ """+ assert find_sql_refs(sql) == {SQLRef(table="items")}++ def test_dml_cte_cannot_reference_later_recursive_cte(self) -> None:+ sql = """+ WITH RECURSIVE changed AS (+ INSERT INTO items SELECT * FROM later RETURNING *+ ), later AS (SELECT 1 AS id)+ SELECT * FROM changed+ """+ assert find_sql_refs(sql) == {+ SQLRef(table="items"),+ SQLRef(table="later"),+ }+def test_multiple_statements_with_optimize_error(self) -> None:# Verify that OptimizeError in one statement doesn't affect others.# The try/except is inside the loop, so each statement is independent.