Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

sql cells can't both define and refer to itself (#6216)

Shahmir Varqhad507d908e2c9
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.py
index 3e6a200d08..6f46878ff7 100644
--- a/marimo/_ast/visitor.py
+++ b/marimo/_ast/visitor.py
@@ -653,21 +653,7 @@ class ScopedVisitor(ast.NodeVisitor):
return node
for statement in statements:
- sql_refs: set[SQLRef] = set()
# Parse the refs and defs of each statement
- try:
- sql_refs = find_sql_refs_cached(statement.query)
- except (duckdb.ProgrammingError, duckdb.IOException):
- LOGGER.debug(
- "Error parsing SQL statement: %s", statement.query
- )
- except BaseException as e:
- LOGGER.warning("Unexpected duckdb error %s", e)
-
- for ref in sql_refs:
- name = ref.qualified_name
- self._add_ref(None, name, deleted=False, sql_ref=ref)
-
# Add all tables/dbs created in the query to the defs
try:
sql_defs = find_sql_defs(sql)
@@ -677,6 +663,8 @@ class ScopedVisitor(ast.NodeVisitor):
LOGGER.warning("Unexpected duckdb error %s", e)
sql_defs = SQLDefs()
+ defined_names = set()
+
for _table in sql_defs.tables:
self._define(
None,
@@ -685,6 +673,7 @@ class ScopedVisitor(ast.NodeVisitor):
"table", qualified_name=_table.qualified_name
),
)
+ defined_names.add(_table.qualified_name)
for _view in sql_defs.views:
self._define(
None,
@@ -693,10 +682,30 @@ class ScopedVisitor(ast.NodeVisitor):
"view", qualified_name=_view.qualified_name
),
)
+ defined_names.add(_view.qualified_name)
for _schema in sql_defs.schemas:
self._define(None, _schema, VariableData("schema"))
+ defined_names.add(_schema)
for _catalog in sql_defs.catalogs:
self._define(None, _catalog, VariableData("catalog"))
+ defined_names.add(_catalog)
+
+ sql_refs: set[SQLRef] = set()
+ try:
+ sql_refs = find_sql_refs_cached(statement.query)
+ except (duckdb.ProgrammingError, duckdb.IOException):
+ LOGGER.debug(
+ "Error parsing SQL statement: %s", statement.query
+ )
+ except BaseException as e:
+ LOGGER.warning("Unexpected duckdb error %s", e)
+
+ for ref in sql_refs:
+ name = ref.qualified_name
+ # Cells that define the same name aren't cycles, so we skip them
+ if name in defined_names:
+ continue
+ self._add_ref(None, name, deleted=False, sql_ref=ref)
# Visit arguments, keyword args, etc.
self.generic_visit(node)
diff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.py
index c9ef8b912c..a2db4f70b1 100644
--- a/tests/_ast/test_visitor.py
+++ b/tests/_ast/test_visitor.py
@@ -1389,3 +1389,150 @@ def test_sql_table_f_string() -> None:
v.visit(mod)
assert not v.defs
assert v.refs == set(["mo", "my_table", "lim"])
+
+
[email protected](not HAS_DEPS, reason="Requires duckdb")
+ ("description", "sql_statement", "expected_defs", "expected_refs"),
+ [
+ (
+ "create and select from same table in same schema",
+ "create table cars (id int); select * from cars",
+ {"cars"},
+ {"mo"},
+ ),
+ (
+ "create and select from same table with schema prefix",
+ "create table schema.cars (id int); select * from schema.cars",
+ {"cars"},
+ {"mo"},
+ ),
+ (
+ "create and select from same table with catalog and schema prefix",
+ "create table catalog.schema.cars (id int); select * from catalog.schema.cars",
+ {"cars"},
+ {"mo"},
+ ),
+ (
+ "create in one schema, select from another schema",
+ "create table schema1.cars (id int); select * from schema2.cars",
+ {"cars"},
+ {"mo", "schema2.cars"},
+ ),
+ (
+ "create in one catalog, select from another catalog",
+ "create table catalog1.schema.cars (id int); select * from catalog2.schema.cars",
+ {"cars"},
+ {"mo", "catalog2.schema.cars"},
+ ),
+ (
+ "create schema and use it in same statement",
+ "create schema my_schema; create table my_schema.cars (id int); select * from my_schema.cars",
+ {"my_schema", "cars"},
+ {"mo"},
+ ),
+ (
+ "drop table and reference it in same statement",
+ "drop table schema.cars; select * from cars",
+ set(),
+ {"mo", "cars"},
+ ),
+ (
+ "drop schema and reference table from it in same statement",
+ "drop schema my_schema; select * from my_schema.cars",
+ set(),
+ {"mo", "my_schema.cars"},
+ ),
+ (
+ "multiple tables in same schema",
+ "create table schema.cars (id int); create table schema.drivers (id int); select * from schema.cars inner join schema.drivers on cars.id = drivers.id",
+ {"cars", "drivers"},
+ {"mo"},
+ ),
+ (
+ "multiple tables in different schemas",
+ "create table schema1.cars (id int); create table schema2.drivers (id int); select * from schema1.cars inner join schema2.drivers on cars.id = drivers.id",
+ {"cars", "drivers"},
+ {"mo"},
+ ),
+ (
+ "multiple tables in different catalogs",
+ "create table catalog1.schema.cars (id int); create table catalog2.schema.drivers (id int); select * from catalog1.schema.cars inner join catalog2.schema.drivers on cars.id = drivers.id",
+ {"cars", "drivers"},
+ {"mo"},
+ ),
+ (
+ "create view and reference it in same statement",
+ "create view cars_view as select * from cars; select * from cars_view",
+ {"cars_view"},
+ {"mo", "cars"},
+ ),
+ ],
+)
+def test_sql_defined_in_same_statement(
+ description: str,
+ sql_statement: str,
+ expected_defs: set[str],
+ expected_refs: set[str],
+) -> None:
+ """Test various SQL scenarios with create/select operations in same statement."""
+ code = f"df = mo.sql('{sql_statement}')"
+ v = visitor.ScopedVisitor()
+ mod = ast.parse(code)
+ v.visit(mod)
+
+ expected_defs.add("df")
+ assert v.defs == expected_defs, f"Failed for: {description}"
+ assert v.refs == expected_refs, f"Failed for: {description}"
+
+
[email protected](not HAS_DEPS, reason="Requires duckdb")
+ (
+ "description",
+ "sql_statement",
+ "expected_defs",
+ "expected_refs",
+ ),
+ [
+ pytest.param(
+ "create and drop table in same statement",
+ "create table cars (id int); drop table cars;",
+ set(),
+ {"mo"},
+ marks=pytest.mark.xfail(
+ reason="We still set the table as defined even if it is deleted in the same statement"
+ ),
+ ),
+ pytest.param(
+ "create and drop schema in same statement",
+ "create schema my_schema; drop schema my_schema;",
+ set(),
+ {"mo"},
+ marks=pytest.mark.xfail(
+ reason="We still set the schema as defined even if it is deleted in the same statement"
+ ),
+ ),
+ (
+ "attach and detach catalog in same statement",
+ 'attach "my_catalog.db" as my_catalog; detach my_catalog;',
+ set(),
+ {"mo"},
+ ),
+ ],
+)
+def test_sql_table_deleted_in_same_statement(
+ description: str,
+ sql_statement: str,
+ expected_defs: set[str],
+ expected_refs: set[str],
+) -> None:
+ """Test various SQL scenarios with create/drop operations in same statement."""
+ code = f"df = mo.sql('{sql_statement}')"
+ v = visitor.ScopedVisitor()
+ mod = ast.parse(code)
+ v.visit(mod)
+
+ expected_defs.add("df")
+ assert v.defs == expected_defs, f"Failed for: {description}"
+ assert v.refs == expected_refs, f"Failed for: {description}"
diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.py
index 66e9c9832d..4426b89001 100644
--- a/tests/_runtime/test_dataflow.py
+++ b/tests/_runtime/test_dataflow.py
@@ -671,6 +671,25 @@ class TestSQL:
assert graph.get_referring_cells("df", language="python") == set(["1"])
+ def test_sql_creation_and_select_from_same_cell(self):
+ graph = dataflow.DirectedGraph()
+ code = "df = mo.sql('CREATE TABLE t1 (i INTEGER); SELECT * FROM t1')"
+ first_cell = parse_cell(code)
+ graph.register_cell("0", first_cell)
+
+ code = "df"
+ second_cell = parse_cell(code)
+ graph.register_cell("1", second_cell)
+
+ assert graph.cells == {
+ "0": first_cell,
+ "1": second_cell,
+ }
+
+ assert graph.parents == {"0": set(), "1": set(["0"])}
+ assert graph.children == {"0": set(["1"]), "1": set([])}
+ assert graph.get_referring_cells("df", language="python") == set(["1"])
+
def test_get_referring_cells_sql_and_python(self) -> None:
"""Test the get_referring_cells method."""
graph = dataflow.DirectedGraph()