A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix(runtime): respect actual schema/catalog when dropping in-memory SQL tables (#10357)
Shahmir Varqhaac7ea4bd3426
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.pyindex bd5efa01f9..aadc2079e0 100644--- a/marimo/_ast/visitor.py+++ b/marimo/_ast/visitor.py@@ -91,8 +91,8 @@ class VariableData:# For kind == importimport_data: ImportData | None = None- # In the sql case, the name may be qualified- qualified_name: str | None = None+ # Only applicable for SQL kinds (table, view, schema, catalog)+ sql_ref: SQLRef | None = None@propertydef language(self) -> Language:@@ -784,18 +784,14 @@ class ScopedVisitor(ast.NodeVisitor):self._define(None,_table.table,- VariableData(- "table", qualified_name=_table.qualified_name- ),+ VariableData("table", sql_ref=_table),)defined_names.add(_table.qualified_name)for _view in sql_defs.views:self._define(None,_view.table,- VariableData(- "view", qualified_name=_view.qualified_name- ),+ VariableData("view", sql_ref=_view),)defined_names.add(_view.qualified_name)for _schema in sql_defs.schemas:diff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.pyindex f0c9d7217a..3ffb858c59 100644--- a/marimo/_runtime/dataflow/definitions.py+++ b/marimo/_runtime/dataflow/definitions.py@@ -48,12 +48,14 @@ class DefinitionRegistry:Return the key used to group definitions that conflict.Only qualified SQL definitions return a non-global key."""+ sql_ref = variable.sql_ref+ qualified_name = sql_ref.qualified_name if sql_ref else Noneif (variable.language == "sql"- and variable.qualified_name is not None- and variable.qualified_name != name+ and qualified_name is not None+ and qualified_name != name):- return ("sql", variable.qualified_name)+ return ("sql", qualified_name)return ("global", name)def register_definition(diff --git a/marimo/_runtime/dataflow/edges.py b/marimo/_runtime/dataflow/edges.pyindex 1f210c1109..5cb57208f0 100644--- a/marimo/_runtime/dataflow/edges.py+++ b/marimo/_runtime/dataflow/edges.py@@ -210,12 +210,17 @@ def compute_edges_for_cell(continueif language == "sql" and cell.language == "sql":# Edges between SQL cells need to respect hierarchy.- if sql_ref and not sql_ref.matches_hierarchical_ref(- variable_name,- other_variable_data.qualified_name or name,- kind=cast(SQLTypes, other_variable_data.kind),- ):- continue+ if sql_ref:+ other_sql_ref = other_variable_data.sql_ref+ qualified_name = (+ other_sql_ref.qualified_name if other_sql_ref else name+ )+ if not sql_ref.matches_hierarchical_ref(+ variable_name,+ qualified_name,+ kind=cast(SQLTypes, other_variable_data.kind),+ ):+ continueparents.add(other_id)# Next, any cell that deletes this referenced variable is madediff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.pyindex b39253c73e..9b8cec8d33 100644--- a/marimo/_runtime/runtime.py+++ b/marimo/_runtime/runtime.py@@ -443,6 +443,53 @@ class CellMetadata:config: CellConfig = dataclasses.field(default_factory=CellConfig)+def _get_attached_catalogs() -> set[str]:+ """Fetch the names of all catalogs currently attached to DuckDB."""+ import duckdb++ try:+ return {+ row[0].lower()+ for row in duckdb.sql(+ "SELECT database_name FROM duckdb_databases()"+ ).fetchall()+ }+ except Exception:+ return set()+++def _in_memory_qualified_name(+ variable: VariableData,+ name: Name,+ attached_catalogs: Callable[[], set[str]],+) -> str | None:+ """Resolve the quoted, qualified name of an in-memory table/view.++ Returns `None` if the object doesn't live in the "memory" catalog.++ `attached_catalogs` lazily resolves DuckDB's attached catalogs; callers+ should memoize it when resolving many names, to avoid repeated queries.+ """+ sql_ref = variable.sql_ref+ catalog = sql_ref.catalog if sql_ref else None+ schema = sql_ref.schema if sql_ref else None++ if catalog is None and schema is not None:+ # A two-part name (e.g. `CREATE TABLE foo.bar ...`) is ambiguous:+ # `foo` could be a schema in the default "memory" catalog, or+ # shorthand for the catalog `foo` (i.e. `foo.main.bar`). Disambiguate+ # against DuckDB's actual attached catalogs, so we don't mistake an+ # attached database for a "memory" schema (or vice versa).+ if schema.lower() in attached_catalogs():+ catalog, schema = schema, "main"++ catalog = catalog or "memory"+ if catalog != "memory":+ return None+ schema = schema or "main"+ return quote_qualified_name(catalog, schema, name)++class Kernel:"""Kernel that manages the dependency graph and its execution.@@ -999,6 +1046,15 @@ class Kernel:exclude_defs: set[Name],) -> None:"""Delete `names` from kernel, except for `exclude_defs`"""+ # Memoize the attached-catalog lookup+ attached_catalogs: set[str] | None = None++ def get_attached_catalogs() -> set[str]:+ nonlocal attached_catalogs+ if attached_catalogs is None:+ attached_catalogs = _get_attached_catalogs()+ return attached_catalogs+for name, variable_data in variables.items():# Take the last definition of the variablevariable = variable_data[-1]@@ -1010,20 +1066,30 @@ class Kernel:# We only drop in-memory tables: we don't want to drop tables# on databases!- try:- qualified = quote_qualified_name("memory", "main", name)- duckdb.execute(f"DROP TABLE IF EXISTS {qualified}")- except Exception as e:- LOGGER.warning("Failed to drop table %s: %s", name, str(e))+ qualified = _in_memory_qualified_name(+ variable, name, get_attached_catalogs+ )+ if qualified is not None:+ try:+ duckdb.execute(f"DROP TABLE IF EXISTS {qualified}")+ except Exception as e:+ LOGGER.warning(+ "Failed to drop table %s: %s", name, str(e)+ )elif variable.kind == "view" and DependencyManager.duckdb.has():import duckdb# We only drop in-memory views for the same reason.- try:- qualified = quote_qualified_name("memory", "main", name)- duckdb.execute(f"DROP VIEW IF EXISTS {qualified}")- except Exception as e:- LOGGER.warning("Failed to drop view %s: %s", name, str(e))+ qualified = _in_memory_qualified_name(+ variable, name, get_attached_catalogs+ )+ if qualified is not None:+ try:+ duckdb.execute(f"DROP VIEW IF EXISTS {qualified}")+ except Exception as e:+ LOGGER.warning(+ "Failed to drop view %s: %s", name, str(e)+ )elif variable.kind == "catalog" and DependencyManager.duckdb.has():import duckdbdiff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.pyindex a5002d3bf4..03a6ee1186 100644--- a/tests/_ast/test_compiler.py+++ b/tests/_ast/test_compiler.py@@ -10,6 +10,7 @@ import pytestfrom marimo._ast import compilerfrom marimo._ast.cell_manager import CellManager+from marimo._ast.sql_visitor import SQLReffrom marimo._ast.visitor import ImportData, VariableDatafrom marimo._dependencies.dependencies import DependencyManager@@ -272,7 +273,7 @@ class TestParseSQLCell:assert cell.refs == {"mo"}assert cell.language == "sql"assert cell.variable_data == {- "t1": [VariableData("table", qualified_name="t1")]+ "t1": [VariableData("table", sql_ref=SQLRef(table="t1"))]}@staticmethod@@ -302,7 +303,7 @@ class TestParseSQLCell:assert cell.refs == {"duckdb"}assert cell.language == "sql"assert cell.variable_data == {- "t1": [VariableData("table", qualified_name="t1")]+ "t1": [VariableData("table", sql_ref=SQLRef(table="t1"))]}@staticmethoddiff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.pyindex f7ea1632ee..eb688f10ce 100644--- a/tests/_ast/test_visitor.py+++ b/tests/_ast/test_visitor.py@@ -10,7 +10,7 @@ import pytestfrom marimo._ast import visitorfrom marimo._ast.errors import ImportStarError-from marimo._ast.sql_visitor import normalize_sql_f_string+from marimo._ast.sql_visitor import SQLRef, normalize_sql_f_stringfrom marimo._ast.visitor import (AnnotationData,ImportData,@@ -1504,7 +1504,7 @@ def test_sql_statement_with_url() -> None:v.visit(mod)assert v.defs == {"cars"}assert v.variable_data == {- "cars": [VariableData("table", qualified_name="cars")]+ "cars": [VariableData("table", sql_ref=SQLRef(table="cars"))]}assert v.refs == {"mo"}diff --git a/tests/_runtime/test_runtime.py b/tests/_runtime/test_runtime.pyindex 20a8ef30af..20bc91c154 100644--- a/tests/_runtime/test_runtime.py+++ b/tests/_runtime/test_runtime.py@@ -47,6 +47,7 @@ from marimo._runtime.runtime import (notebook_location,)from marimo._runtime.scratch import SCRATCH_CELL_ID+from marimo._types.ids import CellId_tfrom marimo._utils.parse_dataclass import parse_rawfrom tests._messaging.mocks import MockStderr, MockStreamfrom tests._runtime._helpers.factories import default_app_metadata@@ -3512,31 +3513,64 @@ class TestSQL:# cell 1 should re-run but will fail to find t1assert "df" not in k.globals+ @pytest.mark.parametrize(+ ("catalog", "schema"),+ [+ (None, None),+ (None, "main"),+ (None, "custom-schema"),+ ("memory", "main"),+ ("memory", "custom-schema"),+ # A schema containing a literal "." must not be confused with+ # a catalog/schema separator when resolving the qualified name+ # to drop.+ (None, "my.schema"),+ ],+ ids=[+ "unqualified",+ "explicit-schema",+ "custom-schema",+ "explicit-catalog-and-schema",+ "explicit-catalog-and-custom-schema",+ "dotted-schema",+ ],+ )async def test_sql_table_with_special_char_name_is_dropped(- self, k: Kernel+ self, k: Kernel, catalog: str | None, schema: str | None) -> None:# Regression test for #10338: an in-memory table whose name needs- # quoting (e.g. a hyphen) must still be cleaned up. The name was- # previously interpolated raw, producing invalid SQL- # (`DROP TABLE IF EXISTS memory.main.manual-holdings`).+ # quoting (e.g. a hyphen) must still be cleaned up.import duckdb+ resolved_schema = schema or "main"+ if resolved_schema != "main":+ duckdb.execute(f'CREATE SCHEMA IF NOT EXISTS "{resolved_schema}"')+def table_exists() -> bool:- return (- duckdb.execute(- "SELECT count(*) FROM information_schema.tables "- "WHERE table_name = 'manual-holdings'"- ).fetchone()[0]- > 0- )+ row = duckdb.execute(+ "SELECT count(*) FROM information_schema.tables "+ "WHERE table_catalog = 'memory' AND table_schema = ? "+ "AND table_name = 'manual-holdings'",+ [resolved_schema],+ ).fetchone()+ assert row is not None+ return bool(row[0] > 0)++ qualified_name = ".".join(+ f'"{part}"'+ for part in (catalog, schema, "manual-holdings")+ if part+ )await k.run([- ExecuteCellCommand(cell_id="0", code="import marimo as mo"),ExecuteCellCommand(- cell_id="1",+ cell_id=CellId_t("0"), code="import marimo as mo"+ ),+ ExecuteCellCommand(+ cell_id=CellId_t("1"),code=(- 'mo.sql(\'CREATE OR REPLACE TABLE "manual-holdings" '+ f"mo.sql('CREATE OR REPLACE TABLE {qualified_name} ""AS SELECT 1 AS a')"),),@@ -3546,7 +3580,107 @@ class TestSQL:assert table_exists()# Deleting the defining cell triggers cleanup of the in-memory table.- await k.delete_cell(DeleteCellCommand(cell_id="1"))+ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))+ assert not table_exists()++ async def test_sql_table_on_attached_catalog_is_not_dropped(+ self, k: Kernel+ ) -> None:+ # `CREATE TABLE <catalog>.<table>` is shorthand for+ # `<catalog>.main.<table>`, which is ambiguous with `<schema>.<table>`+ # in the default "memory" catalog. Cleanup must resolve this against+ # the real attached catalogs, so it neither leaves the attached+ # table undropped-but-mistaken-for-memory, nor drops an unrelated,+ # same-named table living in a "memory" schema of the same name.+ import duckdb++ duckdb.execute("ATTACH ':memory:' AS other_db")++ def attached_table_exists() -> bool:+ row = duckdb.execute(+ "SELECT count(*) FROM information_schema.tables "+ "WHERE table_catalog = 'other_db' AND table_schema = 'main' "+ "AND table_name = 'holdings'"+ ).fetchone()+ assert row is not None+ return bool(row[0] > 0)++ def memory_schema_table_exists() -> bool:+ row = duckdb.execute(+ "SELECT count(*) FROM information_schema.tables "+ "WHERE table_catalog = 'memory' AND table_schema = 'other_db' "+ "AND table_name = 'holdings'"+ ).fetchone()+ assert row is not None+ return bool(row[0] > 0)++ await k.run(+ [+ ExecuteCellCommand(+ cell_id=CellId_t("0"), code="import marimo as mo"+ ),+ ExecuteCellCommand(+ cell_id=CellId_t("1"),+ code=(+ "mo.sql('CREATE OR REPLACE TABLE other_db.holdings "+ "AS SELECT 1 AS a')"+ ),+ ),+ ]+ )+ assert not k.errors+ assert attached_table_exists()++ # An unrelated table happens to be created afterwards in a "memory"+ # schema with the same name as the attached catalog. Cleanup must+ # not confuse the two.+ duckdb.execute('CREATE SCHEMA IF NOT EXISTS "other_db"')+ duckdb.execute(+ 'CREATE TABLE memory."other_db".holdings AS SELECT 2 AS a'+ )++ # Deleting the defining cell must not drop the attached table, and+ # must not touch the unrelated table in memory."other_db".+ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))+ assert attached_table_exists()+ assert memory_schema_table_exists()++ async def test_sql_table_qualified_with_default_catalog_is_dropped(+ self, k: Kernel+ ) -> None:+ # `CREATE TABLE memory.holdings ...` is a two-part+ # name whose qualifier happens to be "memory", the default catalog's+ # own name. DuckDB resolves this to `memory.main.holdings`, not a+ # schema literally named "memory"; cleanup must match.+ import duckdb++ def table_exists() -> bool:+ row = duckdb.execute(+ "SELECT count(*) FROM information_schema.tables "+ "WHERE table_catalog = 'memory' AND table_schema = 'main' "+ "AND table_name = 'holdings'"+ ).fetchone()+ assert row is not None+ return bool(row[0] > 0)++ await k.run(+ [+ ExecuteCellCommand(+ cell_id=CellId_t("0"), code="import marimo as mo"+ ),+ ExecuteCellCommand(+ cell_id=CellId_t("1"),+ code=(+ "mo.sql('CREATE OR REPLACE TABLE memory.holdings "+ "AS SELECT 1 AS a')"+ ),+ ),+ ]+ )+ assert not k.errors+ assert table_exists()++ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))assert not table_exists()async def test_sql_query_as_local_df(self, k: Kernel) -> None: