Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix(runtime): respect actual schema/catalog when dropping in-memory SQL tables (#10357)

Shahmir Varqhaac7ea4bd3426
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.py
index bd5efa01f9..aadc2079e0 100644
--- a/marimo/_ast/visitor.py
+++ b/marimo/_ast/visitor.py
@@ -91,8 +91,8 @@ class VariableData:
# For kind == import
import_data: ImportData | None = None
- # In the sql case, the name may be qualified
- qualified_name: str | None = None
+ # Only applicable for SQL kinds (table, view, schema, catalog)
+ sql_ref: SQLRef | None = None
@property
def language(self) -> Language:
@@ -784,18 +784,14 @@ class ScopedVisitor(ast.NodeVisitor):
self._define(
None,
_table.table,
- VariableData(
- "table", qualified_name=_table.qualified_name
- ),
+ VariableData("table", sql_ref=_table),
)
defined_names.add(_table.qualified_name)
for _view in sql_defs.views:
self._define(
None,
_view.table,
- VariableData(
- "view", qualified_name=_view.qualified_name
- ),
+ VariableData("view", sql_ref=_view),
)
defined_names.add(_view.qualified_name)
for _schema in sql_defs.schemas:
diff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.py
index f0c9d7217a..3ffb858c59 100644
--- a/marimo/_runtime/dataflow/definitions.py
+++ b/marimo/_runtime/dataflow/definitions.py
@@ -48,12 +48,14 @@ class DefinitionRegistry:
Return the key used to group definitions that conflict.
Only qualified SQL definitions return a non-global key.
"""
+ sql_ref = variable.sql_ref
+ qualified_name = sql_ref.qualified_name if sql_ref else None
if (
variable.language == "sql"
- and variable.qualified_name is not None
- and variable.qualified_name != name
+ and qualified_name is not None
+ and qualified_name != name
):
- return ("sql", variable.qualified_name)
+ return ("sql", qualified_name)
return ("global", name)
def register_definition(
diff --git a/marimo/_runtime/dataflow/edges.py b/marimo/_runtime/dataflow/edges.py
index 1f210c1109..5cb57208f0 100644
--- a/marimo/_runtime/dataflow/edges.py
+++ b/marimo/_runtime/dataflow/edges.py
@@ -210,12 +210,17 @@ def compute_edges_for_cell(
continue
if language == "sql" and cell.language == "sql":
# Edges between SQL cells need to respect hierarchy.
- if sql_ref and not sql_ref.matches_hierarchical_ref(
- variable_name,
- other_variable_data.qualified_name or name,
- kind=cast(SQLTypes, other_variable_data.kind),
- ):
- continue
+ if sql_ref:
+ other_sql_ref = other_variable_data.sql_ref
+ qualified_name = (
+ other_sql_ref.qualified_name if other_sql_ref else name
+ )
+ if not sql_ref.matches_hierarchical_ref(
+ variable_name,
+ qualified_name,
+ kind=cast(SQLTypes, other_variable_data.kind),
+ ):
+ continue
parents.add(other_id)
# Next, any cell that deletes this referenced variable is made
diff --git a/marimo/_runtime/runtime.py b/marimo/_runtime/runtime.py
index b39253c73e..9b8cec8d33 100644
--- a/marimo/_runtime/runtime.py
+++ b/marimo/_runtime/runtime.py
@@ -443,6 +443,53 @@ class CellMetadata:
config: CellConfig = dataclasses.field(default_factory=CellConfig)
+def _get_attached_catalogs() -> set[str]:
+ """Fetch the names of all catalogs currently attached to DuckDB."""
+ import duckdb
+
+ try:
+ return {
+ row[0].lower()
+ for row in duckdb.sql(
+ "SELECT database_name FROM duckdb_databases()"
+ ).fetchall()
+ }
+ except Exception:
+ return set()
+
+
+def _in_memory_qualified_name(
+ variable: VariableData,
+ name: Name,
+ attached_catalogs: Callable[[], set[str]],
+) -> str | None:
+ """Resolve the quoted, qualified name of an in-memory table/view.
+
+ Returns `None` if the object doesn't live in the "memory" catalog.
+
+ `attached_catalogs` lazily resolves DuckDB's attached catalogs; callers
+ should memoize it when resolving many names, to avoid repeated queries.
+ """
+ sql_ref = variable.sql_ref
+ catalog = sql_ref.catalog if sql_ref else None
+ schema = sql_ref.schema if sql_ref else None
+
+ if catalog is None and schema is not None:
+ # A two-part name (e.g. `CREATE TABLE foo.bar ...`) is ambiguous:
+ # `foo` could be a schema in the default "memory" catalog, or
+ # shorthand for the catalog `foo` (i.e. `foo.main.bar`). Disambiguate
+ # against DuckDB's actual attached catalogs, so we don't mistake an
+ # attached database for a "memory" schema (or vice versa).
+ if schema.lower() in attached_catalogs():
+ catalog, schema = schema, "main"
+
+ catalog = catalog or "memory"
+ if catalog != "memory":
+ return None
+ schema = schema or "main"
+ return quote_qualified_name(catalog, schema, name)
+
+
class Kernel:
"""Kernel that manages the dependency graph and its execution.
@@ -999,6 +1046,15 @@ class Kernel:
exclude_defs: set[Name],
) -> None:
"""Delete `names` from kernel, except for `exclude_defs`"""
+ # Memoize the attached-catalog lookup
+ attached_catalogs: set[str] | None = None
+
+ def get_attached_catalogs() -> set[str]:
+ nonlocal attached_catalogs
+ if attached_catalogs is None:
+ attached_catalogs = _get_attached_catalogs()
+ return attached_catalogs
+
for name, variable_data in variables.items():
# Take the last definition of the variable
variable = variable_data[-1]
@@ -1010,20 +1066,30 @@ class Kernel:
# We only drop in-memory tables: we don't want to drop tables
# on databases!
- try:
- qualified = quote_qualified_name("memory", "main", name)
- duckdb.execute(f"DROP TABLE IF EXISTS {qualified}")
- except Exception as e:
- LOGGER.warning("Failed to drop table %s: %s", name, str(e))
+ qualified = _in_memory_qualified_name(
+ variable, name, get_attached_catalogs
+ )
+ if qualified is not None:
+ try:
+ duckdb.execute(f"DROP TABLE IF EXISTS {qualified}")
+ except Exception as e:
+ LOGGER.warning(
+ "Failed to drop table %s: %s", name, str(e)
+ )
elif variable.kind == "view" and DependencyManager.duckdb.has():
import duckdb
# We only drop in-memory views for the same reason.
- try:
- qualified = quote_qualified_name("memory", "main", name)
- duckdb.execute(f"DROP VIEW IF EXISTS {qualified}")
- except Exception as e:
- LOGGER.warning("Failed to drop view %s: %s", name, str(e))
+ qualified = _in_memory_qualified_name(
+ variable, name, get_attached_catalogs
+ )
+ if qualified is not None:
+ try:
+ duckdb.execute(f"DROP VIEW IF EXISTS {qualified}")
+ except Exception as e:
+ LOGGER.warning(
+ "Failed to drop view %s: %s", name, str(e)
+ )
elif variable.kind == "catalog" and DependencyManager.duckdb.has():
import duckdb
diff --git a/tests/_ast/test_compiler.py b/tests/_ast/test_compiler.py
index a5002d3bf4..03a6ee1186 100644
--- a/tests/_ast/test_compiler.py
+++ b/tests/_ast/test_compiler.py
@@ -10,6 +10,7 @@ import pytest
from marimo._ast import compiler
from marimo._ast.cell_manager import CellManager
+from marimo._ast.sql_visitor import SQLRef
from marimo._ast.visitor import ImportData, VariableData
from marimo._dependencies.dependencies import DependencyManager
@@ -272,7 +273,7 @@ class TestParseSQLCell:
assert cell.refs == {"mo"}
assert cell.language == "sql"
assert cell.variable_data == {
- "t1": [VariableData("table", qualified_name="t1")]
+ "t1": [VariableData("table", sql_ref=SQLRef(table="t1"))]
}
@staticmethod
@@ -302,7 +303,7 @@ class TestParseSQLCell:
assert cell.refs == {"duckdb"}
assert cell.language == "sql"
assert cell.variable_data == {
- "t1": [VariableData("table", qualified_name="t1")]
+ "t1": [VariableData("table", sql_ref=SQLRef(table="t1"))]
}
@staticmethod
diff --git a/tests/_ast/test_visitor.py b/tests/_ast/test_visitor.py
index f7ea1632ee..eb688f10ce 100644
--- a/tests/_ast/test_visitor.py
+++ b/tests/_ast/test_visitor.py
@@ -10,7 +10,7 @@ import pytest
from marimo._ast import visitor
from marimo._ast.errors import ImportStarError
-from marimo._ast.sql_visitor import normalize_sql_f_string
+from marimo._ast.sql_visitor import SQLRef, normalize_sql_f_string
from marimo._ast.visitor import (
AnnotationData,
ImportData,
@@ -1504,7 +1504,7 @@ def test_sql_statement_with_url() -> None:
v.visit(mod)
assert v.defs == {"cars"}
assert v.variable_data == {
- "cars": [VariableData("table", qualified_name="cars")]
+ "cars": [VariableData("table", sql_ref=SQLRef(table="cars"))]
}
assert v.refs == {"mo"}
diff --git a/tests/_runtime/test_runtime.py b/tests/_runtime/test_runtime.py
index 20a8ef30af..20bc91c154 100644
--- a/tests/_runtime/test_runtime.py
+++ b/tests/_runtime/test_runtime.py
@@ -47,6 +47,7 @@ from marimo._runtime.runtime import (
notebook_location,
)
from marimo._runtime.scratch import SCRATCH_CELL_ID
+from marimo._types.ids import CellId_t
from marimo._utils.parse_dataclass import parse_raw
from tests._messaging.mocks import MockStderr, MockStream
from tests._runtime._helpers.factories import default_app_metadata
@@ -3512,31 +3513,64 @@ class TestSQL:
# cell 1 should re-run but will fail to find t1
assert "df" not in k.globals
+ @pytest.mark.parametrize(
+ ("catalog", "schema"),
+ [
+ (None, None),
+ (None, "main"),
+ (None, "custom-schema"),
+ ("memory", "main"),
+ ("memory", "custom-schema"),
+ # A schema containing a literal "." must not be confused with
+ # a catalog/schema separator when resolving the qualified name
+ # to drop.
+ (None, "my.schema"),
+ ],
+ ids=[
+ "unqualified",
+ "explicit-schema",
+ "custom-schema",
+ "explicit-catalog-and-schema",
+ "explicit-catalog-and-custom-schema",
+ "dotted-schema",
+ ],
+ )
async def test_sql_table_with_special_char_name_is_dropped(
- self, k: Kernel
+ self, k: Kernel, catalog: str | None, schema: str | None
) -> None:
# Regression test for #10338: an in-memory table whose name needs
- # quoting (e.g. a hyphen) must still be cleaned up. The name was
- # previously interpolated raw, producing invalid SQL
- # (`DROP TABLE IF EXISTS memory.main.manual-holdings`).
+ # quoting (e.g. a hyphen) must still be cleaned up.
import duckdb
+ resolved_schema = schema or "main"
+ if resolved_schema != "main":
+ duckdb.execute(f'CREATE SCHEMA IF NOT EXISTS "{resolved_schema}"')
+
def table_exists() -> bool:
- return (
- duckdb.execute(
- "SELECT count(*) FROM information_schema.tables "
- "WHERE table_name = 'manual-holdings'"
- ).fetchone()[0]
- > 0
- )
+ row = duckdb.execute(
+ "SELECT count(*) FROM information_schema.tables "
+ "WHERE table_catalog = 'memory' AND table_schema = ? "
+ "AND table_name = 'manual-holdings'",
+ [resolved_schema],
+ ).fetchone()
+ assert row is not None
+ return bool(row[0] > 0)
+
+ qualified_name = ".".join(
+ f'"{part}"'
+ for part in (catalog, schema, "manual-holdings")
+ if part
+ )
await k.run(
[
- ExecuteCellCommand(cell_id="0", code="import marimo as mo"),
ExecuteCellCommand(
- cell_id="1",
+ cell_id=CellId_t("0"), code="import marimo as mo"
+ ),
+ ExecuteCellCommand(
+ cell_id=CellId_t("1"),
code=(
- 'mo.sql(\'CREATE OR REPLACE TABLE "manual-holdings" '
+ f"mo.sql('CREATE OR REPLACE TABLE {qualified_name} "
"AS SELECT 1 AS a')"
),
),
@@ -3546,7 +3580,107 @@ class TestSQL:
assert table_exists()
# Deleting the defining cell triggers cleanup of the in-memory table.
- await k.delete_cell(DeleteCellCommand(cell_id="1"))
+ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))
+ assert not table_exists()
+
+ async def test_sql_table_on_attached_catalog_is_not_dropped(
+ self, k: Kernel
+ ) -> None:
+ # `CREATE TABLE <catalog>.<table>` is shorthand for
+ # `<catalog>.main.<table>`, which is ambiguous with `<schema>.<table>`
+ # in the default "memory" catalog. Cleanup must resolve this against
+ # the real attached catalogs, so it neither leaves the attached
+ # table undropped-but-mistaken-for-memory, nor drops an unrelated,
+ # same-named table living in a "memory" schema of the same name.
+ import duckdb
+
+ duckdb.execute("ATTACH ':memory:' AS other_db")
+
+ def attached_table_exists() -> bool:
+ row = duckdb.execute(
+ "SELECT count(*) FROM information_schema.tables "
+ "WHERE table_catalog = 'other_db' AND table_schema = 'main' "
+ "AND table_name = 'holdings'"
+ ).fetchone()
+ assert row is not None
+ return bool(row[0] > 0)
+
+ def memory_schema_table_exists() -> bool:
+ row = duckdb.execute(
+ "SELECT count(*) FROM information_schema.tables "
+ "WHERE table_catalog = 'memory' AND table_schema = 'other_db' "
+ "AND table_name = 'holdings'"
+ ).fetchone()
+ assert row is not None
+ return bool(row[0] > 0)
+
+ await k.run(
+ [
+ ExecuteCellCommand(
+ cell_id=CellId_t("0"), code="import marimo as mo"
+ ),
+ ExecuteCellCommand(
+ cell_id=CellId_t("1"),
+ code=(
+ "mo.sql('CREATE OR REPLACE TABLE other_db.holdings "
+ "AS SELECT 1 AS a')"
+ ),
+ ),
+ ]
+ )
+ assert not k.errors
+ assert attached_table_exists()
+
+ # An unrelated table happens to be created afterwards in a "memory"
+ # schema with the same name as the attached catalog. Cleanup must
+ # not confuse the two.
+ duckdb.execute('CREATE SCHEMA IF NOT EXISTS "other_db"')
+ duckdb.execute(
+ 'CREATE TABLE memory."other_db".holdings AS SELECT 2 AS a'
+ )
+
+ # Deleting the defining cell must not drop the attached table, and
+ # must not touch the unrelated table in memory."other_db".
+ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))
+ assert attached_table_exists()
+ assert memory_schema_table_exists()
+
+ async def test_sql_table_qualified_with_default_catalog_is_dropped(
+ self, k: Kernel
+ ) -> None:
+ # `CREATE TABLE memory.holdings ...` is a two-part
+ # name whose qualifier happens to be "memory", the default catalog's
+ # own name. DuckDB resolves this to `memory.main.holdings`, not a
+ # schema literally named "memory"; cleanup must match.
+ import duckdb
+
+ def table_exists() -> bool:
+ row = duckdb.execute(
+ "SELECT count(*) FROM information_schema.tables "
+ "WHERE table_catalog = 'memory' AND table_schema = 'main' "
+ "AND table_name = 'holdings'"
+ ).fetchone()
+ assert row is not None
+ return bool(row[0] > 0)
+
+ await k.run(
+ [
+ ExecuteCellCommand(
+ cell_id=CellId_t("0"), code="import marimo as mo"
+ ),
+ ExecuteCellCommand(
+ cell_id=CellId_t("1"),
+ code=(
+ "mo.sql('CREATE OR REPLACE TABLE memory.holdings "
+ "AS SELECT 1 AS a')"
+ ),
+ ),
+ ]
+ )
+ assert not k.errors
+ assert table_exists()
+
+ await k.delete_cell(DeleteCellCommand(cell_id=CellId_t("1")))
assert not table_exists()
async def test_sql_query_as_local_df(self, k: Kernel) -> None: