Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix sql defs lookup (#9754)

Shahmir Varqha7745bbf85bca
diff --git a/marimo/_lint/validate_graph.py b/marimo/_lint/validate_graph.py
index a0d894aba6..88060ffdfa 100644
--- a/marimo/_lint/validate_graph.py
+++ b/marimo/_lint/validate_graph.py
@@ -52,17 +52,14 @@ def check_for_multiple_definitions(
) -> dict[CellId_t, list[MultipleDefinitionError]]:
"""Check whether multiple cells define the same global name."""
errors = defaultdict(list)
- defs = sorted(set().union(*(cell.defs for cell in graph.cells.values())))
- for name in defs:
- defining_cells = graph.definitions[name]
- if len(defining_cells) > 1:
- for cid in defining_cells:
- errors[cid].append(
- MultipleDefinitionError(
- name=str(name),
- cells=tuple(sorted(defining_cells - {cid})),
- )
+ for name, defining_cells in graph.get_multiply_defined_conflicts():
+ for cid in defining_cells:
+ errors[cid].append(
+ MultipleDefinitionError(
+ name=str(name),
+ cells=tuple(sorted(defining_cells - {cid})),
)
+ )
return errors
diff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.py
index 0fb622c578..f0c9d7217a 100644
--- a/marimo/_runtime/dataflow/definitions.py
+++ b/marimo/_runtime/dataflow/definitions.py
@@ -26,15 +26,35 @@ class DefinitionRegistry:
# A mapping from defs to the cells that define them
definitions: dict[Name, set[CellId_t]] = field(default_factory=dict)
- # Typed definitions for SQL support: (name, type) -> cell_ids
+ # Typed definitions: (name, type) -> cell_ids
# e.g. ("my_table", "table") -> {cell_id_1}
typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(
default_factory=dict
)
- # Track all types for a given definition name
- # e.g. "my_table" -> {"table", "view"}
- definition_types: dict[Name, set[str]] = field(default_factory=dict)
+ # Track definition conflicts separately from lookup indexes.
+ # Qualified SQL definitions with the same leaf name are distinct objects
+ # in the database, but still need to be discoverable by their leaf name
+ # when computing SQL edges.
+ definition_conflicts: dict[tuple[str, str], set[CellId_t]] = field(
+ default_factory=dict
+ )
+ conflict_names: dict[tuple[str, str], Name] = field(default_factory=dict)
+
+ def _conflict_key(
+ self, name: Name, variable: VariableData
+ ) -> tuple[str, str]:
+ """
+ Return the key used to group definitions that conflict.
+ Only qualified SQL definitions return a non-global key.
+ """
+ if (
+ variable.language == "sql"
+ and variable.qualified_name is not None
+ and variable.qualified_name != name
+ ):
+ return ("sql", variable.qualified_name)
+ return ("global", name)
def register_definition(
self,
@@ -54,50 +74,53 @@ class DefinitionRegistry:
"""
variable = variable_data[-1] # Only the last definition matters
typed_def = (name, variable.kind)
+ conflict_key = self._conflict_key(name, variable)
- # Check if this is a duplicate definition
- if (
- name in self.definitions
- and typed_def not in self.typed_definitions
- ):
- # Duplicate if the qualified name is no different
- if variable.qualified_name == name or variable.language != "sql":
- self.definitions[name].add(cell_id)
- else:
- self.definitions.setdefault(name, set()).add(cell_id)
-
+ self.definitions.setdefault(name, set()).add(cell_id)
self.typed_definitions.setdefault(typed_def, set()).add(cell_id)
- self.definition_types.setdefault(name, set()).add(variable.kind)
+ self.definition_conflicts.setdefault(conflict_key, set()).add(cell_id)
+ self.conflict_names.setdefault(conflict_key, name)
- # Return siblings (other cells that define this name)
- siblings = self.definitions[name] - {cell_id}
+ # Return siblings (other cells that define the same semantic object)
+ siblings = self.definition_conflicts[conflict_key] - {cell_id}
return siblings
def unregister_definitions(
self,
cell_id: CellId_t,
- defs: set[Name],
+ variable_data: dict[Name, list[VariableData]],
) -> None:
"""Unregister all definitions for a cell.
Args:
cell_id: The cell being removed
- defs: The set of variable names defined by the cell
+ variable_data: Definitions and metadata for the cell
"""
- for name in defs:
+ for name, data in variable_data.items():
if name not in self.definitions:
continue
+ variable = data[-1]
+ typed_def = (name, variable.kind)
+ conflict_key = self._conflict_key(name, variable)
+
name_defs = self.definitions[name]
name_defs.discard(cell_id)
+ if typed_def in self.typed_definitions:
+ self.typed_definitions[typed_def].discard(cell_id)
+ if not self.typed_definitions[typed_def]:
+ del self.typed_definitions[typed_def]
+
+ if conflict_key in self.definition_conflicts:
+ conflict_defs = self.definition_conflicts[conflict_key]
+ conflict_defs.discard(cell_id)
+ if not conflict_defs:
+ del self.definition_conflicts[conflict_key]
+ self.conflict_names.pop(conflict_key, None)
if not name_defs:
# No more cells define this name, so we remove it
del self.definitions[name]
- # Clean up all typed definitions
- for typed_def in self.definition_types.get(name, set()):
- self.typed_definitions.pop((name, typed_def), None)
- self.definition_types.pop(name, None)
def get_defining_cells(self, name: Name) -> set[CellId_t]:
"""Get all cells that define a variable name.
@@ -144,10 +167,25 @@ class DefinitionRegistry:
return matching_cell_ids_list
- def get_multiply_defined(self) -> list[Name]:
- """Return a list of names that are defined in multiple cells."""
- names: list[Name] = []
- for name, definers in self.definitions.items():
- if len(definers) > 1:
- names.append(name)
- return names
+ def get_multiply_defined(self) -> list[tuple[Name, set[CellId_t]]]:
+ """
+ Return multiply-defined names with their conflicting cells.
+
+ Results are sorted by display name and conflict key to keep diagnostics
+ deterministic.
+ """
+
+ def sort_key(
+ item: tuple[tuple[str, str], set[CellId_t]],
+ ) -> tuple[Name, tuple[str, str]]:
+ conflict_key, _definers = item
+ return (self.conflict_names[conflict_key], conflict_key)
+
+ return [
+ (self.conflict_names[conflict_key], definers)
+ for conflict_key, definers in sorted(
+ self.definition_conflicts.items(),
+ key=sort_key,
+ )
+ if len(definers) > 1
+ ]
diff --git a/marimo/_runtime/dataflow/graph.py b/marimo/_runtime/dataflow/graph.py
index f974c3d348..8a0d558b36 100644
--- a/marimo/_runtime/dataflow/graph.py
+++ b/marimo/_runtime/dataflow/graph.py
@@ -210,7 +210,9 @@ class DirectedGraph(GraphTopology):
# Removing this cell from its defs' definer sets
cell = self.topology.cells[cell_id]
- self.definition_registry.unregister_definitions(cell_id, cell.defs)
+ self.definition_registry.unregister_definitions(
+ cell_id, cell.variable_data
+ )
# Remove cycles that are broken from removing this cell
edges = [
@@ -265,6 +267,12 @@ class DirectedGraph(GraphTopology):
def get_multiply_defined(self) -> list[Name]:
"""Return a list of names that are defined in multiple cells."""
+ return [name for name, _ in self.get_multiply_defined_conflicts()]
+
+ def get_multiply_defined_conflicts(
+ self,
+ ) -> list[tuple[Name, set[CellId_t]]]:
+ """Return multiply-defined names with their conflicting cells."""
return self.definition_registry.get_multiply_defined()
def get_deleted_nonlocal_ref(self) -> list[Name]:
diff --git a/tests/_lint/test_validate_graph.py b/tests/_lint/test_validate_graph.py
index 8251f48044..0f2081e2f3 100644
--- a/tests/_lint/test_validate_graph.py
+++ b/tests/_lint/test_validate_graph.py
@@ -4,8 +4,11 @@ from __future__ import annotations
from functools import partial
from typing import cast
+import pytest
+
from marimo._ast import compiler
from marimo._ast.names import SETUP_CELL_NAME
+from marimo._dependencies.dependencies import DependencyManager
from marimo._lint.validate_graph import check_for_errors
from marimo._messaging.errors import (
CycleError,
@@ -13,8 +16,10 @@ from marimo._messaging.errors import (
SetupRootError,
)
from marimo._runtime import dataflow
+from marimo._types.ids import CellId_t
parse_cell = partial(compiler.compile_cell, cell_id="0")
+HAS_DUCKDB = DependencyManager.duckdb.has()
def test_multiple_definition_error() -> None:
@@ -55,6 +60,35 @@ def test_overlapping_multiple_definition_errors() -> None:
assert errors["2"] == (MultipleDefinitionError(name="y", cells=("1",)),)
[email protected](not HAS_DUCKDB, reason="duckdb is required")
+def test_sql_multiple_definition_errors_use_qualified_names() -> None:
+ graph = dataflow.DirectedGraph()
+ graph.register_cell(
+ CellId_t("0"),
+ parse_cell('mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'),
+ )
+ graph.register_cell(
+ CellId_t("1"),
+ parse_cell(
+ 'mo.sql("CREATE TABLE finance.information_layer.xx (id INT)")'
+ ),
+ )
+ graph.register_cell(
+ CellId_t("2"),
+ parse_cell('mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'),
+ )
+
+ errors = check_for_errors(graph)
+
+ assert set(errors.keys()) == {CellId_t("0"), CellId_t("2")}
+ assert errors[CellId_t("0")] == (
+ MultipleDefinitionError(name="xx", cells=(CellId_t("2"),)),
+ )
+ assert errors[CellId_t("2")] == (
+ MultipleDefinitionError(name="xx", cells=(CellId_t("0"),)),
+ )
+
+
def test_underscore_variables_are_private() -> None:
graph = dataflow.DirectedGraph()
graph.register_cell("0", parse_cell("_x = 0"))
diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.py
index 0f54e79216..7ddb050187 100644
--- a/tests/_runtime/test_dataflow.py
+++ b/tests/_runtime/test_dataflow.py
@@ -12,6 +12,7 @@ from marimo._dependencies.dependencies import DependencyManager
from marimo._runtime import dataflow
from marimo._runtime.runner import by_refs
from marimo._runtime.runner.by_refs import _get_ancestors
+from marimo._types.ids import CellId_t
parse_cell = partial(compiler.compile_cell, cell_id="0")
@@ -580,6 +581,90 @@ class TestSQL:
multiply_defined = graph.get_multiply_defined()
assert multiply_defined == []
+ def test_qualified_sql_table_in_defining_cells(self):
+ graph = dataflow.DirectedGraph()
+ code = "t1 = 123"
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE schema1.t1 (i INTEGER, j INTEGER)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.get_defining_cells("t1") == {
+ CellId_t("0"),
+ CellId_t("1"),
+ }
+ assert graph.get_multiply_defined() == []
+
+ def test_redefine_sql_table_same_name_diff_schema(self):
+ graph = dataflow.DirectedGraph()
+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE finance.information_layer.xx (id INT)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.cells == {
+ "0": first_cell,
+ "1": second_cell,
+ }
+ assert graph.get_multiply_defined() == []
+
+ def test_redefine_sql_table_same_name_diff_catalog(self):
+ graph = dataflow.DirectedGraph()
+ code = 'mo.sql("CREATE TABLE catalog_one.schema_name.xx (id INT)")'
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE catalog_two.schema_name.xx (id INT)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.get_multiply_defined() == []
+
+ def test_redefine_sql_table_same_qualified_name(self):
+ graph = dataflow.DirectedGraph()
+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.get_multiply_defined() == ["xx"]
+
+ def test_redefine_sql_table_unqualified_same_name(self):
+ graph = dataflow.DirectedGraph()
+ code = 'mo.sql("CREATE TABLE xx (id INT)")'
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE xx (id INT)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.get_multiply_defined() == ["xx"]
+
+ def test_delete_redefined_sql_table_clears_conflict(self):
+ graph = dataflow.DirectedGraph()
+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'
+ first_cell = parse_cell(code)
+ graph.register_cell(CellId_t("0"), first_cell)
+
+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'
+ second_cell = parse_cell(code)
+ graph.register_cell(CellId_t("1"), second_cell)
+
+ assert graph.get_multiply_defined() == ["xx"]
+
+ graph.delete_cell(CellId_t("1"))
+
+ assert graph.get_multiply_defined() == []
+
def test_sql_table_schema_to_python_ref(self):
graph = dataflow.DirectedGraph()
code = 'df = mo.sql("CREATE TABLE t1 (i INTEGER, j INTEGER)")'