A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.
Python artificial-intelligencedagdata-sciencedata-visualizationdataflowdeveloper-toolsmachine-learningnotebookspipelinepythonreactivesqlweb-app
fix sql defs lookup (#9754)
Shahmir Varqha7745bbf85bca
diff --git a/marimo/_lint/validate_graph.py b/marimo/_lint/validate_graph.pyindex a0d894aba6..88060ffdfa 100644--- a/marimo/_lint/validate_graph.py+++ b/marimo/_lint/validate_graph.py@@ -52,17 +52,14 @@ def check_for_multiple_definitions() -> dict[CellId_t, list[MultipleDefinitionError]]:"""Check whether multiple cells define the same global name."""errors = defaultdict(list)- defs = sorted(set().union(*(cell.defs for cell in graph.cells.values())))- for name in defs:- defining_cells = graph.definitions[name]- if len(defining_cells) > 1:- for cid in defining_cells:- errors[cid].append(- MultipleDefinitionError(- name=str(name),- cells=tuple(sorted(defining_cells - {cid})),- )+ for name, defining_cells in graph.get_multiply_defined_conflicts():+ for cid in defining_cells:+ errors[cid].append(+ MultipleDefinitionError(+ name=str(name),+ cells=tuple(sorted(defining_cells - {cid})),)+ )return errorsdiff --git a/marimo/_runtime/dataflow/definitions.py b/marimo/_runtime/dataflow/definitions.pyindex 0fb622c578..f0c9d7217a 100644--- a/marimo/_runtime/dataflow/definitions.py+++ b/marimo/_runtime/dataflow/definitions.py@@ -26,15 +26,35 @@ class DefinitionRegistry:# A mapping from defs to the cells that define themdefinitions: dict[Name, set[CellId_t]] = field(default_factory=dict)- # Typed definitions for SQL support: (name, type) -> cell_ids+ # Typed definitions: (name, type) -> cell_ids# e.g. ("my_table", "table") -> {cell_id_1}typed_definitions: dict[tuple[Name, str], set[CellId_t]] = field(default_factory=dict)- # Track all types for a given definition name- # e.g. "my_table" -> {"table", "view"}- definition_types: dict[Name, set[str]] = field(default_factory=dict)+ # Track definition conflicts separately from lookup indexes.+ # Qualified SQL definitions with the same leaf name are distinct objects+ # in the database, but still need to be discoverable by their leaf name+ # when computing SQL edges.+ definition_conflicts: dict[tuple[str, str], set[CellId_t]] = field(+ default_factory=dict+ )+ conflict_names: dict[tuple[str, str], Name] = field(default_factory=dict)++ def _conflict_key(+ self, name: Name, variable: VariableData+ ) -> tuple[str, str]:+ """+ Return the key used to group definitions that conflict.+ Only qualified SQL definitions return a non-global key.+ """+ if (+ variable.language == "sql"+ and variable.qualified_name is not None+ and variable.qualified_name != name+ ):+ return ("sql", variable.qualified_name)+ return ("global", name)def register_definition(self,@@ -54,50 +74,53 @@ class DefinitionRegistry:"""variable = variable_data[-1] # Only the last definition matterstyped_def = (name, variable.kind)+ conflict_key = self._conflict_key(name, variable)- # Check if this is a duplicate definition- if (- name in self.definitions- and typed_def not in self.typed_definitions- ):- # Duplicate if the qualified name is no different- if variable.qualified_name == name or variable.language != "sql":- self.definitions[name].add(cell_id)- else:- self.definitions.setdefault(name, set()).add(cell_id)-+ self.definitions.setdefault(name, set()).add(cell_id)self.typed_definitions.setdefault(typed_def, set()).add(cell_id)- self.definition_types.setdefault(name, set()).add(variable.kind)+ self.definition_conflicts.setdefault(conflict_key, set()).add(cell_id)+ self.conflict_names.setdefault(conflict_key, name)- # Return siblings (other cells that define this name)- siblings = self.definitions[name] - {cell_id}+ # Return siblings (other cells that define the same semantic object)+ siblings = self.definition_conflicts[conflict_key] - {cell_id}return siblingsdef unregister_definitions(self,cell_id: CellId_t,- defs: set[Name],+ variable_data: dict[Name, list[VariableData]],) -> None:"""Unregister all definitions for a cell.Args:cell_id: The cell being removed- defs: The set of variable names defined by the cell+ variable_data: Definitions and metadata for the cell"""- for name in defs:+ for name, data in variable_data.items():if name not in self.definitions:continue+ variable = data[-1]+ typed_def = (name, variable.kind)+ conflict_key = self._conflict_key(name, variable)+name_defs = self.definitions[name]name_defs.discard(cell_id)+ if typed_def in self.typed_definitions:+ self.typed_definitions[typed_def].discard(cell_id)+ if not self.typed_definitions[typed_def]:+ del self.typed_definitions[typed_def]++ if conflict_key in self.definition_conflicts:+ conflict_defs = self.definition_conflicts[conflict_key]+ conflict_defs.discard(cell_id)+ if not conflict_defs:+ del self.definition_conflicts[conflict_key]+ self.conflict_names.pop(conflict_key, None)if not name_defs:# No more cells define this name, so we remove itdel self.definitions[name]- # Clean up all typed definitions- for typed_def in self.definition_types.get(name, set()):- self.typed_definitions.pop((name, typed_def), None)- self.definition_types.pop(name, None)def get_defining_cells(self, name: Name) -> set[CellId_t]:"""Get all cells that define a variable name.@@ -144,10 +167,25 @@ class DefinitionRegistry:return matching_cell_ids_list- def get_multiply_defined(self) -> list[Name]:- """Return a list of names that are defined in multiple cells."""- names: list[Name] = []- for name, definers in self.definitions.items():- if len(definers) > 1:- names.append(name)- return names+ def get_multiply_defined(self) -> list[tuple[Name, set[CellId_t]]]:+ """+ Return multiply-defined names with their conflicting cells.++ Results are sorted by display name and conflict key to keep diagnostics+ deterministic.+ """++ def sort_key(+ item: tuple[tuple[str, str], set[CellId_t]],+ ) -> tuple[Name, tuple[str, str]]:+ conflict_key, _definers = item+ return (self.conflict_names[conflict_key], conflict_key)++ return [+ (self.conflict_names[conflict_key], definers)+ for conflict_key, definers in sorted(+ self.definition_conflicts.items(),+ key=sort_key,+ )+ if len(definers) > 1+ ]diff --git a/marimo/_runtime/dataflow/graph.py b/marimo/_runtime/dataflow/graph.pyindex f974c3d348..8a0d558b36 100644--- a/marimo/_runtime/dataflow/graph.py+++ b/marimo/_runtime/dataflow/graph.py@@ -210,7 +210,9 @@ class DirectedGraph(GraphTopology):# Removing this cell from its defs' definer setscell = self.topology.cells[cell_id]- self.definition_registry.unregister_definitions(cell_id, cell.defs)+ self.definition_registry.unregister_definitions(+ cell_id, cell.variable_data+ )# Remove cycles that are broken from removing this celledges = [@@ -265,6 +267,12 @@ class DirectedGraph(GraphTopology):def get_multiply_defined(self) -> list[Name]:"""Return a list of names that are defined in multiple cells."""+ return [name for name, _ in self.get_multiply_defined_conflicts()]++ def get_multiply_defined_conflicts(+ self,+ ) -> list[tuple[Name, set[CellId_t]]]:+ """Return multiply-defined names with their conflicting cells."""return self.definition_registry.get_multiply_defined()def get_deleted_nonlocal_ref(self) -> list[Name]:diff --git a/tests/_lint/test_validate_graph.py b/tests/_lint/test_validate_graph.pyindex 8251f48044..0f2081e2f3 100644--- a/tests/_lint/test_validate_graph.py+++ b/tests/_lint/test_validate_graph.py@@ -4,8 +4,11 @@ from __future__ import annotationsfrom functools import partialfrom typing import cast+import pytest+from marimo._ast import compilerfrom marimo._ast.names import SETUP_CELL_NAME+from marimo._dependencies.dependencies import DependencyManagerfrom marimo._lint.validate_graph import check_for_errorsfrom marimo._messaging.errors import (CycleError,@@ -13,8 +16,10 @@ from marimo._messaging.errors import (SetupRootError,)from marimo._runtime import dataflow+from marimo._types.ids import CellId_tparse_cell = partial(compiler.compile_cell, cell_id="0")+HAS_DUCKDB = DependencyManager.duckdb.has()def test_multiple_definition_error() -> None:@@ -55,6 +60,35 @@ def test_overlapping_multiple_definition_errors() -> None:assert errors["2"] == (MultipleDefinitionError(name="y", cells=("1",)),)[email protected](not HAS_DUCKDB, reason="duckdb is required")+def test_sql_multiple_definition_errors_use_qualified_names() -> None:+ graph = dataflow.DirectedGraph()+ graph.register_cell(+ CellId_t("0"),+ parse_cell('mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'),+ )+ graph.register_cell(+ CellId_t("1"),+ parse_cell(+ 'mo.sql("CREATE TABLE finance.information_layer.xx (id INT)")'+ ),+ )+ graph.register_cell(+ CellId_t("2"),+ parse_cell('mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'),+ )++ errors = check_for_errors(graph)++ assert set(errors.keys()) == {CellId_t("0"), CellId_t("2")}+ assert errors[CellId_t("0")] == (+ MultipleDefinitionError(name="xx", cells=(CellId_t("2"),)),+ )+ assert errors[CellId_t("2")] == (+ MultipleDefinitionError(name="xx", cells=(CellId_t("0"),)),+ )++def test_underscore_variables_are_private() -> None:graph = dataflow.DirectedGraph()graph.register_cell("0", parse_cell("_x = 0"))diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.pyindex 0f54e79216..7ddb050187 100644--- a/tests/_runtime/test_dataflow.py+++ b/tests/_runtime/test_dataflow.py@@ -12,6 +12,7 @@ from marimo._dependencies.dependencies import DependencyManagerfrom marimo._runtime import dataflowfrom marimo._runtime.runner import by_refsfrom marimo._runtime.runner.by_refs import _get_ancestors+from marimo._types.ids import CellId_tparse_cell = partial(compiler.compile_cell, cell_id="0")@@ -580,6 +581,90 @@ class TestSQL:multiply_defined = graph.get_multiply_defined()assert multiply_defined == []+ def test_qualified_sql_table_in_defining_cells(self):+ graph = dataflow.DirectedGraph()+ code = "t1 = 123"+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE schema1.t1 (i INTEGER, j INTEGER)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.get_defining_cells("t1") == {+ CellId_t("0"),+ CellId_t("1"),+ }+ assert graph.get_multiply_defined() == []++ def test_redefine_sql_table_same_name_diff_schema(self):+ graph = dataflow.DirectedGraph()+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE finance.information_layer.xx (id INT)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.cells == {+ "0": first_cell,+ "1": second_cell,+ }+ assert graph.get_multiply_defined() == []++ def test_redefine_sql_table_same_name_diff_catalog(self):+ graph = dataflow.DirectedGraph()+ code = 'mo.sql("CREATE TABLE catalog_one.schema_name.xx (id INT)")'+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE catalog_two.schema_name.xx (id INT)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.get_multiply_defined() == []++ def test_redefine_sql_table_same_qualified_name(self):+ graph = dataflow.DirectedGraph()+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.get_multiply_defined() == ["xx"]++ def test_redefine_sql_table_unqualified_same_name(self):+ graph = dataflow.DirectedGraph()+ code = 'mo.sql("CREATE TABLE xx (id INT)")'+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE xx (id INT)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.get_multiply_defined() == ["xx"]++ def test_delete_redefined_sql_table_clears_conflict(self):+ graph = dataflow.DirectedGraph()+ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'+ first_cell = parse_cell(code)+ graph.register_cell(CellId_t("0"), first_cell)++ code = 'mo.sql("CREATE TABLE finance.datalake.xx (id INT)")'+ second_cell = parse_cell(code)+ graph.register_cell(CellId_t("1"), second_cell)++ assert graph.get_multiply_defined() == ["xx"]++ graph.delete_cell(CellId_t("1"))++ assert graph.get_multiply_defined() == []+def test_sql_table_schema_to_python_ref(self):graph = dataflow.DirectedGraph()code = 'df = mo.sql("CREATE TABLE t1 (i INTEGER, j INTEGER)")'