Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

fix: get fully qualified name in sql refs (#5992)

Shahmir Varqhab5c3f0e9a587
diff --git a/marimo/_ast/cell.py b/marimo/_ast/cell.py
index 4e462de577..d8430969a8 100644
--- a/marimo/_ast/cell.py
+++ b/marimo/_ast/cell.py
@@ -10,7 +10,7 @@ from functools import cached_property
from typing import TYPE_CHECKING, Any, Literal, Optional
from marimo import _loggers
-from marimo._ast.sql_visitor import SQLVisitor
+from marimo._ast.sql_visitor import SQLRef, SQLVisitor
from marimo._ast.visitor import ImportData, Language, Name, VariableData
from marimo._runtime.exceptions import MarimoRuntimeException
from marimo._types.ids import CellId_t
@@ -156,6 +156,8 @@ class CellImpl:
mod: ast.Module
defs: set[Name]
refs: set[Name]
+ # metadata about refs, currently only tracks SQL refs
+ sql_refs: dict[Name, SQLRef]
# Variables that should only live for the duration of the cell
temporaries: set[Name]
diff --git a/marimo/_ast/codegen.py b/marimo/_ast/codegen.py
index a8bc12f04f..f8f019dee8 100644
--- a/marimo/_ast/codegen.py
+++ b/marimo/_ast/codegen.py
@@ -8,6 +8,7 @@ import sys
import textwrap
from typing import TYPE_CHECKING, Any, Literal, Optional
+from marimo import _loggers
from marimo._ast.app_config import _AppConfig
from marimo._ast.cell import CellConfig, CellImpl
from marimo._ast.compiler import compile_cell
@@ -40,6 +41,8 @@ BRACES: dict[Literal["(", "["], tuple[str, str]] = {
Decorators = Literal["cell", "function", "class_definition"]
+LOGGER = _loggers.marimo_logger()
+
def pop_setup_cell(
code: list[str],
@@ -210,7 +213,18 @@ def to_functiondef(
# already in globals)
if allowed_refs is None:
allowed_refs = BUILTINS
- refs = tuple(ref for ref in sorted(cell.refs) if ref not in allowed_refs)
+
+ refs: tuple[str, ...] = tuple()
+ sorted_refs = sorted(cell.refs)
+ for ref in sorted_refs:
+ if ref not in allowed_refs:
+ if not ref.isidentifier():
+ # Filter out refs that are not valid Python identifiers
+ # If not, function signatures will have invalid parameters
+ # eg. `def _(mo, my_schema.my_table):`
+ LOGGER.debug(f"Found non-identifier ref: {ref}")
+ continue
+ refs += (ref,)
# Check to see if any of the refs have an explicit type assigned to them.
annotation_lookups = to_annotated_string(
diff --git a/marimo/_ast/compiler.py b/marimo/_ast/compiler.py
index 645e401c52..ca11ed78e3 100644
--- a/marimo/_ast/compiler.py
+++ b/marimo/_ast/compiler.py
@@ -175,6 +175,7 @@ def compile_cell(
mod=module,
defs=set(),
refs=set(),
+ sql_refs={},
temporaries=set(),
variable_data={},
deleted_refs=set(),
@@ -277,6 +278,7 @@ def compile_cell(
mod=original_module,
defs=nonlocals,
refs=v.refs,
+ sql_refs=v.sql_refs,
temporaries=temporaries,
variable_data=variable_data,
import_workspace=ImportWorkspace(
diff --git a/marimo/_ast/sql_visitor.py b/marimo/_ast/sql_visitor.py
index 82b9b1413f..9af1fff185 100644
--- a/marimo/_ast/sql_visitor.py
+++ b/marimo/_ast/sql_visitor.py
@@ -12,6 +12,16 @@ from marimo._dependencies.dependencies import DependencyManager
LOGGER = _loggers.marimo_logger()
+COMMON_FILE_EXTENSIONS = (
+ ".csv",
+ ".parquet",
+ ".json",
+ ".txt",
+ ".db",
+ ".tsv",
+ ".xlsx",
+)
+
class SQLVisitor(ast.NodeVisitor):
"""
@@ -311,9 +321,79 @@ def find_sql_defs(sql_statement: str) -> SQLDefs:
)
-def find_sql_refs(
- sql_statement: str,
-) -> list[str]:
+@dataclass(frozen=True)
+class SQLRef:
+ # Tables are synonymous with views,
+ # since we can't know the difference in queries
+ table: str
+ schema: Optional[str] = None
+ catalog: Optional[str] = None
+
+ @property
+ def qualified_name(self) -> str:
+ """Convert a SQLRef to a fully qualified name to be used as a reference in the visitor"""
+ parts = []
+ if self.catalog is not None:
+ parts.append(self.catalog)
+ if self.schema is not None:
+ parts.append(self.schema)
+
+ # Table is always required
+ parts.append(self.table)
+ name = ".".join(parts)
+ return name
+
+ def matches_hierarchical_ref(self, name: str, ref: str) -> bool:
+ """
+ Determine if a hierarchical reference string matches a SQLRef.
+
+ Args:
+ name: The name to match against (could be catalog, schema, or table).
+ ref: The fully qualified reference string (e.g., "schema.table", "catalog.schema.table").
+
+ Returns:
+ True if the reference matches the SQLRef's structure and values, False otherwise.
+ """
+ parts = ref.split(".")
+ num_parts = len(parts)
+
+ if num_parts == 1:
+ # Only table name provided
+ return name == self.table == parts[0]
+
+ if num_parts == 2:
+ # Format: schema.table or catalog.table
+ # sqlglot cannot differentiate between schema and catalog
+ # so we check if the qualifier matches either
+ qualifier, table = parts
+ if table != self.table:
+ return False
+ # Try matching as schema or catalog
+ if qualifier not in (self.schema, self.catalog):
+ return False
+ return qualifier == name
+
+ if num_parts == 3:
+ # Format: catalog.schema.table
+ catalog, schema, table = parts
+ if table != self.table:
+ return False
+ if catalog == self.catalog:
+ return name == self.catalog
+ if schema == self.schema:
+ return name == self.schema
+
+ return False
+
+ def contains_hierarchical_ref(self, ref: str, kind: str) -> bool:
+ if kind in ("table", "view"):
+ return ref == self.table
+ if kind == "catalog":
+ return ref == self.catalog or ref == self.schema
+ return False
+
+
+def find_sql_refs(sql_statement: str) -> set[SQLRef]:
"""
Find table and schema references in a SQL statement.
@@ -321,7 +401,18 @@ def find_sql_refs(
sql_statement: The SQL statement to parse.
Returns:
- A list of table and schema names referenced in the statement.
+ A set of unique SQLRefs, one for each table reference in the statement.
+ Eg. SELECT * FROM schema1.test_table INNER JOIN schema2.test_table2
+ would return two SQLRefs, one for the first table and one for the second.
+
+ Note:
+ When providing only a single qualification,
+ DuckDB will interpret as either a catalog or a schema, as long as there are no conflicts.
+
+ Eg. SELECT * FROM my_db.my_table, my_db can be a catalog or schema. If a catalog exists,
+ then it would resolve to my_db.main.my_table.
+
+ At the moment, we don't know this, so my_db is treated as a schema.
"""
# Use sqlglot to parse ast (https://github.com/tobymao/sqlglot/blob/main/posts/ast_primer.md)
@@ -332,30 +423,34 @@ def find_sql_refs(
from sqlglot.errors import ParseError
from sqlglot.optimizer.scope import build_scope
- refs: list[str] = []
+ def get_ref_from_table(table: exp.Table) -> Optional[SQLRef]:
+ # The variables might be empty strings, if they are, we set them to None
+ table_name = table.name or None
+ schema_name = table.db or None
+ catalog_name = table.catalog or None
- def append_refs_from_table(table: exp.Table) -> None:
- if table.catalog == "memory":
- # Default in-memory catalog, only include table name
- refs.append(table.name)
- else:
- # We skip schema if there is a catalog
- # Because it may be called "public" or "main" across all catalogs
- # and they aren't referenced in the code
- if table.catalog:
- refs.append(table.catalog)
- elif table.db:
- refs.append(table.db) # schema
+ if table_name is None:
+ LOGGER.warning("Table name cannot be found in the SQL statement")
+ return None
+
+ # Check if the table name looks like a URL or has a file extension.
+ # These are often not actual table references, so we skip them.
+ # Note that they can be valid table names, but we skip them to avoid circular deps
+ if "://" in table_name or table_name.endswith(COMMON_FILE_EXTENSIONS):
+ return None
- if table.name:
- refs.append(table.name)
+ return SQLRef(
+ table=table_name, schema=schema_name, catalog=catalog_name
+ )
try:
with _loggers.suppress_warnings_logs("sqlglot"):
expression_list = parse(sql_statement, dialect="duckdb")
except ParseError as e:
LOGGER.error(f"Unable to parse SQL. Error: {e}")
- return []
+ return set()
+
+ refs: set[SQLRef] = set()
for expression in expression_list:
if expression is None:
@@ -363,14 +458,15 @@ def find_sql_refs(
if bool(expression.find(exp.Update, exp.Insert, exp.Delete)):
for table in expression.find_all(exp.Table):
- append_refs_from_table(table)
+ if ref := get_ref_from_table(table):
+ refs.add(ref)
# build_scope only works for select statements
if root := build_scope(expression):
for scope in root.traverse(): # type: ignore
for _node, source in scope.selected_sources.values():
if isinstance(source, exp.Table):
- append_refs_from_table(source)
+ if ref := get_ref_from_table(source):
+ refs.add(ref)
- # remove duplicates while preserving order
- return list(dict.fromkeys(refs))
+ return refs
diff --git a/marimo/_ast/visitor.py b/marimo/_ast/visitor.py
index c5a748d80e..31e9217581 100644
--- a/marimo/_ast/visitor.py
+++ b/marimo/_ast/visitor.py
@@ -2,7 +2,6 @@
from __future__ import annotations
import ast
-import itertools
import sys
from collections import defaultdict
from copy import deepcopy
@@ -15,6 +14,7 @@ from marimo import _loggers
from marimo._ast.errors import ImportStarError
from marimo._ast.sql_visitor import (
SQLDefs,
+ SQLRef,
find_sql_defs,
find_sql_refs,
normalize_sql_f_string,
@@ -140,6 +140,8 @@ class RefData:
block: Block
# Ancestors of the block in which this ref was used
parent_blocks: list[Block]
+ # Only applicable for SQL cells
+ sql_ref: Optional[SQLRef] = None
NamedNode = Union[
@@ -161,7 +163,7 @@ NamedNode = Union[
# Cache SQL refs to avoid parsing the same SQL statement multiple times
# since this can be called for each SQL cell on save.
@lru_cache(maxsize=200)
-def find_sql_refs_cached(sql_statement: str) -> list[str]:
+def find_sql_refs_cached(sql_statement: str) -> set[SQLRef]:
return find_sql_refs(sql_statement)
@@ -210,6 +212,17 @@ class ScopedVisitor(ast.NodeVisitor):
"""Names referenced but not defined."""
return set(self._refs.keys())
+ @property
+ def sql_refs(self) -> dict[Name, SQLRef]:
+ """Names and their SQLRefs"""
+ refs = {}
+ for name, ref_data in self._refs.items():
+ # Take the last ref data because it's the most recent ref?
+ sql_ref = ref_data[-1].sql_ref
+ if sql_ref is not None:
+ refs[name] = sql_ref
+ return refs
+
@property
def deleted_refs(self) -> set[Name]:
"""Referenced names that were deleted with `del`."""
@@ -284,7 +297,12 @@ class ScopedVisitor(ast.NodeVisitor):
return any(block.is_defined(identifier) for block in self.block_stack)
def _add_ref(
- self, node: NamedNode | None, name: Name, deleted: bool
+ self,
+ node: NamedNode | None,
+ name: Name,
+ *,
+ deleted: bool,
+ sql_ref: Optional[SQLRef] = None,
) -> None:
"""Register a referenced name."""
if name not in self._refs:
@@ -311,6 +329,7 @@ class ScopedVisitor(ast.NodeVisitor):
deleted=deleted,
parent_blocks=parents,
block=current_block,
+ sql_ref=sql_ref,
)
)
@@ -591,7 +610,7 @@ class ScopedVisitor(ast.NodeVisitor):
first_arg = node.args[0]
sql: Optional[str] = None
if isinstance(first_arg, ast.Constant):
- sql = first_arg.s
+ sql = first_arg.value
elif isinstance(first_arg, ast.JoinedStr):
sql = normalize_sql_f_string(first_arg)
@@ -603,6 +622,8 @@ class ScopedVisitor(ast.NodeVisitor):
and sql
):
import duckdb # type: ignore[import-not-found,import-untyped,unused-ignore] # noqa: E501
+ # TODO: Handle other SQL languages
+ # TODO: Get the engine so we can differentiate tables in diff engines
# Add all tables in the query to the ref scope
try:
@@ -629,22 +650,10 @@ class ScopedVisitor(ast.NodeVisitor):
return node
for statement in statements:
- tables: set[str] = set()
- from_targets: list[str] = []
+ sql_refs: set[SQLRef] = set()
# Parse the refs and defs of each statement
try:
- tables = duckdb.get_table_names(statement.query)
- except (duckdb.ProgrammingError, duckdb.IOException):
- LOGGER.debug(
- "Error parsing SQL statement: %s", statement.query
- )
- except BaseException as e:
- LOGGER.warning("Unexpected duckdb error %s", e)
- try:
- # TODO(akshayka): more comprehensive parsing
- # of the statement -- schemas can show up in
- # joins, queries, ...
- from_targets = find_sql_refs_cached(statement.query)
+ sql_refs = find_sql_refs_cached(statement.query)
except (duckdb.ProgrammingError, duckdb.IOException):
LOGGER.debug(
"Error parsing SQL statement: %s", statement.query
@@ -652,11 +661,9 @@ class ScopedVisitor(ast.NodeVisitor):
except BaseException as e:
LOGGER.warning("Unexpected duckdb error %s", e)
- for name in itertools.chain(tables, from_targets):
- # Name (table, db) may be a URL or something else that
- # isn't a Python variable
- if name.isidentifier():
- self._add_ref(None, name, deleted=False)
+ for ref in sql_refs:
+ name = ref.qualified_name
+ self._add_ref(None, name, deleted=False, sql_ref=ref)
# Add all tables/dbs created in the query to the defs
try:
diff --git a/marimo/_runtime/dataflow.py b/marimo/_runtime/dataflow.py
index 9f71f866b3..6c968c9aad 100644
--- a/marimo/_runtime/dataflow.py
+++ b/marimo/_runtime/dataflow.py
@@ -11,6 +11,7 @@ from marimo._ast.cell import (
CellImpl,
)
from marimo._ast.compiler import code_key
+from marimo._ast.sql_visitor import SQLRef
from marimo._ast.variables import is_mangled_local
from marimo._ast.visitor import ImportData, Name, VariableData
from marimo._runtime.executor import (
@@ -96,17 +97,75 @@ class DirectedGraph:
"""
if language == "sql":
# For SQL, only return SQL cells that reference the name
- return {
- cid
- for cid, cell in self.cells.items()
- if name in cell.refs and cell.language == "sql"
- }
+ cells = set()
+ for cid, cell in self.cells.items():
+ if cell.language != "sql":
+ continue
+
+ for ref in cell.refs:
+ # Direct reference match
+ if ref == name:
+ cells.add(cid)
+ break
+
+ sql_ref = cell.sql_refs.get(ref)
+
+ # Hierarchical reference match
+ if sql_ref and sql_ref.matches_hierarchical_ref(name, ref):
+ cells.add(cid)
+ break
+
+ return cells
else:
# For Python, return all cells that reference the name
return {
cid for cid, cell in self.cells.items() if name in cell.refs
}
+ def _find_sql_hierarchical_matches(
+ self, sql_ref: SQLRef
+ ) -> tuple[set[CellId_t], Name]:
+ """
+ This method searches through all definitions in the graph to find cells
+ that define the individual components (table, schema, or catalog) of the
+ hierarchical reference.
+
+ For example, given a reference "my_schema.my_table", this method will:
+ - Look for cells that define a table/view named "my_table"
+ - Look for cells that define a catalog named "my_schema"
+ (when the reference has at least 2 parts)
+
+ Args:
+ sql_ref: A hierarchical SQL reference (e.g., "schema.table",
+ "catalog.schema.table") to find matching definitions for.
+
+ Returns:
+ A tuple containing:
+ - A set of cell IDs that define components of the hierarchical reference
+ - The definition of the name that was found (e.g., "schema.table" -> "table")
+ """
+ variable_name: Name = sql_ref.qualified_name
+ matching_cell_ids = set()
+
+ for def_name, cell_ids in self.definitions.items():
+ # NB. Only the last definition matters.
+ # Technically more nuanced with branching statements, but this is
+ # the best we can do with static analysis.
+ var_data = [
+ v[-1]
+ for cell_id in cell_ids
+ if (v := self.cells[cell_id].variable_data.get(def_name, []))
+ ]
+ # Only consider definitions with a single variable data entry
+ # since multiple defining cells is undefined behavior
+ var, *_ = var_data
+ # Match table/view definitions
+ if sql_ref.contains_hierarchical_ref(def_name, var.kind):
+ variable_name = def_name
+ matching_cell_ids.update(cell_ids)
+
+ return matching_cell_ids, variable_name
+
def get_path(self, source: CellId_t, dst: CellId_t) -> list[Edge]:
"""Get a path from `source` to `dst`, if any."""
if source == dst:
@@ -196,11 +255,23 @@ class DirectedGraph:
if name in self.definitions
else set()
) - set((cell_id,))
+
+ variable_name: Name = name
+ if not other_ids_defining_name:
+ # Handle SQL matching for hierarchical references
+ sql_ref = cell.sql_refs.get(name)
+ if sql_ref:
+ other_ids_defining_name, variable_name = (
+ self._find_sql_hierarchical_matches(sql_ref)
+ )
+
# If other_ids_defining_name is empty, the user will get a
# NameError at runtime (unless the symbol is a builtin).
for other_id in other_ids_defining_name:
language = (
- self.cells[other_id].variable_data[name][-1].language
+ self.cells[other_id]
+ .variable_data[variable_name][-1]
+ .language
)
if language == "sql" and cell.language == "python":
# SQL table/db def -> Python ref is not an edge
@@ -400,6 +471,7 @@ class DirectedGraph:
return imports
def get_multiply_defined(self) -> list[Name]:
+ """Return a list of names that are defined in multiple cells"""
names: list[Name] = []
for name, definers in self.definitions.items():
if len(definers) > 1:
diff --git a/marimo/_utils/timer.py b/marimo/_utils/timer.py
index 162d136298..8f7578bd4b 100644
--- a/marimo/_utils/timer.py
+++ b/marimo/_utils/timer.py
@@ -1,3 +1,4 @@
+# Copyright 2025 Marimo. All rights reserved.
import functools
import time
from typing import Any, Callable
diff --git a/tests/_ast/test_codegen.py b/tests/_ast/test_codegen.py
index f5ac75b566..9a8a5e603e 100644
--- a/tests/_ast/test_codegen.py
+++ b/tests/_ast/test_codegen.py
@@ -559,6 +559,31 @@ class TestToFunctionDef:
)
assert fndef == expected
+ def test_dotted_names_filtered_from_signature(self) -> None:
+ """Test that dotted names (like SQL schema.table references) are filtered out from function signatures."""
+ # Create a cell with a dotted reference in refs (simulating SQL schema.table)
+ code = "result = some_function()"
+ cell = compile_cell(code)
+
+ # Manually add a dotted reference to simulate SQL schema.table reference
+ # This would normally happen when SQL references are parsed
+ cell.refs.add("my_schema.pokemon_db")
+
+ fndef = codegen.to_functiondef(cell, "foo")
+ expected = "\n".join(
+ [
+ "@app.cell",
+ "def foo(some_function):",
+ " result = some_function()",
+ " return (result,)",
+ ]
+ )
+ assert fndef == expected
+
+ # Verify that the dotted name is not in the function signature
+ assert "my_schema.pokemon_db" not in fndef
+ assert "def foo(my_schema.pokemon_db" not in fndef
+
def test_should_remove_defaults(self) -> None:
code = "x = 0"
cell = compile_cell(code)
diff --git a/tests/_ast/test_sql_visitor.py b/tests/_ast/test_sql_visitor.py
index a9a0c8aa1c..e08f10a1f0 100644
--- a/tests/_ast/test_sql_visitor.py
+++ b/tests/_ast/test_sql_visitor.py
@@ -7,6 +7,7 @@ import pytest
from marimo._ast.sql_visitor import (
SQLDefs,
+ SQLRef,
SQLVisitor,
find_sql_defs,
find_sql_refs,
@@ -470,102 +471,140 @@ def test_find_sql_defs_duckdb_not_available() -> None:
assert find_sql_defs("CREATE TABLE test (id INT);") == SQLDefs()
+class TestSQLRefsConvertName:
+ def test_simple(self) -> None:
+ ref = SQLRef(table="test_table")
+ assert ref.qualified_name == "test_table"
+
+ def test_with_catalog(self) -> None:
+ ref = SQLRef(table="test_table", catalog="test_catalog")
+ assert ref.qualified_name == "test_catalog.test_table"
+
+ def test_with_schema(self) -> None:
+ ref = SQLRef(table="test_table", schema="test_schema")
+ assert ref.qualified_name == "test_schema.test_table"
+
+ def test_with_catalog_and_schema(self) -> None:
+ ref = SQLRef(
+ table="test_table", catalog="test_catalog", schema="test_schema"
+ )
+ assert ref.qualified_name == "test_catalog.test_schema.test_table"
+
+ def test_with_catalog_and_table(self) -> None:
+ ref = SQLRef(table="test_table", catalog="test_catalog")
+ assert ref.qualified_name == "test_catalog.test_table"
+
+
@pytest.mark.skipif(not HAS_SQLGLOT, reason="Missing sqlglot")
class TestFindSQLRefs:
- @staticmethod
- def test_find_sql_refs_simple() -> None:
+ def test_simple(self) -> None:
sql = "SELECT * FROM test_table;"
- assert find_sql_refs(sql) == ["test_table"]
+ assert find_sql_refs(sql) == {SQLRef(table="test_table")}
- @staticmethod
- def test_find_sql_refs_multiple() -> None:
+ def test_multiple(self) -> None:
sql = """
SELECT * FROM table1;
SELECT * FROM table2;
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_without_duplicates() -> None:
+ def test_without_duplicates(self) -> None:
sql = """
SELECT * FROM table1;
SELECT * FROM table2;
SELECT * FROM table1;
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_with_function() -> None:
+ def test_with_function(self) -> None:
sql = """
SELECT *, embedding(text) as text_embedding
FROM prompts;
"""
- assert find_sql_refs(sql) == ["prompts"]
-
- @staticmethod
- def test_find_sql_refs_with_schema() -> None:
- sql = "SELECT * FROM my_schema.my_table;"
- assert find_sql_refs(sql) == ["my_schema", "my_table"]
-
- @staticmethod
- def test_find_sql_refs_with_catalog() -> None:
- # Skip the schema if it's coming from a catalog
- sql = "SELECT * FROM my_catalog.my_schema.my_table;"
- assert find_sql_refs(sql) == ["my_catalog", "my_table"]
-
- @staticmethod
- def test_find_sql_refs_skip_memory_main() -> None:
- # This is the default in-memory catalog and schema
- # and we don't want to include them in the references
- sql = "SELECT * FROM memory.main.my_table;"
- assert find_sql_refs(sql) == ["my_table"]
-
- @staticmethod
- def test_find_sql_refs_with_join() -> None:
- sql = """
- SELECT * FROM table1
- JOIN table2 ON table1.id = table2.id;
- """
- assert find_sql_refs(sql) == ["table1", "table2"]
-
- @staticmethod
- def test_find_sql_refs_with_subquery() -> None:
+ assert find_sql_refs(sql) == {SQLRef(table="prompts")}
+
+ def test_with_schema(self) -> None:
+ sql = "SELECT * FROM schema1.table1;"
+ assert find_sql_refs(sql) == {SQLRef(table="table1", schema="schema1")}
+
+ def test_with_catalog(self) -> None:
+ sql = "SELECT * FROM catalog1.schema1.table1;"
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="schema1", catalog="catalog1")
+ }
+
+ def test_memory_catalog(self) -> None:
+ sql = "SELECT * FROM memory.main.table1;"
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="main", catalog="memory")
+ }
+
+ def test_with_multiple_tables(self) -> None:
+ sql = "SELECT * FROM table1 JOIN table2 ON table1.id = table2.id;"
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
+
+ def test_with_join_schema(self) -> None:
+ sql = "SELECT * FROM schema1.table1 JOIN schema2.table2 ON table1.id = table2.id;"
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="schema1"),
+ SQLRef(table="table2", schema="schema2"),
+ }
+
+ def test_with_subquery(self) -> None:
+ sql = "SELECT * FROM table1 WHERE id IN (SELECT id FROM table2);"
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table2"),
+ SQLRef(table="table1"),
+ }
+
+ def test_with_subquery_2(self) -> None:
sql = """
SELECT * FROM (
SELECT * FROM inner_table
) t;
"""
- assert find_sql_refs(sql) == ["inner_table"]
+ assert find_sql_refs(sql) == {SQLRef(table="inner_table")}
- @staticmethod
- def test_find_sql_refs_with_cte() -> None:
+ def test_with_cte(self) -> None:
sql = """
WITH cte AS (
SELECT * FROM source_table
)
SELECT * FROM cte;
"""
- assert find_sql_refs(sql) == ["source_table"]
+ assert find_sql_refs(sql) == {SQLRef(table="source_table")}
- @staticmethod
- def test_find_sql_refs_with_union() -> None:
+ def test_with_union(self) -> None:
sql = """
SELECT * FROM table1
UNION
SELECT * FROM table2;
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_with_quoted_names() -> None:
+ def test_with_quoted_names(self) -> None:
sql = """
SELECT * FROM "My Table"
JOIN "Weird.Name" ON "My Table".id = "Weird.Name".id;
"""
- assert find_sql_refs(sql) == ["My Table", "Weird.Name"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="My Table"),
+ SQLRef(table="Weird.Name"),
+ }
- @staticmethod
- def test_find_sql_refs_with_multiple_ctes() -> None:
+ def test_with_multiple_ctes(self) -> None:
sql = """
WITH
cte1 AS (SELECT * FROM table1),
@@ -573,35 +612,44 @@ class TestFindSQLRefs:
cte3 AS (SELECT * FROM cte1 JOIN cte2)
SELECT * FROM cte3;
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_with_nested_joins() -> None:
+ def test_with_nested_joins(self) -> None:
sql = """
SELECT * FROM t1
JOIN (t2 JOIN t3 ON t2.id = t3.id)
ON t1.id = t2.id;
"""
- assert find_sql_refs(sql) == ["t1", "t2", "t3"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="t1"),
+ SQLRef(table="t2"),
+ SQLRef(table="t3"),
+ }
- @staticmethod
- def test_find_sql_refs_with_lateral_join() -> None:
+ def test_with_lateral_join(self) -> None:
sql = """
SELECT * FROM employees,
LATERAL (SELECT * FROM departments WHERE departments.id = employees.dept_id) dept;
"""
- assert find_sql_refs(sql) == ["departments", "employees"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="departments"),
+ SQLRef(table="employees"),
+ }
- @staticmethod
- def test_find_sql_refs_with_schema_switching() -> None:
+ def test_with_schema_switching(self) -> None:
sql = """
SELECT * FROM schema1.table1
JOIN schema2.table2 ON schema1.table1.id = schema2.table2.id;
"""
- assert find_sql_refs(sql) == ["schema1", "table1", "schema2", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="schema1"),
+ SQLRef(table="table2", schema="schema2"),
+ }
- @staticmethod
- def test_find_sql_refs_with_complex_subqueries() -> None:
+ def test_with_complex_subqueries(self) -> None:
sql = """
SELECT * FROM (
SELECT * FROM (
@@ -610,10 +658,12 @@ class TestFindSQLRefs:
JOIN another_table
) t2;
"""
- assert find_sql_refs(sql) == ["deeply", "table", "another_table"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table", schema="nested", catalog="deeply"),
+ SQLRef(table="another_table"),
+ }
- @staticmethod
- def test_find_sql_refs_nested_intersect() -> None:
+ def test_nested_intersect(self) -> None:
sql = """
SELECT * FROM table1
WHERE id IN (
@@ -624,30 +674,31 @@ class TestFindSQLRefs:
SELECT id FROM table4
);
"""
- assert find_sql_refs(sql) == ["table2", "table3", "table4", "table1"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table2"),
+ SQLRef(table="table3"),
+ SQLRef(table="table4"),
+ SQLRef(table="table1"),
+ }
- @staticmethod
- def test_find_sql_refs_with_alias() -> None:
+ def test_with_alias(self) -> None:
sql = "SELECT * FROM employees AS e;"
- assert find_sql_refs(sql) == ["employees"]
+ assert find_sql_refs(sql) == {SQLRef(table="employees")}
- @staticmethod
- def test_find_sql_refs_comment() -> None:
+ def test_comment(self) -> None:
sql = """
-- comment
SELECT * FROM table1;
-- comment
"""
- assert find_sql_refs(sql) == ["table1"]
+ assert find_sql_refs(sql) == {SQLRef(table="table1")}
- @staticmethod
- def test_find_sql_refs_ddl() -> None:
+ def test_ddl(self) -> None:
# we are not referencing any table hence no refs
sql = "CREATE TABLE t1 (id int);"
- assert find_sql_refs(sql) == []
+ assert find_sql_refs(sql) == set()
- @staticmethod
- def test_find_sql_refs_ddl_with_reference() -> None:
+ def test_ddl_with_reference(self) -> None:
sql = """
CREATE TABLE table2 AS
WITH x AS (
@@ -655,34 +706,39 @@ class TestFindSQLRefs:
)
SELECT * FROM x;
"""
- assert find_sql_refs(sql) == ["table1"]
+ assert find_sql_refs(sql) == {SQLRef(table="table1")}
- @staticmethod
- def test_find_sql_refs_update() -> None:
+ def test_update(self) -> None:
sql = "UPDATE my_schema.table1 SET id = 1"
- assert find_sql_refs(sql) == ["my_schema", "table1"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="my_schema")
+ }
- @staticmethod
- def test_find_sql_refs_insert() -> None:
+ def test_insert(self) -> None:
sql = "INSERT INTO my_schema.table1 (id INT) VALUES (1,2);"
- assert find_sql_refs(sql) == ["my_schema", "table1"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="my_schema")
+ }
- @staticmethod
- def test_find_sql_refs_delete() -> None:
+ def test_delete(self) -> None:
sql = "DELETE FROM my_schema.table1 WHERE true;"
- assert find_sql_refs(sql) == ["my_schema", "table1"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="my_schema")
+ }
- @staticmethod
- def test_find_sql_refs_multi_dml() -> None:
+ def test_multi_dml(self) -> None:
sql = """
INSERT INTO table1 (id INT) VALUES (1,2);
DELETE FROM table2 WHERE true;
UPDATE table3 SET id = 1;
"""
- assert find_sql_refs(sql) == ["table1", "table2", "table3"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ SQLRef(table="table3"),
+ }
- @staticmethod
- def test_find_sql_refs_multiple_selects_in_update() -> None:
+ def test_multiple_selects_in_update(self) -> None:
sql = """
UPDATE schema1.table1
SET table1.column1 = (
@@ -698,26 +754,26 @@ class TestFindSQLRefs:
SELECT table4.column4 FROM table4
);
"""
- assert find_sql_refs(sql) == [
- "schema1",
- "table1",
- "schema2",
- "table2",
- "table3",
- "table4",
- ]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1", schema="schema1"),
+ SQLRef(table="table2", schema="schema2"),
+ SQLRef(table="table3"),
+ SQLRef(table="table2"), # Table2 comes from main schema
+ SQLRef(table="table4"),
+ }
- @staticmethod
- def test_find_sql_refs_select_in_insert() -> None:
+ def test_select_in_insert(self) -> None:
sql = """
INSERT INTO table1 (column1, column2)
SELECT column1, column2 FROM table2
WHERE column3 = 'value';
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_select_in_delete() -> None:
+ def test_select_in_delete(self) -> None:
sql = """
DELETE FROM table1
WHERE column1 IN (
@@ -725,53 +781,60 @@ class TestFindSQLRefs:
WHERE column2 = 'value'
);
"""
- assert find_sql_refs(sql) == ["table1", "table2"]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
- @staticmethod
- def test_find_sql_refs_invalid_sql() -> None:
+ def test_invalid_sql(self) -> None:
sql = "SELECT * FROM"
- assert find_sql_refs(sql) == []
+ assert find_sql_refs(sql) == set()
- @staticmethod
- def test_dml_with_subquery() -> None:
+ def test_dml_with_subquery(self) -> None:
# first
sql = """
insert into table1 (column1) select distinct column1 from table2 order by random();
"""
- assert find_sql_refs(sql) == [
- "table1",
- "table2",
- ]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ }
# second
sql = """
update table3 set column1=(select column2 from table1 t where t.column1=table3.column1);
"""
- assert find_sql_refs(sql) == [
- "table3",
- "table1",
- ]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table3"),
+ SQLRef(table="table1"),
+ }
# combined
sql = """
insert into table1 (column1) select distinct column1 from table2 order by random();
update table3 set column1=(select column2 from table1 t where t.column1=table3.column1);
"""
- assert find_sql_refs(sql) == [
- "table1",
- "table2",
- "table3",
- ]
+ assert find_sql_refs(sql) == {
+ SQLRef(table="table1"),
+ SQLRef(table="table2"),
+ SQLRef(table="table3"),
+ }
- @staticmethod
- def test_find_sql_refs_with_regex() -> None:
+ def test_with_regex(self) -> None:
# regex and replacement
sql = """
SELECT name, REGEXP_REPLACE(name, '^.*/', '') regex_name
FROM pdb.database
LIMIT 10;
"""
- assert find_sql_refs(sql) == [
- "pdb",
- "database",
- ]
+ assert find_sql_refs(sql) == {SQLRef(table="database", schema="pdb")}
+
+ def test_read_file_and_urls(self) -> None:
+ sql = "SELECT * FROM 'file.csv'"
+ assert find_sql_refs(sql) == set()
+
+ sql = "SELECT * FROM 'https://example.com/file.csv'"
+ assert find_sql_refs(sql) == set()
+
+ sql = "SELECT * FROM read_csv('/dev/stdin')"
+ assert find_sql_refs(sql) == set()
diff --git a/tests/_runtime/test_dataflow.py b/tests/_runtime/test_dataflow.py
index 46e835c209..4834f090c2 100644
--- a/tests/_runtime/test_dataflow.py
+++ b/tests/_runtime/test_dataflow.py
@@ -564,6 +564,45 @@ class TestSQL:
"3": set(),
}
+ @pytest.mark.xfail(reason="TODO: A bug in finding multiply defined names")
+ def test_redefine_sql_table_diff_schema(self):
+ graph = dataflow.DirectedGraph()
+ code = "t1 = 123"
+ first_cell = parse_cell(code)
+ graph.register_cell("0", first_cell)
+
+ code = 'df = mo.sql("CREATE TABLE schema1.t1 (i INTEGER, j INTEGER)")'
+ second_cell = parse_cell(code)
+ graph.register_cell("1", second_cell)
+
+ assert graph.cells == {
+ "0": first_cell,
+ "1": second_cell,
+ }
+
+ # Because t1 is qualified with schema1, it is not considered multiply defined
+ multiply_defined = graph.get_multiply_defined()
+ assert multiply_defined == []
+
+ def test_sql_table_schema_to_python_ref(self):
+ graph = dataflow.DirectedGraph()
+ code = 'df = mo.sql("CREATE TABLE t1 (i INTEGER, j INTEGER)")'
+ first_cell = parse_cell(code)
+ graph.register_cell("0", first_cell)
+
+ code = "t1 = 123"
+ second_cell = parse_cell(code)
+ graph.register_cell("1", second_cell)
+
+ assert graph.cells == {
+ "0": first_cell,
+ "1": second_cell,
+ }
+
+ multiply_defined = graph.get_multiply_defined()
+ # Without the qualification, t1 is considered multiply defined
+ assert multiply_defined == ["t1"]
+
def test_no_sql_table_to_python_ref(self):
graph = dataflow.DirectedGraph()
code = 'df = mo.sql("CREATE TABLE t1 (i INTEGER, j INTEGER)")'
@@ -677,6 +716,24 @@ class TestSQL:
== set()
)
+ def test_referring_cells_sql_and_python(self) -> None:
+ graph = dataflow.DirectedGraph()
+ code = 'df = mo.sql("select * from my_schema.my_table")'
+ first_cell = parse_cell(code)
+ assert first_cell.refs == {"mo", "my_schema.my_table"}
+
+ graph.register_cell("0", first_cell)
+
+ # my_table should not be passed in as a reference to sql cell
+ code = "df; my_table = ..."
+ second_cell = parse_cell(code)
+ graph.register_cell("1", second_cell)
+
+ assert (
+ graph.get_referring_cells("my_table", language="python") == set()
+ )
+ assert not graph.cycles
+
def test_attached_db(self):
graph = dataflow.DirectedGraph()
code = "mo.sql(\"ATTACH 'my_db.db'\")"
diff --git a/tests/_runtime/test_dataflow_cases.py b/tests/_runtime/test_dataflow_cases.py
index 4d03b9f0dc..fd6428bfd3 100644
--- a/tests/_runtime/test_dataflow_cases.py
+++ b/tests/_runtime/test_dataflow_cases.py
@@ -20,6 +20,10 @@ if TYPE_CHECKING:
from collections.abc import Iterable
+class KnownFailure(Exception):
+ """An expected failure in a test case."""
+
+
@dataclass
class GraphTestCase:
"""A test case for dataflow graph operations."""
@@ -70,7 +74,7 @@ class GraphTestCase:
}
-CASES = [
+PYTHON_CASES = [
# Basic Python Cases
GraphTestCase(
name="single node",
@@ -145,7 +149,9 @@ CASES = [
"2": [],
},
),
- # SQL Cases
+]
+
+SQL_CASES = [
GraphTestCase(
name="python -> sql",
code={
@@ -191,7 +197,7 @@ CASES = [
expected_defs={"0": ["df"], "1": ["df"]},
),
GraphTestCase(
- name="python and sql not related because has schema",
+ name="sql should not reference python variables when schema",
enabled=HAS_DUCKDB,
code={
"0": "df = pd.read_csv('data.csv')",
@@ -199,11 +205,10 @@ CASES = [
},
expected_parents={"0": [], "1": []},
expected_children={"0": [], "1": []},
- expected_refs={"0": ["pd"], "1": ["df", "mo", "my_schema"]},
+ expected_refs={"0": ["pd"], "1": ["mo", "my_schema.df"]},
# This is correct
# expected_refs={"0": ["pd"], "1": ["df.my_schema", "mo"]},
expected_defs={"0": ["df"], "1": ["result"]},
- xfail=True,
),
GraphTestCase(
name="sql should not reference python variables when schema",
@@ -214,11 +219,10 @@ CASES = [
},
expected_parents={"0": [], "1": []},
expected_children={"0": [], "1": []},
- expected_refs={"0": [], "1": ["mo", "my_schema", "df"]},
+ expected_refs={"0": [], "1": ["mo", "my_schema.df"]},
# This is correct
# expected_refs={"0": ["pd"], "1": ["my_schema.df", "mo"]},
expected_defs={"0": ["my_schema"], "1": []},
- xfail=True,
),
GraphTestCase(
name="sql should not reference python variables when catalog",
@@ -229,14 +233,241 @@ CASES = [
},
expected_parents={"0": [], "1": []},
expected_children={"0": [], "1": []},
- expected_refs={"0": [], "1": ["mo", "my_catalog", "my_schema", "df"]},
+ expected_refs={"0": [], "1": ["mo", "my_catalog.my_schema.df"]},
# This is correct
# expected_refs={"0": ["pd"], "1": ["my_catalog.my_schema.df", "mo"]},
expected_defs={"0": ["my_catalog"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql table reference resolves to table name even if created with schema",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_df = mo.sql(f'CREATE TABLE my_schema.my_table (name STRING)')",
+ "1": "_df = mo.sql(f'FROM my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={"0": ["mo"], "1": ["my_table", "mo"]},
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql table reference resolves to table name even if created with catalog and schema",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_df = mo.sql(f'CREATE TABLE my_catalog.my_schema.my_table (name STRING)')",
+ "1": "_df = mo.sql(f'FROM my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={"0": ["mo"], "1": ["my_table", "mo"]},
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql table created from another table reference",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_df = mo.sql(f'CREATE TABLE schema_one.my_table (name STRING)')",
+ "1": "_df = mo.sql(f'CREATE TABLE schema_two.my_table_two AS SELECT * FROM schema_one.my_table')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={"0": ["mo"], "1": ["mo", "schema_one.my_table"]},
+ expected_defs={"0": ["my_table"], "1": ["my_table_two"]},
+ ),
+ GraphTestCase(
+ name="sql table reference with catalog and schema",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE my_catalog.my_schema.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM my_catalog.my_schema.my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["my_catalog.my_schema.my_table", "mo"],
+ },
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
+ GraphTestCase(
+ name="different schemas with same table name",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_df = mo.sql(f'CREATE TABLE schema_one.my_table (name STRING)')",
+ "1": "_df = mo.sql(f'CREATE TABLE schema_two.my_table (name STRING)')",
+ "2": "_df = mo.sql(f'FROM schema_one.my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": [], "2": ["0"]},
+ expected_children={"0": ["2"], "1": [], "2": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo"],
+ "2": ["mo", "schema_one.my_table"],
+ },
+ # TODO: What should the defs be?
+ expected_defs={"0": ["my_table"], "1": ["my_table"], "2": []},
+ # currently:
+ # {'2': {'1', '0'}}
xfail=True,
),
+ GraphTestCase(
+ name="sql definitions with same name as qualified schema and table",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "my_table = mo.sql(f'CREATE TABLE schema_one.my_table (name STRING)')",
+ "1": "schema_one = mo.sql(f'CREATE TABLE schema_one.my_table (name STRING)')",
+ "2": "my_table",
+ "3": "schema_one",
+ },
+ expected_parents={"0": [], "1": [], "2": ["0"], "3": ["1"]},
+ expected_children={"0": ["2"], "1": ["3"], "2": [], "3": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo"],
+ "2": ["my_table"],
+ "3": ["schema_one"],
+ },
+ expected_defs={
+ "0": ["my_table"],
+ "1": ["my_table", "schema_one"],
+ },
+ ),
+ GraphTestCase(
+ name="sql catalog and schema with same name",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE my_db.my_db.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM my_db.my_db.my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "my_db.my_db.my_table"],
+ },
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql view creation with schema reference",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE my_schema.base_table (id INT)')",
+ "1": "_ = mo.sql(f'CREATE VIEW my_schema.my_view AS SELECT * FROM my_schema.base_table')",
+ "2": "_ = mo.sql(f'FROM my_schema.my_view SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"], "2": ["1"]},
+ expected_children={"0": ["1"], "1": ["2"], "2": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "my_schema.base_table"],
+ "2": ["mo", "my_schema.my_view"],
+ },
+ expected_defs={"0": ["base_table"], "1": ["my_view"], "2": []},
+ ),
+ GraphTestCase(
+ name="sql case insensitive schema matching",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE MY_SCHEMA.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM my_schema.my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"]},
+ expected_children={"0": ["1"], "1": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "my_schema.my_table"],
+ },
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql no reference to python variable when using catalog",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "my_catalog_var = 'test_catalog'",
+ "1": "_ = mo.sql(f'FROM my_catalog_var.schema.table SELECT *')",
+ },
+ expected_parents={"0": [], "1": []},
+ expected_children={"0": [], "1": []},
+ expected_refs={
+ "0": [],
+ "1": ["mo", "my_catalog_var.schema.table"],
+ },
+ expected_defs={"0": ["my_catalog_var"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql no reference to python variable when using schema",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "my_schema_var = 'test_schema'",
+ "1": "_ = mo.sql(f'FROM my_schema_var.table SELECT *')",
+ },
+ expected_parents={"0": [], "1": []},
+ expected_children={"0": [], "1": []},
+ expected_refs={
+ "0": [],
+ "1": ["mo", "my_schema_var.table"],
+ },
+ expected_defs={"0": ["my_schema_var"], "1": []},
+ ),
+ GraphTestCase(
+ name="sql catalog.schema.table requires both catalog and schema to match",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE catalog_one.schema_one.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM catalog_one.schema_one.my_table SELECT *')",
+ "2": "_ = mo.sql(f'FROM catalog_two.schema_one.my_table SELECT *')",
+ "3": "_ = mo.sql(f'FROM catalog_one.schema_two.my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": ["0"], "2": [], "3": []},
+ expected_children={"0": ["1"], "1": [], "2": [], "3": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "catalog_one.schema_one.my_table"],
+ "2": ["mo", "catalog_two.schema_one.my_table"],
+ "3": ["mo", "catalog_one.schema_two.my_table"],
+ },
+ expected_defs={"0": ["my_table"], "1": [], "2": [], "3": []},
+ # Currently:
+ # '2': {'0'},
+ # '3': {'0'},
+ xfail=True,
+ ),
+ GraphTestCase(
+ name="sql table substring doesn't cause false positive",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE catalog_one.schema_one.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM catalog_one.schema_one.my_table_suffix SELECT *')",
+ "2": "_ = mo.sql(f'FROM catalog_one.schema_one.prefix_my_table SELECT *')",
+ },
+ expected_parents={"0": [], "1": [], "2": []},
+ expected_children={"0": [], "1": [], "2": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "catalog_one.schema_one.my_table_suffix"],
+ "2": ["mo", "catalog_one.schema_one.prefix_my_table"],
+ },
+ expected_defs={"0": ["my_table"], "1": [], "2": []},
+ ),
+ GraphTestCase(
+ name="sql table schema substring doesn't cause false positive",
+ enabled=HAS_DUCKDB,
+ code={
+ "0": "_ = mo.sql(f'CREATE TABLE catalog_one.schema_one.my_table (name STRING)')",
+ "1": "_ = mo.sql(f'FROM catalog_one.my_table.suffix SELECT *')",
+ },
+ expected_parents={"0": [], "1": []},
+ expected_children={"0": [], "1": []},
+ expected_refs={
+ "0": ["mo"],
+ "1": ["mo", "catalog_one.my_table.suffix"],
+ },
+ expected_defs={"0": ["my_table"], "1": []},
+ ),
]
+CASES = PYTHON_CASES + SQL_CASES
+
@pytest.mark.parametrize("case", CASES)
def test_cases(case: GraphTestCase) -> None:
@@ -271,7 +502,10 @@ def test_cases(case: GraphTestCase) -> None:
if case.xfail:
if isinstance(case.xfail, str):
print(case.xfail)
- with pytest.raises(AssertionError):
+ try:
make_assertions()
+ except AssertionError as e:
+ pytest.xfail(str(e))
+ raise KnownFailure(str(e)) from e
else:
make_assertions()