Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Compile transforms with nested union all layer to prevent inheriting IDENTITY property (#77539)

Ben Grabow8d4757803217
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index 507622bfdd0..1a1fae497db 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -912,17 +912,98 @@
(deftest ^:parallel compile-transform-test
(mt/test-driver :sqlserver
- (testing "compile-transform creates SELECT INTO"
- ;; Both formats are valid T-SQL: double-quoted identifiers (Macaw/JSQLParser)
- ;; and bracketed identifiers (SQLGlot). SQL Server accepts both.
- (is (contains? #{["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]
- ["SELECT * INTO [PRODUCTS_COPY] FROM products" nil]}
- (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}
- :output-table "PRODUCTS_COPY"}))))
+ (testing "compile-transform wraps each base table in a self-UNION subquery so the target doesn't inherit
+ a source column's IDENTITY property, and injects a SELECT INTO"
+ (testing "single table"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "ORDER BY is preserved on the outer query"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products ORDER BY date DESC" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM products ORDER BY date DESC"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "TOP + ORDER BY are both preserved on the outer query"
+ (is (= ["SELECT TOP 10 * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products ORDER BY date DESC" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT TOP 10 * FROM products ORDER BY date DESC"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a schema-qualified table is wrapped, schema qualification preserved on the inner reference"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM dbo.products UNION ALL SELECT * FROM dbo.products WHERE 1 = 0) AS products" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM dbo.products"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a two-table join wraps both tables, each keeping its own alias"
+ (is (= ["SELECT p.id, o.total INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS p JOIN (SELECT * FROM orders UNION ALL SELECT * FROM orders WHERE 1 = 0) AS o ON p.id = o.product_id" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT p.id, o.total FROM products p JOIN orders o ON p.id = o.product_id"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a self-join wraps the same table twice, each occurrence keeping its own distinct alias"
+ (is (= ["SELECT a.id, b.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS a JOIN (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS b ON a.parent_id = b.id" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT a.id, b.id FROM products a JOIN products b ON a.parent_id = b.id"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a CTE is left untouched; the base table it references is wrapped"
+ (is (= ["WITH cte AS (SELECT * FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products) SELECT * INTO [PRODUCTS_COPY] FROM cte" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "WITH cte AS (SELECT * FROM products) SELECT * FROM cte"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "an OR-ed WHERE passes through unchanged on the outer query"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE a = 1 OR b = 2" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM products WHERE a = 1 OR b = 2"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a derived table (subquery in FROM) is left untouched; the base table inside it is wrapped"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products) AS x" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM (SELECT * FROM products) x"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a table-valued function is left untouched"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM dbo.SOME_FUNC(1, 2) AS f" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM dbo.some_func(1, 2) AS f"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a subquery in WHERE also has its base table wrapped"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE id IN (SELECT product_id FROM (SELECT * FROM orders UNION ALL SELECT * FROM orders WHERE 1 = 0) AS orders)" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM products WHERE id IN (SELECT product_id FROM orders)"}
+ :output-table "PRODUCTS_COPY"})))))
+ (testing "positional params appear exactly once (only bare table references are wrapped, not filters)"
+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE id = ?" [42]]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT * FROM products WHERE id = ?" :params [42]}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a schema-qualified column reference is rewritten to the derived-table alias"
+ (is (= ["SELECT products.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM dbo.products UNION ALL SELECT * FROM dbo.products WHERE 1 = 0) AS products" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT dbo.products.id FROM dbo.products"}
+ :output-table "PRODUCTS_COPY"}))))
+ (testing "a catalog-qualified column reference is rewritten to the derived-table alias"
+ (is (= ["SELECT products.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM mydb.dbo.products UNION ALL SELECT * FROM mydb.dbo.products WHERE 1 = 0) AS products" nil]
+ (driver/compile-transform
+ :sqlserver
+ {:query {:query "SELECT mydb.dbo.products.id FROM mydb.dbo.products"}
+ :output-table "PRODUCTS_COPY"}))))
(testing "compile-insert generates INSERT INTO"
(is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil]
- (driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}
- :output-table "PRODUCTS_COPY"}))))))
+ (driver/compile-insert
+ :sqlserver
+ {:query {:query "SELECT * FROM products"}
+ :output-table "PRODUCTS_COPY"}))))))
(deftest table-privileges-test
(mt/test-driver :sqlserver
diff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.py
index d0f40f3eea0..9cf4f3cf43f 100644
--- a/resources/python-sources/sql_tools.py
+++ b/resources/python-sources/sql_tools.py
@@ -378,30 +378,80 @@ def simple_query(sql: str, dialect: str = None) -> str:
except Exception as e:
return json.dumps({"is_simple": False, "reason": f"Unexpected error: {str(e)}"})
+def _wrap_base_table(table, alias):
+ """Build a self-UNION subquery, aliased `alias`, that scans `table` once but strips any IDENTITY
+ property inherited from it (see `add_into_clause`)."""
+ bare_table = table.copy()
+ bare_table.set("alias", None)
+ self_union = exp.union(
+ exp.select("*").from_(bare_table.copy()),
+ exp.select("*").from_(bare_table.copy()).where("1 = 0"),
+ distinct=False,
+ )
+ return self_union.subquery(alias=alias)
+
def add_into_clause(sql: str, table_name: str, dialect: str = None) -> str:
"""
- Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.
+ Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax, and rewrite every
+ base-table reference into a self-UNION subquery so the new table doesn't inherit a source
+ column's IDENTITY property.
Transforms: SELECT * FROM products
- Into: SELECT * INTO "new_table" FROM products
+ Into: SELECT * INTO "new_table" FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products
+
+ Used by SQL Server transforms which require SELECT INTO syntax instead of CREATE TABLE AS SELECT.
+ A plain `SELECT ... INTO` copies a directly-projected source IDENTITY column onto the new table,
+ which later breaks an incremental append (`INSERT INTO <target> SELECT ...`) with SQL Server error
+ 8101. Wrapping each base table in a self-UNION breaks that IDENTITY lineage -- the UNION's second
+ branch is always empty and gets elided by SQL Server's contradiction detection, so each table is
+ still scanned once -- without touching filters, ORDER BY, TOP, or query params. CTEs, derived
+ tables, and table-valued functions are left untouched.
- Used by SQL Server transforms which require SELECT INTO syntax
- instead of CREATE TABLE AS SELECT.
+ A table with no explicit alias is wrapped under an alias equal to its bare name, and any column
+ in the same scope that referenced it via a schema/catalog-qualified name (e.g. `dbo.products.id`,
+ legal only against a real table) has that now-invalid qualifier stripped, since a derived table can
+ only be referenced by its single-part alias.
:param sql: The SELECT SQL query string
:param table_name: The target table name (already formatted/quoted)
:param dialect: SQL dialect (e.g., "tsql" for SQL Server)
- :return: Modified SQL string with INTO clause
+ :return: Modified SQL string with INTO clause and base tables wrapped
Examples:
add_into_clause("SELECT * FROM products", '"PRODUCTS_COPY"', "tsql")
- => 'SELECT * INTO "PRODUCTS_COPY" FROM products'
+ => 'SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products'
"""
ast = sqlglot.parse_one(sql, read=dialect)
if not isinstance(ast, exp.Select):
raise ValueError("SQL must be a SELECT statement")
+ cte_names = {cte.alias for cte in ast.find_all(exp.CTE) if cte.alias}
+ seen_ids = set()
+
+ for scope in optimizer.build_scope(ast).traverse():
+ scope_replacements = []
+ for alias, source in scope.sources.items():
+ if not isinstance(source, exp.Table):
+ continue
+ table_name_only = source.name
+ if not table_name_only or table_name_only in cte_names or alias in cte_names:
+ continue
+ if id(source) in seen_ids:
+ continue
+ seen_ids.add(id(source))
+ scope_replacements.append((source, alias))
+
+ for table, alias in scope_replacements:
+ # Columns in this scope may reference the table via a schema/catalog-qualified name
+ # (e.g. `dbo.products.id`), which only resolves against a real table. Once the table
+ # becomes a derived table it's only reachable by its single-part alias, so strip the
+ # now-invalid qualifiers.
+ for column in scope.source_columns(alias):
+ column.set("db", None)
+ column.set("catalog", None)
+ table.replace(_wrap_base_table(table, alias))
+
# Create the Into expression with the table identifier
# The table_name is pre-formatted, so parse it to preserve quotes
into = exp.Into(this=exp.to_table(table_name))
diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.clj
index 642e6a7c0b6..d1e681b0d74 100644
--- a/src/metabase/sql_tools/core.clj
+++ b/src/metabase/sql_tools/core.clj
@@ -176,16 +176,24 @@
(interface/simple-query?-impl parser sql-string))))
(mu/defn add-into-clause :- :string
- "Add an INTO clause to a SELECT statement.
+ "Rewrite a SELECT into a SQL Server `SELECT ... INTO` statement, wrapping each base table in a
+ self-UNION subquery so the created table doesn't inherit a source column's IDENTITY property.
Transforms: 'SELECT * FROM products'
- Into: 'SELECT * INTO \"TABLE\" FROM products'
+ Into: 'SELECT * INTO \"TABLE\" FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products'
- Used by SQL Server compile-transform which requires SELECT INTO syntax
- instead of CREATE TABLE AS SELECT."
+ Used by SQL Server compile-transform, which requires SELECT INTO instead of CREATE TABLE AS SELECT;
+ the self-UNION wrap prevents a later incremental append from hitting SQL Server error 8101."
[driver :- :keyword
sql :- :string
table-name :- :string]
+ (when-not (= driver :sqlserver)
+ (throw (ex-info (str "add-into-clause is only supported for :sqlserver. It wraps each base table in a "
+ "self-UNION subquery to strip inherited IDENTITY columns. Before enabling another "
+ "driver, verify this does not cause a performance regression: some engines (e.g. "
+ "MySQL) may fully materialize the inner subquery instead of pushing the outer WHERE "
+ "down into it, so a filtered query over a large table would scan the whole table.")
+ {:driver driver})))
(let [parser (sql-tools.settings/current-parser-backend)]
(metrics/with-operation-timing [parser "add-into-clause"]
(interface/add-into-clause-impl parser driver sql table-name))))