The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Compile transforms with nested union all layer to prevent inheriting IDENTITY property (#77539)
Ben Grabow8d4757803217
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 507622bfdd0..1a1fae497db 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -912,17 +912,98 @@(deftest ^:parallel compile-transform-test(mt/test-driver :sqlserver- (testing "compile-transform creates SELECT INTO"- ;; Both formats are valid T-SQL: double-quoted identifiers (Macaw/JSQLParser)- ;; and bracketed identifiers (SQLGlot). SQL Server accepts both.- (is (contains? #{["SELECT * INTO \"PRODUCTS_COPY\" FROM products" nil]- ["SELECT * INTO [PRODUCTS_COPY] FROM products" nil]}- (driver/compile-transform :sqlserver {:query {:query "SELECT * FROM products"}- :output-table "PRODUCTS_COPY"}))))+ (testing "compile-transform wraps each base table in a self-UNION subquery so the target doesn't inherit+ a source column's IDENTITY property, and injects a SELECT INTO"+ (testing "single table"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "ORDER BY is preserved on the outer query"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products ORDER BY date DESC" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM products ORDER BY date DESC"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "TOP + ORDER BY are both preserved on the outer query"+ (is (= ["SELECT TOP 10 * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products ORDER BY date DESC" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT TOP 10 * FROM products ORDER BY date DESC"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a schema-qualified table is wrapped, schema qualification preserved on the inner reference"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM dbo.products UNION ALL SELECT * FROM dbo.products WHERE 1 = 0) AS products" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM dbo.products"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a two-table join wraps both tables, each keeping its own alias"+ (is (= ["SELECT p.id, o.total INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS p JOIN (SELECT * FROM orders UNION ALL SELECT * FROM orders WHERE 1 = 0) AS o ON p.id = o.product_id" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT p.id, o.total FROM products p JOIN orders o ON p.id = o.product_id"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a self-join wraps the same table twice, each occurrence keeping its own distinct alias"+ (is (= ["SELECT a.id, b.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS a JOIN (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS b ON a.parent_id = b.id" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT a.id, b.id FROM products a JOIN products b ON a.parent_id = b.id"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a CTE is left untouched; the base table it references is wrapped"+ (is (= ["WITH cte AS (SELECT * FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products) SELECT * INTO [PRODUCTS_COPY] FROM cte" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "WITH cte AS (SELECT * FROM products) SELECT * FROM cte"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "an OR-ed WHERE passes through unchanged on the outer query"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE a = 1 OR b = 2" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM products WHERE a = 1 OR b = 2"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a derived table (subquery in FROM) is left untouched; the base table inside it is wrapped"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products) AS x" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM (SELECT * FROM products) x"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a table-valued function is left untouched"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM dbo.SOME_FUNC(1, 2) AS f" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM dbo.some_func(1, 2) AS f"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a subquery in WHERE also has its base table wrapped"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE id IN (SELECT product_id FROM (SELECT * FROM orders UNION ALL SELECT * FROM orders WHERE 1 = 0) AS orders)" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM products WHERE id IN (SELECT product_id FROM orders)"}+ :output-table "PRODUCTS_COPY"})))))+ (testing "positional params appear exactly once (only bare table references are wrapped, not filters)"+ (is (= ["SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products WHERE id = ?" [42]]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT * FROM products WHERE id = ?" :params [42]}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a schema-qualified column reference is rewritten to the derived-table alias"+ (is (= ["SELECT products.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM dbo.products UNION ALL SELECT * FROM dbo.products WHERE 1 = 0) AS products" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT dbo.products.id FROM dbo.products"}+ :output-table "PRODUCTS_COPY"}))))+ (testing "a catalog-qualified column reference is rewritten to the derived-table alias"+ (is (= ["SELECT products.id INTO [PRODUCTS_COPY] FROM (SELECT * FROM mydb.dbo.products UNION ALL SELECT * FROM mydb.dbo.products WHERE 1 = 0) AS products" nil]+ (driver/compile-transform+ :sqlserver+ {:query {:query "SELECT mydb.dbo.products.id FROM mydb.dbo.products"}+ :output-table "PRODUCTS_COPY"}))))(testing "compile-insert generates INSERT INTO"(is (= ["INSERT INTO \"PRODUCTS_COPY\" SELECT * FROM products" nil]- (driver/compile-insert :sqlserver {:query {:query "SELECT * FROM products"}- :output-table "PRODUCTS_COPY"}))))))+ (driver/compile-insert+ :sqlserver+ {:query {:query "SELECT * FROM products"}+ :output-table "PRODUCTS_COPY"}))))))(deftest table-privileges-test(mt/test-driver :sqlserverdiff --git a/resources/python-sources/sql_tools.py b/resources/python-sources/sql_tools.pyindex d0f40f3eea0..9cf4f3cf43f 100644--- a/resources/python-sources/sql_tools.py+++ b/resources/python-sources/sql_tools.py@@ -378,30 +378,80 @@ def simple_query(sql: str, dialect: str = None) -> str:except Exception as e:return json.dumps({"is_simple": False, "reason": f"Unexpected error: {str(e)}"})+def _wrap_base_table(table, alias):+ """Build a self-UNION subquery, aliased `alias`, that scans `table` once but strips any IDENTITY+ property inherited from it (see `add_into_clause`)."""+ bare_table = table.copy()+ bare_table.set("alias", None)+ self_union = exp.union(+ exp.select("*").from_(bare_table.copy()),+ exp.select("*").from_(bare_table.copy()).where("1 = 0"),+ distinct=False,+ )+ return self_union.subquery(alias=alias)+def add_into_clause(sql: str, table_name: str, dialect: str = None) -> str:"""- Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax.+ Add an INTO clause to a SELECT statement for SQL Server SELECT INTO syntax, and rewrite every+ base-table reference into a self-UNION subquery so the new table doesn't inherit a source+ column's IDENTITY property.Transforms: SELECT * FROM products- Into: SELECT * INTO "new_table" FROM products+ Into: SELECT * INTO "new_table" FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products++ Used by SQL Server transforms which require SELECT INTO syntax instead of CREATE TABLE AS SELECT.+ A plain `SELECT ... INTO` copies a directly-projected source IDENTITY column onto the new table,+ which later breaks an incremental append (`INSERT INTO <target> SELECT ...`) with SQL Server error+ 8101. Wrapping each base table in a self-UNION breaks that IDENTITY lineage -- the UNION's second+ branch is always empty and gets elided by SQL Server's contradiction detection, so each table is+ still scanned once -- without touching filters, ORDER BY, TOP, or query params. CTEs, derived+ tables, and table-valued functions are left untouched.- Used by SQL Server transforms which require SELECT INTO syntax- instead of CREATE TABLE AS SELECT.+ A table with no explicit alias is wrapped under an alias equal to its bare name, and any column+ in the same scope that referenced it via a schema/catalog-qualified name (e.g. `dbo.products.id`,+ legal only against a real table) has that now-invalid qualifier stripped, since a derived table can+ only be referenced by its single-part alias.:param sql: The SELECT SQL query string:param table_name: The target table name (already formatted/quoted):param dialect: SQL dialect (e.g., "tsql" for SQL Server)- :return: Modified SQL string with INTO clause+ :return: Modified SQL string with INTO clause and base tables wrappedExamples:add_into_clause("SELECT * FROM products", '"PRODUCTS_COPY"', "tsql")- => 'SELECT * INTO "PRODUCTS_COPY" FROM products'+ => 'SELECT * INTO [PRODUCTS_COPY] FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products'"""ast = sqlglot.parse_one(sql, read=dialect)if not isinstance(ast, exp.Select):raise ValueError("SQL must be a SELECT statement")+ cte_names = {cte.alias for cte in ast.find_all(exp.CTE) if cte.alias}+ seen_ids = set()++ for scope in optimizer.build_scope(ast).traverse():+ scope_replacements = []+ for alias, source in scope.sources.items():+ if not isinstance(source, exp.Table):+ continue+ table_name_only = source.name+ if not table_name_only or table_name_only in cte_names or alias in cte_names:+ continue+ if id(source) in seen_ids:+ continue+ seen_ids.add(id(source))+ scope_replacements.append((source, alias))++ for table, alias in scope_replacements:+ # Columns in this scope may reference the table via a schema/catalog-qualified name+ # (e.g. `dbo.products.id`), which only resolves against a real table. Once the table+ # becomes a derived table it's only reachable by its single-part alias, so strip the+ # now-invalid qualifiers.+ for column in scope.source_columns(alias):+ column.set("db", None)+ column.set("catalog", None)+ table.replace(_wrap_base_table(table, alias))+# Create the Into expression with the table identifier# The table_name is pre-formatted, so parse it to preserve quotesinto = exp.Into(this=exp.to_table(table_name))diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.cljindex 642e6a7c0b6..d1e681b0d74 100644--- a/src/metabase/sql_tools/core.clj+++ b/src/metabase/sql_tools/core.clj@@ -176,16 +176,24 @@(interface/simple-query?-impl parser sql-string))))(mu/defn add-into-clause :- :string- "Add an INTO clause to a SELECT statement.+ "Rewrite a SELECT into a SQL Server `SELECT ... INTO` statement, wrapping each base table in a+ self-UNION subquery so the created table doesn't inherit a source column's IDENTITY property.Transforms: 'SELECT * FROM products'- Into: 'SELECT * INTO \"TABLE\" FROM products'+ Into: 'SELECT * INTO \"TABLE\" FROM (SELECT * FROM products UNION ALL SELECT * FROM products WHERE 1 = 0) AS products'- Used by SQL Server compile-transform which requires SELECT INTO syntax- instead of CREATE TABLE AS SELECT."+ Used by SQL Server compile-transform, which requires SELECT INTO instead of CREATE TABLE AS SELECT;+ the self-UNION wrap prevents a later incremental append from hitting SQL Server error 8101."[driver :- :keywordsql :- :stringtable-name :- :string]+ (when-not (= driver :sqlserver)+ (throw (ex-info (str "add-into-clause is only supported for :sqlserver. It wraps each base table in a "+ "self-UNION subquery to strip inherited IDENTITY columns. Before enabling another "+ "driver, verify this does not cause a performance regression: some engines (e.g. "+ "MySQL) may fully materialize the inner subquery instead of pushing the outer WHERE "+ "down into it, so a filtered query over a large table would scan the whole table.")+ {:driver driver})))(let [parser (sql-tools.settings/current-parser-backend)](metrics/with-operation-timing [parser "add-into-clause"](interface/add-into-clause-impl parser driver sql table-name))))