The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Support template tags in Workspace transforms (#69561)
Chris Truter3d5b673c9a1b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 73e9650bd92..bda3cb4caf5 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2521,6 +2521,7 @@liblib-bemodels+ premium-featuresqueries;; NOTE: we will remove this once we get "pure transforms" module.query-processordiff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex 13f27c7cb66..6440ef0923e 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -7,12 +7,10 @@[clojure.string :as str][metabase-enterprise.transforms-python.python-runner :as python-runner][metabase.query-processor :as qp]- [metabase.sql-tools.core :as sql-tools][metabase.transforms-base.core :as transforms-base][metabase.transforms-base.util :as transforms-base.u][metabase.util.log :as log]- [metabase.util.malli :as mu]- [toucan2.core :as t2]))+ [metabase.util.malli :as mu]))(set! *warn-on-reflection* true)@@ -33,30 +31,23 @@entry))](update source :source-tables (fn [entries] (mapv remap entries)))))-(defn- remap-sql-source [table-mapping source]- (let [remapping (reduce- (fn [remapping [[_ source-schema source-table] {target-schema :schema, target-table :table}]]- (assoc-in remapping [:tables {:schema source-schema- :table source-table}]- {:schema target-schema- :table target-table}))- {:schemas {}- :tables {}}- ;; Strip out the numeric keys (table ids)- (filter (comp vector? key) table-mapping))- database-id (get-in source [:query :database])- driver (some->> database-id (t2/select-one-fn :engine :model/Database))]- (update-in source [:query :stages 0 :native]- #(sql-tools/replace-names driver % remapping {:allow-unused? true}))))--(defn- remap-mbql-source [_table-mapping _field-map source]+(defn- attach-table-remapping [table-mapping source]+ (let [tables (into {}+ (keep (fn [[[_ src-schema src-table] {tgt-schema :schema tgt-table :table}]]+ [{:schema src-schema :table src-table}+ {:schema tgt-schema :table tgt-table}]))+ ;; Keep only the named reference tuples that we will destructure, the table-id keys are irrelevant.+ (filter (comp vector? key) table-mapping))]+ (cond-> source+ (seq tables) (assoc-in [:query :middleware :workspace-remapping] {:tables tables}))))++(defn- attach-table-field-remapping [_table-mapping _field-map source](throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))(defn- remap-source [table-map field-map source-type source](case source-type- :mbql (remap-mbql-source table-map field-map source)- ;; TODO (Chris 2025-12-12) -- make sure it's actually a SQL dialect though..- :native (remap-sql-source table-map source)+ :mbql (attach-table-field-remapping table-map field-map source)+ :native (attach-table-remapping table-map source):python (remap-python-source table-map source)));; You might prefer a multi-method? I certainly would.diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.clj b/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.cljnew file mode 100644index 00000000000..a0fba955819--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.clj@@ -0,0 +1,33 @@+(ns metabase-enterprise.workspaces.query-processor.middleware+ "QP preprocessing middleware that rewrites table references in native SQL queries for workspace transforms.++ When a query carries a `:workspace-remapping` key in its `:middleware` map (attached by workspace execute+ code), this middleware rewrites table names in the native SQL using [[sql-tools/replace-names]]."+ (:require+ [metabase.driver :as driver]+ [metabase.lib.schema :as lib.schema]+ [metabase.premium-features.core :refer [defenterprise]]+ [metabase.sql-tools.core :as sql-tools]+ [metabase.util :as u]))++(set! *warn-on-reflection* true)++(defenterprise apply-workspace-remapping+ "Pre-processing middleware. Rewrites table references in native SQL queries for workspace transforms."+ :feature :workspaces+ [{{remapping :workspace-remapping} :middleware :as query}]+ (cond+ ;; Not in a workspace that requires remapping, or no tables to remap.+ (or (not remapping) (empty? (:tables remapping)))+ query++ (not (lib.schema/native-only-query? query))+ (throw (ex-info "Workspace remapping is currently only supported for native queries"+ {:query query, :remapping remapping}))++ :else+ (u/update-in-if-exists query [:stages 0 :native]+ (fn [sql] (sql-tools/replace-names driver/*driver*+ sql+ remapping+ {:allow-unused? true})))))diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex 943f05ef1b4..88e1101c37e 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -1283,7 +1283,9 @@(let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))](mt/with-temp [:model/Transform x1 {:name "Transform":source {:type "query"- :query (mt/native-query (ws.tu/mbql->native (mt/mbql-query orders {:aggregation [[:count]]})))}+ :query (-> (mt/mbql-query orders {:aggregation [[:count]]})+ ws.tu/mbql->native+ mt/native-query)}:target {:type "table":database (mt/id):schema target-schema@@ -2078,9 +2080,7 @@(testing "GET /api/ee/workspace/checkout returns checkout status for a transform"(mt/with-temp [:model/Transform tx {:name "Native Transform":source {:type :query- :query {:database (mt/id)- :type :native- :native {:query "SELECT 1"}}}+ :query (mt/native-query {:query "SELECT 1"})}:target {:type "table":database (mt/id):schema "public"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex 9fc705712e0..c6fb7061924 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -5,10 +5,15 @@[metabase-enterprise.workspaces.common :as ws.common][metabase-enterprise.workspaces.execute :as ws.execute][metabase-enterprise.workspaces.impl :as ws.impl]+ [metabase-enterprise.workspaces.query-processor.middleware :as ws.qp.middleware][metabase-enterprise.workspaces.test-util :as ws.tu]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.compile :as qp.compile][metabase.test :as mt][metabase.transforms.test-util :as transforms.tu]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import (clojure.lang ExceptionInfo)))(set! *warn-on-reflection* true)@@ -60,6 +65,31 @@{:xf (t2/count :model/Transform):xfrun (t2/count :model/TransformRun)})))))))+(deftest dry-run-workspace-transform-with-template-tags-test+ (testing "Dry-running a workspace transform with template tags succeeds"+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}+ {:workspace {:definitions {:x1 []}}}]+ (let [ref-id (workspace-map :x1)+ workspace (t2/select-one :model/Workspace workspace-id)+ ;; Update the transform source to use template tags+ native-query (-> (lib/native-query (mt/metadata-provider)+ "SELECT {{num}} as id")+ (lib/with-template-tags+ {"num" {:id "num"+ :name "num"+ :display-name "Num"+ :type :number+ :default "1"}}))+ _ (t2/update! :model/WorkspaceTransform+ {:workspace_id workspace-id :ref_id ref-id}+ {:source {:type "query" :query native-query}})+ ws-transform (t2/select-one :model/WorkspaceTransform :workspace_id workspace-id :ref_id ref-id)+ graph (ws.impl/get-or-calculate-graph! workspace)]+ (is (=? {:status :succeeded+ :data {:rows [[1]]}}+ (mt/with-current-user (mt/user->id :crowberto)+ (ws.impl/dry-run-transform workspace graph ws-transform))))))))+(deftest dry-run-sql-workspace-transform-test(testing "Dry-running a workspace transform returns rows without persisting"(let [workspace (ws.tu/create-ready-ws! "Dry-Run Test Workspace")@@ -248,42 +278,56 @@:schema "public" :table "orders" :table_id 123}](:source-tables (remap-python-source table-mapping source))))))))-(deftest remap-sql-source-test- (let [remap-sql-source #'ws.execute/remap-sql-source- make-source (fn [sql] {:type "query", :query {:database 1, :type :native, :stages [{:native sql}]}})]- ;; Test cases as [label, table-mapping, input-sql, expected-sql]+(deftest workspace-sql-remapping-test+ (let [db-id (mt/id)](doseq [[label table-mapping input-sql expected-sql][["remaps qualified table reference to isolated table"- {[1 "public" "orders"] {:db-id 1, :schema "ws_isolated_123", :table "public__orders", :id 456}}+ {[db-id "public" "orders"] {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}"SELECT * FROM public.orders""SELECT * FROM ws_isolated_123.public__orders"]["remaps unqualified table reference using nil-schema mapping"- {[1 nil "orders"] {:db-id 1, :schema "ws_isolated_123", :table "public__orders", :id 456}}+ {[db-id nil "orders"] {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}"SELECT * FROM orders""SELECT * FROM ws_isolated_123.public__orders"]- ;; This is what build-remapping creates for unqualified input tables["qualifies unqualified input table reference (no isolation, just adds schema)"- {[1 nil "orders"] {:db-id 1, :schema "public", :table "orders", :id 123}}+ {[db-id nil "orders"] {:db-id db-id :schema "public" :table "orders" :id 123}}"SELECT * FROM orders""SELECT * FROM public.orders"]["leaves unmapped tables unchanged"- {[1 nil "other_table"] {:db-id 1, :schema "public", :table "other_table", :id 999}}+ {[db-id nil "other_table"] {:db-id db-id :schema "public" :table "other_table" :id 789}}"SELECT * FROM orders""SELECT * FROM orders"];; Note: macaw only renames tables in FROM/JOIN clauses, not table qualifiers in column references.- ;; The output SQL is semantically valid since `orders` resolves to `public.orders` in scope.+ ;; The output SQL is semantically valid since orders resolves to public.orders in scope.["handles multiple tables in same query"- {[1 nil "orders"] {:db-id 1, :schema "public", :table "orders", :id 123}- [1 nil "products"] {:db-id 1, :schema "public", :table "products", :id 456}}+ {[db-id nil "orders"] {:db-id db-id :schema "public" :table "orders" :id 123}+ [db-id nil "products"] {:db-id db-id :schema "public" :table "products" :id 124}}"SELECT * FROM orders JOIN products ON orders.product_id = products.id"- "SELECT * FROM public.orders JOIN public.products ON orders.product_id = products.id"]]]+ "SELECT * FROM public.orders JOIN public.products ON orders.product_id = products.id"]++ ["ignores numeric keys in table-mapping"+ {123 {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}+ "SELECT * FROM orders"+ "SELECT * FROM orders"]]](testing label- (is (= (make-source expected-sql)- (remap-sql-source table-mapping (make-source input-sql))))))))+ (let [query {:type "query", :query (lib/native-query (mt/metadata-provider) input-sql)}+ source (#'ws.execute/attach-table-remapping table-mapping query)+ compiled (qp.compile/compile (:query source))]+ (is (= expected-sql (:query compiled))))))))++(deftest workspace-mbql-remapping-throws-test+ (testing "workspace remapping throws on MBQL queries (not yet supported)"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (assoc-in [:middleware :workspace-remapping]+ {:tables {{:schema "public" :table "orders"}+ {:schema "ws_isolated" :table "public__orders"}}}))]+ (is (thrown-with-msg? ExceptionInfo+ #"Workspace remapping is currently only supported for native queries"+ (ws.qp.middleware/apply-workspace-remapping query))))))(deftest run-transform-marks-not-stale-on-success(testing "Successful transform run marks definition_changed=false"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.cljindex 609468b0d45..b4a20c1b746 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj@@ -53,9 +53,7 @@(str "ee/workspace/" ws-id "/transform"){:name "Test Transform":source {:type "query"- :query {:database (mt/id)- :type :native- :native {:query "SELECT 1 as col"}}}+ :query (mt/native-query {:query "SELECT 1 as col"})}:target {:type "table":schema "public":name "test_output_table"}})@@ -97,9 +95,7 @@(str "ee/workspace/" ws-id "/transform"){:name "Workspace Transform":source {:type "query"- :query {:database (mt/id)- :type :native- :native {:query "SELECT 1 as col"}}}+ :query (mt/native-query {:query "SELECT 1 as col"})}:target {:type "table":schema "public":name "global_output_table"}})]diff --git a/src/metabase/query_processor/middleware/enterprise.clj b/src/metabase/query_processor/middleware/enterprise.cljindex f65293dfb26..3131823c049 100644--- a/src/metabase/query_processor/middleware/enterprise.clj+++ b/src/metabase/query_processor/middleware/enterprise.clj@@ -57,6 +57,12 @@[query]query)+(defenterprise apply-workspace-remapping+ "Pre-processing middleware to rewrite 'global' table references with 'isolated' tables in the workspace schema."+ metabase-enterprise.workspaces.query-processor.middleware+ [query]+ query)+;;;; Execution middleware;;; (f qp) => qpdiff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex 349188083d3..fd277a13b49 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -103,6 +103,7 @@#'optimize-temporal-filters/optimize-temporal-filters#'limit/add-default-limit#'qp.middleware.enterprise/apply-download-limit+ #'qp.middleware.enterprise/apply-workspace-remapping#'check-features/check-features])(def ^:private ^Long slow-middleware-warning-threshold-ms