Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Support template tags in Workspace transforms (#69561)

Chris Truter3d5b673c9a1b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 73e9650bd92..bda3cb4caf5 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2521,6 +2521,7 @@
lib
lib-be
models
+ premium-features
queries
;; NOTE: we will remove this once we get "pure transforms" module.
query-processor
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index 13f27c7cb66..6440ef0923e 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -7,12 +7,10 @@
[clojure.string :as str]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase.query-processor :as qp]
- [metabase.sql-tools.core :as sql-tools]
[metabase.transforms-base.core :as transforms-base]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util.log :as log]
- [metabase.util.malli :as mu]
- [toucan2.core :as t2]))
+ [metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
@@ -33,30 +31,23 @@
entry))]
(update source :source-tables (fn [entries] (mapv remap entries)))))
-(defn- remap-sql-source [table-mapping source]
- (let [remapping (reduce
- (fn [remapping [[_ source-schema source-table] {target-schema :schema, target-table :table}]]
- (assoc-in remapping [:tables {:schema source-schema
- :table source-table}]
- {:schema target-schema
- :table target-table}))
- {:schemas {}
- :tables {}}
- ;; Strip out the numeric keys (table ids)
- (filter (comp vector? key) table-mapping))
- database-id (get-in source [:query :database])
- driver (some->> database-id (t2/select-one-fn :engine :model/Database))]
- (update-in source [:query :stages 0 :native]
- #(sql-tools/replace-names driver % remapping {:allow-unused? true}))))
-
-(defn- remap-mbql-source [_table-mapping _field-map source]
+(defn- attach-table-remapping [table-mapping source]
+ (let [tables (into {}
+ (keep (fn [[[_ src-schema src-table] {tgt-schema :schema tgt-table :table}]]
+ [{:schema src-schema :table src-table}
+ {:schema tgt-schema :table tgt-table}]))
+ ;; Keep only the named reference tuples that we will destructure, the table-id keys are irrelevant.
+ (filter (comp vector? key) table-mapping))]
+ (cond-> source
+ (seq tables) (assoc-in [:query :middleware :workspace-remapping] {:tables tables}))))
+
+(defn- attach-table-field-remapping [_table-mapping _field-map source]
(throw (ex-info "Remapping MBQL queries is not supported yet" {:source source})))
(defn- remap-source [table-map field-map source-type source]
(case source-type
- :mbql (remap-mbql-source table-map field-map source)
- ;; TODO (Chris 2025-12-12) -- make sure it's actually a SQL dialect though..
- :native (remap-sql-source table-map source)
+ :mbql (attach-table-field-remapping table-map field-map source)
+ :native (attach-table-remapping table-map source)
:python (remap-python-source table-map source)))
;; You might prefer a multi-method? I certainly would.
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.clj b/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.clj
new file mode 100644
index 00000000000..a0fba955819
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/query_processor/middleware.clj
@@ -0,0 +1,33 @@
+(ns metabase-enterprise.workspaces.query-processor.middleware
+ "QP preprocessing middleware that rewrites table references in native SQL queries for workspace transforms.
+
+ When a query carries a `:workspace-remapping` key in its `:middleware` map (attached by workspace execute
+ code), this middleware rewrites table names in the native SQL using [[sql-tools/replace-names]]."
+ (:require
+ [metabase.driver :as driver]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.premium-features.core :refer [defenterprise]]
+ [metabase.sql-tools.core :as sql-tools]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defenterprise apply-workspace-remapping
+ "Pre-processing middleware. Rewrites table references in native SQL queries for workspace transforms."
+ :feature :workspaces
+ [{{remapping :workspace-remapping} :middleware :as query}]
+ (cond
+ ;; Not in a workspace that requires remapping, or no tables to remap.
+ (or (not remapping) (empty? (:tables remapping)))
+ query
+
+ (not (lib.schema/native-only-query? query))
+ (throw (ex-info "Workspace remapping is currently only supported for native queries"
+ {:query query, :remapping remapping}))
+
+ :else
+ (u/update-in-if-exists query [:stages 0 :native]
+ (fn [sql] (sql-tools/replace-names driver/*driver*
+ sql
+ remapping
+ {:allow-unused? true})))))
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index 943f05ef1b4..88e1101c37e 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -1283,7 +1283,9 @@
(let [target-schema (t2/select-one-fn :schema :model/Table (mt/id :orders))]
(mt/with-temp [:model/Transform x1 {:name "Transform"
:source {:type "query"
- :query (mt/native-query (ws.tu/mbql->native (mt/mbql-query orders {:aggregation [[:count]]})))}
+ :query (-> (mt/mbql-query orders {:aggregation [[:count]]})
+ ws.tu/mbql->native
+ mt/native-query)}
:target {:type "table"
:database (mt/id)
:schema target-schema
@@ -2078,9 +2080,7 @@
(testing "GET /api/ee/workspace/checkout returns checkout status for a transform"
(mt/with-temp [:model/Transform tx {:name "Native Transform"
:source {:type :query
- :query {:database (mt/id)
- :type :native
- :native {:query "SELECT 1"}}}
+ :query (mt/native-query {:query "SELECT 1"})}
:target {:type "table"
:database (mt/id)
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index 9fc705712e0..c6fb7061924 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -5,10 +5,15 @@
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.execute :as ws.execute]
[metabase-enterprise.workspaces.impl :as ws.impl]
+ [metabase-enterprise.workspaces.query-processor.middleware :as ws.qp.middleware]
[metabase-enterprise.workspaces.test-util :as ws.tu]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.query-processor.compile :as qp.compile]
[metabase.test :as mt]
[metabase.transforms.test-util :as transforms.tu]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import (clojure.lang ExceptionInfo)))
(set! *warn-on-reflection* true)
@@ -60,6 +65,31 @@
{:xf (t2/count :model/Transform)
:xfrun (t2/count :model/TransformRun)})))))))
+(deftest dry-run-workspace-transform-with-template-tags-test
+ (testing "Dry-running a workspace transform with template tags succeeds"
+ (ws.tu/with-resources! [{:keys [workspace-id workspace-map]}
+ {:workspace {:definitions {:x1 []}}}]
+ (let [ref-id (workspace-map :x1)
+ workspace (t2/select-one :model/Workspace workspace-id)
+ ;; Update the transform source to use template tags
+ native-query (-> (lib/native-query (mt/metadata-provider)
+ "SELECT {{num}} as id")
+ (lib/with-template-tags
+ {"num" {:id "num"
+ :name "num"
+ :display-name "Num"
+ :type :number
+ :default "1"}}))
+ _ (t2/update! :model/WorkspaceTransform
+ {:workspace_id workspace-id :ref_id ref-id}
+ {:source {:type "query" :query native-query}})
+ ws-transform (t2/select-one :model/WorkspaceTransform :workspace_id workspace-id :ref_id ref-id)
+ graph (ws.impl/get-or-calculate-graph! workspace)]
+ (is (=? {:status :succeeded
+ :data {:rows [[1]]}}
+ (mt/with-current-user (mt/user->id :crowberto)
+ (ws.impl/dry-run-transform workspace graph ws-transform))))))))
+
(deftest dry-run-sql-workspace-transform-test
(testing "Dry-running a workspace transform returns rows without persisting"
(let [workspace (ws.tu/create-ready-ws! "Dry-Run Test Workspace")
@@ -248,42 +278,56 @@
:schema "public" :table "orders" :table_id 123}]
(:source-tables (remap-python-source table-mapping source))))))))
-(deftest remap-sql-source-test
- (let [remap-sql-source #'ws.execute/remap-sql-source
- make-source (fn [sql] {:type "query", :query {:database 1, :type :native, :stages [{:native sql}]}})]
- ;; Test cases as [label, table-mapping, input-sql, expected-sql]
+(deftest workspace-sql-remapping-test
+ (let [db-id (mt/id)]
(doseq [[label table-mapping input-sql expected-sql]
[["remaps qualified table reference to isolated table"
- {[1 "public" "orders"] {:db-id 1, :schema "ws_isolated_123", :table "public__orders", :id 456}}
+ {[db-id "public" "orders"] {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}
"SELECT * FROM public.orders"
"SELECT * FROM ws_isolated_123.public__orders"]
["remaps unqualified table reference using nil-schema mapping"
- {[1 nil "orders"] {:db-id 1, :schema "ws_isolated_123", :table "public__orders", :id 456}}
+ {[db-id nil "orders"] {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}
"SELECT * FROM orders"
"SELECT * FROM ws_isolated_123.public__orders"]
- ;; This is what build-remapping creates for unqualified input tables
["qualifies unqualified input table reference (no isolation, just adds schema)"
- {[1 nil "orders"] {:db-id 1, :schema "public", :table "orders", :id 123}}
+ {[db-id nil "orders"] {:db-id db-id :schema "public" :table "orders" :id 123}}
"SELECT * FROM orders"
"SELECT * FROM public.orders"]
["leaves unmapped tables unchanged"
- {[1 nil "other_table"] {:db-id 1, :schema "public", :table "other_table", :id 999}}
+ {[db-id nil "other_table"] {:db-id db-id :schema "public" :table "other_table" :id 789}}
"SELECT * FROM orders"
"SELECT * FROM orders"]
;; Note: macaw only renames tables in FROM/JOIN clauses, not table qualifiers in column references.
- ;; The output SQL is semantically valid since `orders` resolves to `public.orders` in scope.
+ ;; The output SQL is semantically valid since orders resolves to public.orders in scope.
["handles multiple tables in same query"
- {[1 nil "orders"] {:db-id 1, :schema "public", :table "orders", :id 123}
- [1 nil "products"] {:db-id 1, :schema "public", :table "products", :id 456}}
+ {[db-id nil "orders"] {:db-id db-id :schema "public" :table "orders" :id 123}
+ [db-id nil "products"] {:db-id db-id :schema "public" :table "products" :id 124}}
"SELECT * FROM orders JOIN products ON orders.product_id = products.id"
- "SELECT * FROM public.orders JOIN public.products ON orders.product_id = products.id"]]]
+ "SELECT * FROM public.orders JOIN public.products ON orders.product_id = products.id"]
+
+ ["ignores numeric keys in table-mapping"
+ {123 {:db-id db-id :schema "ws_isolated_123" :table "public__orders" :id 456}}
+ "SELECT * FROM orders"
+ "SELECT * FROM orders"]]]
(testing label
- (is (= (make-source expected-sql)
- (remap-sql-source table-mapping (make-source input-sql))))))))
+ (let [query {:type "query", :query (lib/native-query (mt/metadata-provider) input-sql)}
+ source (#'ws.execute/attach-table-remapping table-mapping query)
+ compiled (qp.compile/compile (:query source))]
+ (is (= expected-sql (:query compiled))))))))
+
+(deftest workspace-mbql-remapping-throws-test
+ (testing "workspace remapping throws on MBQL queries (not yet supported)"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (assoc-in [:middleware :workspace-remapping]
+ {:tables {{:schema "public" :table "orders"}
+ {:schema "ws_isolated" :table "public__orders"}}}))]
+ (is (thrown-with-msg? ExceptionInfo
+ #"Workspace remapping is currently only supported for native queries"
+ (ws.qp.middleware/apply-workspace-remapping query))))))
(deftest run-transform-marks-not-stale-on-success
(testing "Successful transform run marks definition_changed=false"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
index 609468b0d45..b4a20c1b746 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
@@ -53,9 +53,7 @@
(str "ee/workspace/" ws-id "/transform")
{:name "Test Transform"
:source {:type "query"
- :query {:database (mt/id)
- :type :native
- :native {:query "SELECT 1 as col"}}}
+ :query (mt/native-query {:query "SELECT 1 as col"})}
:target {:type "table"
:schema "public"
:name "test_output_table"}})
@@ -97,9 +95,7 @@
(str "ee/workspace/" ws-id "/transform")
{:name "Workspace Transform"
:source {:type "query"
- :query {:database (mt/id)
- :type :native
- :native {:query "SELECT 1 as col"}}}
+ :query (mt/native-query {:query "SELECT 1 as col"})}
:target {:type "table"
:schema "public"
:name "global_output_table"}})]
diff --git a/src/metabase/query_processor/middleware/enterprise.clj b/src/metabase/query_processor/middleware/enterprise.clj
index f65293dfb26..3131823c049 100644
--- a/src/metabase/query_processor/middleware/enterprise.clj
+++ b/src/metabase/query_processor/middleware/enterprise.clj
@@ -57,6 +57,12 @@
[query]
query)
+(defenterprise apply-workspace-remapping
+ "Pre-processing middleware to rewrite 'global' table references with 'isolated' tables in the workspace schema."
+ metabase-enterprise.workspaces.query-processor.middleware
+ [query]
+ query)
+
;;;; Execution middleware
;;; (f qp) => qp
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index 349188083d3..fd277a13b49 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -103,6 +103,7 @@
#'optimize-temporal-filters/optimize-temporal-filters
#'limit/add-default-limit
#'qp.middleware.enterprise/apply-download-limit
+ #'qp.middleware.enterprise/apply-workspace-remapping
#'check-features/check-features])
(def ^:private ^Long slow-middleware-warning-threshold-ms