Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Run transforms with a temp table + rename to avoid blocking queries (#68756)

Riley Thompson3142c394354b
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
index fa13ee42ec9..5d6f728e0a2 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -33,12 +33,10 @@
[:map
[:overwrite? :boolean]])
-(defn- transform-opts [{:keys [transform-type]}]
- (case transform-type
- :table {:overwrite? true}
-
- ;; once we have more than just append, dispatch on :target-incremental-strategy
- :table-incremental {}))
+(defn- transform-opts [_transform-details]
+ ;; once we have more than just :table and :table-incremental as transform-types,
+ ;; then we can dispatch on :target-incremental-strategy
+ {})
(defn- throw-if-db-routing-enabled [transform driver database]
(when (transforms.util/db-routing-enabled? database)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index f9b43fbbb7a..fc208c3cdad 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -41,10 +41,6 @@
(set! *warn-on-reflection* true)
-(def ^:const transform-temp-table-prefix
- "Prefix used for temporary tables created during transform execution."
- "mb_transform_temp_table")
-
(defn qualified-table-name
"Return the name of the target table of a transform as a possibly qualified symbol."
[_driver {:keys [schema name]}]
@@ -499,21 +495,6 @@
(log/infof "Dropping table %s" table-name)
(driver/drop-table! driver database-id table-name))
-(defn temp-table-name
- "Generate a temporary table name with current timestamp in milliseconds.
- If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"
- [driver schema]
- (let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
- timestamp (str (System/currentTimeMillis))
- prefix (str transform-temp-table-prefix "_")
- available (- max-len (count prefix))
- ;; If we don't have enough space, take the later digits of the timestamp
- suffix (if (>= available (count timestamp))
- timestamp
- (subs timestamp (- (count timestamp) available)))
- table-name (str prefix suffix)]
- (keyword schema table-name)))
-
(defn rename-tables!
"Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
This is a simpler, composable operation that only handles renaming."
@@ -526,7 +507,7 @@
:feature :transforms
[table]
(when-let [table-name (:name table)]
- (str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))
+ (str/starts-with? (u/lower-case-en table-name) driver.u/transform-temp-table-prefix)))
(defn db-routing-enabled?
"Returns whether or not the given database is either a router or destination database"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index a51f726228e..c833f919d0c 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -166,11 +166,8 @@
"Transfer data using the rename-tables*! multimethod with atomicity guarantees.
Creates new table, then atomically renames target->old and new->target, then drops old."
[driver db-id table-name metadata data-source]
- (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))
- temp-table-name (u/poll {:thunk #(transforms.util/temp-table-name driver (namespace table-name))
- :done? #(not= source-table-name %)
- ;; Poll every 1ms to quickly generate a different timestamp-based table name
- :interval-ms 1})]
+ (let [source-table-name (driver.u/temp-table-name driver table-name)
+ temp-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using rename-tables strategy with atomicity guarantees")
(try
@@ -190,7 +187,7 @@
"Transfer data using create + drop + rename to minimize time without data.
Creates new table, drops old table, then renames new->target."
[driver db-id table-name metadata data-source]
- (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))]
+ (let [source-table-name (driver.u/temp-table-name driver table-name)]
(log/info "Using create-drop-rename strategy to minimize downtime")
(try
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
index 2464471c0f8..e76ce13487e 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj
@@ -1,5 +1,6 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.query-test-util :as query-test-util]
@@ -13,6 +14,7 @@
[metabase.query-processor :as qp]
[metabase.sync.core :as sync]
[metabase.test :as mt]
+ [metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
[toucan2.core :as t2]
[toucan2.util :as u]))
@@ -268,6 +270,140 @@
(mt/with-current-user (mt/user->id :crowberto)
(transforms.execute/execute! transform {:run-method :manual}))))))))))
+(doseq [driver [:postgres :mysql :clickhouse :snowflake :bigquery-cloud-sdk]]
+ (defmethod driver/database-supports? [driver ::sleep-query]
+ [_driver _feature _database]
+ true))
+
+(defmethod driver/database-supports? [:redshift ::sleep-query]
+ [_driver _feature _database]
+ false)
+
+(defmulti sleep-numbers-query
+ "Returns a query that will sleep for a few seconds and return a list of numbers."
+ {:arglists '([driver sleep-sec num])}
+ tx/dispatch-on-driver-with-test-extensions
+ :hierarchy #'driver/hierarchy)
+
+(defmethod sleep-numbers-query :postgres [_driver sleep-sec num]
+ (format "SELECT a FROM (SELECT pg_sleep(%d)) x, generate_series(1, %d) a;" sleep-sec num))
+
+(defmethod sleep-numbers-query :mysql [_driver sleep-sec num]
+ (format "SELECT a FROM (SELECT SLEEP(%d)) x, (%s) a;"
+ sleep-sec
+ (->> (range 1 (inc num))
+ (map #(str "SELECT " % " AS a"))
+ (str/join " UNION ALL "))))
+
+(defmethod sleep-numbers-query :clickhouse [_driver sleep-sec num]
+ (let [q (quot sleep-sec 3)
+ r (rem sleep-sec 3)
+ threes (repeat q "sleep(3)") ;; max time clickhouse can sleep at once is 3 seconds
+ parts (concat threes [(format "sleep(%d)" r)])
+ sleeps (str/join " + " parts)]
+ (format "SELECT number + 1 AS a FROM numbers(%d) WHERE %s = 0;" num sleeps)))
+
+(defmethod sleep-numbers-query :snowflake [_driver sleep-sec num]
+ (format "SELECT SEQ4() + 1 AS a FROM (SELECT SYSTEM$WAIT(%d)), TABLE(GENERATOR(ROWCOUNT => %d))" sleep-sec num))
+
+;; Bigquery doesn't have a generic sleep function so we can't set a specific sleep time.
+;; 110000 has been tested to take 5-10 seconds, adjust this value as needed.
+(defmethod sleep-numbers-query :bigquery-cloud-sdk [_driver _sleep-sec num]
+ (format "SELECT s
+ FROM UNNEST(GENERATE_ARRAY(1, %d)) AS s
+ JOIN (
+ SELECT ARRAY_LENGTH(
+ ARRAY(
+ SELECT SUM(SQRT(x * y))
+ FROM UNNEST(GENERATE_ARRAY(1, 110000)) AS x
+ CROSS JOIN UNNEST(GENERATE_ARRAY(1, 1000)) AS y
+ )
+ ) AS force_eval
+ ) ON force_eval = 1
+ ORDER BY s ASC;"
+ num))
+
+;; Commented out to avoid potential flakes but keeping it as a reference for QUE-2394
+#_(deftest run-mbql-transform-long-running-transform-test
+ (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table ::sleep-query]})
+ (with-transform-cleanup! [target-table {:type :table
+ :schema (t2/select-one-fn :schema :model/Table (mt/id :products))
+ :name "sleep_table"}]
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp (sleep-numbers-query driver/*driver* 5 5))
+ new-query (lib/native-query mp (sleep-numbers-query driver/*driver* 5 6))]
+ (mt/with-temp [:model/Transform transform {:name "transform"
+ :source {:type :query
+ :query query}
+ :target target-table}]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)
+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))
+ transform-id (:id transform)
+ original-result [[1] [2] [3] [4] [5]]
+ query-fn (fn []
+ (->> (lib/query mp table-result)
+ (qp/process-query)
+ (mt/formatted-rows [int])))]
+ (is (= original-result (query-fn)))
+ (let [transform-future (future
+ (t2/update! :model/Transform transform-id {:source {:type :query
+ :query new-query}})
+ (let [new-transform (t2/select-one :model/Transform transform-id)]
+ (transforms.execute/execute! new-transform {:run-method :manual})))
+ query-futures (vec
+ (for [i (range 10)]
+ (future
+ (Thread/sleep (* i 100))
+ (query-fn))))]
+ @transform-future
+ (let [query-results (map deref query-futures)]
+ (doseq [result query-results]
+ (is (= original-result result))))
+ (is (= [[1] [2] [3] [4] [5] [6]] (query-fn))))))))))
+
+(deftest run-transforms-with-strategy-test
+ (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table]})
+ (let [calls (atom [])
+ record-call! (fn [call-name] (swap! calls conj call-name))
+ mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"
+ mock-rows [[1]]
+ transform-details {:conn-spec {}
+ :output-table :test_schema/test_table
+ :database {:id 1}
+ :transform-type :table}]
+ (with-redefs [driver/table-exists? (constantly true)
+ driver/compile-transform (fn [& _] (record-call! :compile-transform) mock-query)
+ driver/execute-raw-queries! (fn [& _] (record-call! :execute-raw-queries!) mock-rows)
+ driver/rename-tables! (fn [& _] (record-call! :rename-tables!))
+ driver/rename-table! (fn [& _] (record-call! :rename-table!))
+ driver/drop-table! (fn [& _] (record-call! :drop-table!))]
+ (testing "only create if table doesn't exist"
+ (reset! calls [])
+ (with-redefs [driver/table-exists? (constantly false)]
+ (driver/run-transform! driver/*driver* transform-details nil)
+ (is (= [:compile-transform :execute-raw-queries!] @calls))))
+ (testing "prefer create or replace strategy first"
+ (when (driver/database-supports? driver/*driver* :create-or-replace-table nil)
+ (reset! calls [])
+ (driver/run-transform! driver/*driver* transform-details nil)
+ (is (= [:compile-transform :execute-raw-queries!] @calls))))
+ (testing "prefer atomic renames strategy second"
+ (when (driver/database-supports? driver/*driver* :atomic-renames nil)
+ (reset! calls [])
+ (driver/run-transform! driver/*driver* transform-details nil)
+ (is (= [:compile-transform :execute-raw-queries! :rename-tables! :drop-table!] @calls))))
+ (testing "prefer partial atomic renames strategy third"
+ (when (and (driver/database-supports? driver/*driver* :rename nil)
+ (not (driver/database-supports? driver/*driver* :atomic-renames nil)))
+ (reset! calls [])
+ (driver/run-transform! driver/*driver* transform-details nil)
+ (is (= [:compile-transform :execute-raw-queries! :drop-table! :rename-table!] @calls))))
+ (testing "fallback to drop and create strategy"
+ (reset! calls [])
+ (driver/run-transform! :h2 transform-details nil)
+ (is (= [:drop-table! :compile-transform :execute-raw-queries!] @calls)))))))
+
(deftest run-mbql-transform-anonymous-user-routing-error-test
(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table]})
(mt/with-premium-features #{:database-routing}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
index 936ca037011..242fe5da276 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
@@ -1,11 +1,11 @@
(ns ^:mb/driver-tests metabase-enterprise.transforms.util-test
"Tests for transform utility functions."
(:require
- [clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.api.common :as api]
[metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
@@ -19,17 +19,16 @@
(let [driver driver/*driver*]
(testing "Basic table name generation"
- (let [result (transforms.util/temp-table-name driver nil)
+ (let [result (driver.u/temp-table-name driver nil)
table-name (name result)]
(is (keyword? result))
(is (nil? (namespace result)))
- (is (str/starts-with? table-name "mb_transform_temp_table_"))
- (is (re-matches #"mb_transform_temp_table_\d+" table-name))))
+ (is (re-matches #"mb_transform_temp_table_[a-f0-9]{8}" table-name))))
(testing "Table name preserves namespace when present"
- (let [result (transforms.util/temp-table-name driver "schema")]
+ (let [result (driver.u/temp-table-name driver :schema/orders)]
(is (= "schema" (namespace result)))
- (is (str/starts-with? (name result) "mb_transform_temp_table_"))))))))
+ (is (re-matches #"mb_transform_temp_table_[a-f0-9]{8}" (name result)))))))))
(deftest temp-table-name-creates-table-test
(testing "temp-table-name produces names that can actually create tables"
@@ -37,7 +36,7 @@
(let [driver driver/*driver*
db-id (mt/id)
- table-name (transforms.util/temp-table-name driver nil)
+ table-name (driver.u/temp-table-name driver :test_table)
schema-name (when (get-method sql.tx/session-schema driver)
(sql.tx/session-schema driver))
qualified-table-name (if schema-name
@@ -58,9 +57,9 @@
nil)))))))))
(deftest is-temp-transform-tables-test
- (testing "tables with shcema"
- (let [table-with-schema {:name (name (transforms.util/temp-table-name :postgres "schema"))}
- table-without-schema {:name (name (transforms.util/temp-table-name :postgres "schema"))}]
+ (testing "tables with schema"
+ (let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}
+ table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}]
(mt/with-premium-features #{}
(is (false? (transforms.util/is-temp-transform-table? table-with-schema)))
(is (false? (transforms.util/is-temp-transform-table? table-without-schema))))
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index c505521d880..34bce70e965 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -776,6 +776,7 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(doseq [[feature supported?] {:convert-timezone true
+ :create-or-replace-table true
:database-routing true
:datetime-diff true
:describe-fields true
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index d2ccd693c24..d923be84fc7 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -644,6 +644,9 @@
;; Does the driver support atomic multi-table renaming
:atomic-renames
+ ;; Does the driver support CREATE OR REPLACE TABLE syntax
+ :create-or-replace-table
+
;; Does the driver support custom writeback actions. Drivers that support this must
;; implement [[execute-write-query!]]
:actions/custom
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index fad16f793e2..cbdadfb2d55 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -18,6 +18,7 @@
[metabase.driver.sql.util :as sql.u]
[metabase.driver.util :as driver.u]
[metabase.util.humanization :as u.humanization]
+ [metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.performance :refer [some]]
[potemkin :as p]))
@@ -122,11 +123,73 @@
;;; | Transforms |
;;; +----------------------------------------------------------------------------------------------------------------+
+(defn- create-table-and-insert-data!
+ [driver transform-details conn-spec]
+ (let [create-query (driver/compile-transform driver transform-details)
+ rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]
+ rows-affected))
+
+(defn- run-with-rename-tables-strategy!
+ [driver database output-table transform-details conn-spec]
+ (let [new-temp (driver.u/temp-table-name driver output-table)
+ old-temp (driver.u/temp-table-name driver output-table)]
+ (try
+ (let [new-temp-details (assoc transform-details :output-table new-temp)
+ rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]
+ (driver/rename-tables! driver (:id database) {output-table old-temp
+ new-temp output-table})
+ (driver/drop-table! driver (:id database) old-temp)
+ {:rows-affected rows-affected})
+ (catch Exception e
+ (log/error e "Failed to run transform using rename-tables strategy")
+ (try (driver/drop-table! driver (:id database) new-temp) (catch Exception _))
+ (throw e)))))
+
+(defn- run-with-create-drop-rename-strategy!
+ [driver database output-table transform-details conn-spec]
+ (let [tmp-table (driver.u/temp-table-name driver output-table)]
+ (try
+ (let [tmp-table-details (assoc transform-details :output-table tmp-table)
+ rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]
+ (driver/drop-table! driver (:id database) output-table)
+ (driver/rename-table! driver (:id database) tmp-table output-table)
+ {:rows-affected rows-affected})
+ (catch Exception e
+ (log/error e "Failed to run transform using create-drop-rename strategy")
+ (try (driver/drop-table! driver (:id database) tmp-table) (catch Exception _))
+ (throw e)))))
+
+(defn- run-with-drop-create-fallback-strategy!
+ [driver database output-table transform-details conn-spec]
+ (try
+ (driver/drop-table! driver (:id database) output-table)
+ {:rows-affected (create-table-and-insert-data! driver transform-details conn-spec)}
+ (catch Exception e
+ (log/error e "Failed to run transform using drop-create strategy")
+ (throw e))))
+
+;; Follows similar logic to `transfer-file-to-db :table`
(defmethod driver/run-transform! [:sql :table]
- [driver {:keys [conn-spec output-table] :as transform-details} {:keys [overwrite?]}]
- (let [queries (cond->> [(driver/compile-transform driver transform-details)]
- overwrite? (cons (driver/compile-drop-table driver output-table)))]
- {:rows-affected (last (driver/execute-raw-queries! driver conn-spec queries))}))
+ [driver {:keys [conn-spec output-table database] :as transform-details} _opts]
+ (let [table-exists? (driver/table-exists? driver database
+ {:schema (namespace output-table)
+ :name (name output-table)})]
+ (cond
+ (or (not table-exists?)
+ (driver/database-supports? driver :create-or-replace-table database))
+ (create-table-and-insert-data! driver transform-details conn-spec)
+
+ ;; Atomic renames fully supported
+ (driver/database-supports? driver :atomic-renames database)
+ (run-with-rename-tables-strategy! driver database output-table transform-details conn-spec)
+
+ ;; Single rename supported, partial atomicity
+ (driver/database-supports? driver :rename database)
+ (run-with-create-drop-rename-strategy! driver database output-table transform-details conn-spec)
+
+ ;; Drop then create, no atomicity
+ :else
+ (run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))
(defmethod driver/run-transform! [:sql :table-incremental]
[driver {:keys [conn-spec database output-table] :as transform-details} _opts]
diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.clj
index 0224cb3ac36..ead1c976c5d 100644
--- a/src/metabase/driver/util.clj
+++ b/src/metabase/driver/util.clj
@@ -822,3 +822,18 @@
{:macaw-error (:error result)}
(-> result :context :cause))))
result))
+
+(def ^:const transform-temp-table-prefix
+ "Prefix used for temporary tables created during transform execution."
+ "mb_transform_temp_table")
+
+(defn temp-table-name
+ "Generate a temporary table name with a random suffix for uniqueness.
+
+ Takes a `table` keyword like `:schema/table_name` where the namespace is the schema.
+
+ Format: <prefix>_<random_suffix>"
+ [_driver table]
+ (let [schema (some-> table namespace)
+ rand (subs (str (random-uuid)) 0 8)]
+ (keyword schema (str transform-temp-table-prefix "_" rand))))