The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Run transforms with a temp table + rename to avoid blocking queries (#68756)
Riley Thompson3142c394354b
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljindex fa13ee42ec9..5d6f728e0a2 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -33,12 +33,10 @@[:map[:overwrite? :boolean]])-(defn- transform-opts [{:keys [transform-type]}]- (case transform-type- :table {:overwrite? true}-- ;; once we have more than just append, dispatch on :target-incremental-strategy- :table-incremental {}))+(defn- transform-opts [_transform-details]+ ;; once we have more than just :table and :table-incremental as transform-types,+ ;; then we can dispatch on :target-incremental-strategy+ {})(defn- throw-if-db-routing-enabled [transform driver database](when (transforms.util/db-routing-enabled? database)diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex f9b43fbbb7a..fc208c3cdad 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -41,10 +41,6 @@(set! *warn-on-reflection* true)-(def ^:const transform-temp-table-prefix- "Prefix used for temporary tables created during transform execution."- "mb_transform_temp_table")-(defn qualified-table-name"Return the name of the target table of a transform as a possibly qualified symbol."[_driver {:keys [schema name]}]@@ -499,21 +495,6 @@(log/infof "Dropping table %s" table-name)(driver/drop-table! driver database-id table-name))-(defn temp-table-name- "Generate a temporary table name with current timestamp in milliseconds.- If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"- [driver schema]- (let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))- timestamp (str (System/currentTimeMillis))- prefix (str transform-temp-table-prefix "_")- available (- max-len (count prefix))- ;; If we don't have enough space, take the later digits of the timestamp- suffix (if (>= available (count timestamp))- timestamp- (subs timestamp (- (count timestamp) available)))- table-name (str prefix suffix)]- (keyword schema table-name)))-(defn rename-tables!"Rename multiple tables atomically within a transaction using the new driver/rename-tables method.This is a simpler, composable operation that only handles renaming."@@ -526,7 +507,7 @@:feature :transforms[table](when-let [table-name (:name table)]- (str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))+ (str/starts-with? (u/lower-case-en table-name) driver.u/transform-temp-table-prefix)))(defn db-routing-enabled?"Returns whether or not the given database is either a router or destination database"diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex a51f726228e..c833f919d0c 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -166,11 +166,8 @@"Transfer data using the rename-tables*! multimethod with atomicity guarantees.Creates new table, then atomically renames target->old and new->target, then drops old."[driver db-id table-name metadata data-source]- (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))- temp-table-name (u/poll {:thunk #(transforms.util/temp-table-name driver (namespace table-name))- :done? #(not= source-table-name %)- ;; Poll every 1ms to quickly generate a different timestamp-based table name- :interval-ms 1})]+ (let [source-table-name (driver.u/temp-table-name driver table-name)+ temp-table-name (driver.u/temp-table-name driver table-name)](log/info "Using rename-tables strategy with atomicity guarantees")(try@@ -190,7 +187,7 @@"Transfer data using create + drop + rename to minimize time without data.Creates new table, drops old table, then renames new->target."[driver db-id table-name metadata data-source]- (let [source-table-name (transforms.util/temp-table-name driver (namespace table-name))]+ (let [source-table-name (driver.u/temp-table-name driver table-name)](log/info "Using create-drop-rename strategy to minimize downtime")(trydiff --git a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.cljindex 2464471c0f8..e76ce13487e 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/execute_test.clj@@ -1,5 +1,6 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.execute-test(:require+ [clojure.string :as str][clojure.test :refer :all][metabase-enterprise.transforms.execute :as transforms.execute][metabase-enterprise.transforms.query-test-util :as query-test-util]@@ -13,6 +14,7 @@[metabase.query-processor :as qp][metabase.sync.core :as sync][metabase.test :as mt]+ [metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx][toucan2.core :as t2][toucan2.util :as u]))@@ -268,6 +270,140 @@(mt/with-current-user (mt/user->id :crowberto)(transforms.execute/execute! transform {:run-method :manual}))))))))))+(doseq [driver [:postgres :mysql :clickhouse :snowflake :bigquery-cloud-sdk]]+ (defmethod driver/database-supports? [driver ::sleep-query]+ [_driver _feature _database]+ true))++(defmethod driver/database-supports? [:redshift ::sleep-query]+ [_driver _feature _database]+ false)++(defmulti sleep-numbers-query+ "Returns a query that will sleep for a few seconds and return a list of numbers."+ {:arglists '([driver sleep-sec num])}+ tx/dispatch-on-driver-with-test-extensions+ :hierarchy #'driver/hierarchy)++(defmethod sleep-numbers-query :postgres [_driver sleep-sec num]+ (format "SELECT a FROM (SELECT pg_sleep(%d)) x, generate_series(1, %d) a;" sleep-sec num))++(defmethod sleep-numbers-query :mysql [_driver sleep-sec num]+ (format "SELECT a FROM (SELECT SLEEP(%d)) x, (%s) a;"+ sleep-sec+ (->> (range 1 (inc num))+ (map #(str "SELECT " % " AS a"))+ (str/join " UNION ALL "))))++(defmethod sleep-numbers-query :clickhouse [_driver sleep-sec num]+ (let [q (quot sleep-sec 3)+ r (rem sleep-sec 3)+ threes (repeat q "sleep(3)") ;; max time clickhouse can sleep at once is 3 seconds+ parts (concat threes [(format "sleep(%d)" r)])+ sleeps (str/join " + " parts)]+ (format "SELECT number + 1 AS a FROM numbers(%d) WHERE %s = 0;" num sleeps)))++(defmethod sleep-numbers-query :snowflake [_driver sleep-sec num]+ (format "SELECT SEQ4() + 1 AS a FROM (SELECT SYSTEM$WAIT(%d)), TABLE(GENERATOR(ROWCOUNT => %d))" sleep-sec num))++;; Bigquery doesn't have a generic sleep function so we can't set a specific sleep time.+;; 110000 has been tested to take 5-10 seconds, adjust this value as needed.+(defmethod sleep-numbers-query :bigquery-cloud-sdk [_driver _sleep-sec num]+ (format "SELECT s+ FROM UNNEST(GENERATE_ARRAY(1, %d)) AS s+ JOIN (+ SELECT ARRAY_LENGTH(+ ARRAY(+ SELECT SUM(SQRT(x * y))+ FROM UNNEST(GENERATE_ARRAY(1, 110000)) AS x+ CROSS JOIN UNNEST(GENERATE_ARRAY(1, 1000)) AS y+ )+ ) AS force_eval+ ) ON force_eval = 1+ ORDER BY s ASC;"+ num))++;; Commented out to avoid potential flakes but keeping it as a reference for QUE-2394+#_(deftest run-mbql-transform-long-running-transform-test+ (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table ::sleep-query]})+ (with-transform-cleanup! [target-table {:type :table+ :schema (t2/select-one-fn :schema :model/Table (mt/id :products))+ :name "sleep_table"}]+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp (sleep-numbers-query driver/*driver* 5 5))+ new-query (lib/native-query mp (sleep-numbers-query driver/*driver* 5 6))]+ (mt/with-temp [:model/Transform transform {:name "transform"+ :source {:type :query+ :query query}+ :target target-table}]+ (transforms.execute/execute! transform {:run-method :manual})+ (let [_ (transforms.tu/wait-for-table (:name target-table) 10000)+ table-result (lib.metadata/table mp (mt/id (keyword (:name target-table))))+ transform-id (:id transform)+ original-result [[1] [2] [3] [4] [5]]+ query-fn (fn []+ (->> (lib/query mp table-result)+ (qp/process-query)+ (mt/formatted-rows [int])))]+ (is (= original-result (query-fn)))+ (let [transform-future (future+ (t2/update! :model/Transform transform-id {:source {:type :query+ :query new-query}})+ (let [new-transform (t2/select-one :model/Transform transform-id)]+ (transforms.execute/execute! new-transform {:run-method :manual})))+ query-futures (vec+ (for [i (range 10)]+ (future+ (Thread/sleep (* i 100))+ (query-fn))))]+ @transform-future+ (let [query-results (map deref query-futures)]+ (doseq [result query-results]+ (is (= original-result result))))+ (is (= [[1] [2] [3] [4] [5] [6]] (query-fn))))))))))++(deftest run-transforms-with-strategy-test+ (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table]})+ (let [calls (atom [])+ record-call! (fn [call-name] (swap! calls conj call-name))+ mock-query "CREATE TABLE test_schema.test_table AS (SELECT 1 AS x);"+ mock-rows [[1]]+ transform-details {:conn-spec {}+ :output-table :test_schema/test_table+ :database {:id 1}+ :transform-type :table}]+ (with-redefs [driver/table-exists? (constantly true)+ driver/compile-transform (fn [& _] (record-call! :compile-transform) mock-query)+ driver/execute-raw-queries! (fn [& _] (record-call! :execute-raw-queries!) mock-rows)+ driver/rename-tables! (fn [& _] (record-call! :rename-tables!))+ driver/rename-table! (fn [& _] (record-call! :rename-table!))+ driver/drop-table! (fn [& _] (record-call! :drop-table!))]+ (testing "only create if table doesn't exist"+ (reset! calls [])+ (with-redefs [driver/table-exists? (constantly false)]+ (driver/run-transform! driver/*driver* transform-details nil)+ (is (= [:compile-transform :execute-raw-queries!] @calls))))+ (testing "prefer create or replace strategy first"+ (when (driver/database-supports? driver/*driver* :create-or-replace-table nil)+ (reset! calls [])+ (driver/run-transform! driver/*driver* transform-details nil)+ (is (= [:compile-transform :execute-raw-queries!] @calls))))+ (testing "prefer atomic renames strategy second"+ (when (driver/database-supports? driver/*driver* :atomic-renames nil)+ (reset! calls [])+ (driver/run-transform! driver/*driver* transform-details nil)+ (is (= [:compile-transform :execute-raw-queries! :rename-tables! :drop-table!] @calls))))+ (testing "prefer partial atomic renames strategy third"+ (when (and (driver/database-supports? driver/*driver* :rename nil)+ (not (driver/database-supports? driver/*driver* :atomic-renames nil)))+ (reset! calls [])+ (driver/run-transform! driver/*driver* transform-details nil)+ (is (= [:compile-transform :execute-raw-queries! :drop-table! :rename-table!] @calls))))+ (testing "fallback to drop and create strategy"+ (reset! calls [])+ (driver/run-transform! :h2 transform-details nil)+ (is (= [:drop-table! :compile-transform :execute-raw-queries!] @calls)))))))+(deftest run-mbql-transform-anonymous-user-routing-error-test(mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table]})(mt/with-premium-features #{:database-routing}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.cljindex 936ca037011..242fe5da276 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj@@ -1,11 +1,11 @@(ns ^:mb/driver-tests metabase-enterprise.transforms.util-test"Tests for transform utility functions."(:require- [clojure.string :as str][clojure.test :refer :all][metabase-enterprise.transforms.util :as transforms.util][metabase.api.common :as api][metabase.driver :as driver]+ [metabase.driver.util :as driver.u][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]@@ -19,17 +19,16 @@(let [driver driver/*driver*](testing "Basic table name generation"- (let [result (transforms.util/temp-table-name driver nil)+ (let [result (driver.u/temp-table-name driver nil)table-name (name result)](is (keyword? result))(is (nil? (namespace result)))- (is (str/starts-with? table-name "mb_transform_temp_table_"))- (is (re-matches #"mb_transform_temp_table_\d+" table-name))))+ (is (re-matches #"mb_transform_temp_table_[a-f0-9]{8}" table-name))))(testing "Table name preserves namespace when present"- (let [result (transforms.util/temp-table-name driver "schema")]+ (let [result (driver.u/temp-table-name driver :schema/orders)](is (= "schema" (namespace result)))- (is (str/starts-with? (name result) "mb_transform_temp_table_"))))))))+ (is (re-matches #"mb_transform_temp_table_[a-f0-9]{8}" (name result)))))))))(deftest temp-table-name-creates-table-test(testing "temp-table-name produces names that can actually create tables"@@ -37,7 +36,7 @@(let [driver driver/*driver*db-id (mt/id)- table-name (transforms.util/temp-table-name driver nil)+ table-name (driver.u/temp-table-name driver :test_table)schema-name (when (get-method sql.tx/session-schema driver)(sql.tx/session-schema driver))qualified-table-name (if schema-name@@ -58,9 +57,9 @@nil)))))))))(deftest is-temp-transform-tables-test- (testing "tables with shcema"- (let [table-with-schema {:name (name (transforms.util/temp-table-name :postgres "schema"))}- table-without-schema {:name (name (transforms.util/temp-table-name :postgres "schema"))}]+ (testing "tables with schema"+ (let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}+ table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}](mt/with-premium-features #{}(is (false? (transforms.util/is-temp-transform-table? table-with-schema)))(is (false? (transforms.util/is-temp-transform-table? table-without-schema))))diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex c505521d880..34bce70e965 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -776,6 +776,7 @@;;; +----------------------------------------------------------------------------------------------------------------+(doseq [[feature supported?] {:convert-timezone true+ :create-or-replace-table true:database-routing true:datetime-diff true:describe-fields truediff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex d2ccd693c24..d923be84fc7 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -644,6 +644,9 @@;; Does the driver support atomic multi-table renaming:atomic-renames+ ;; Does the driver support CREATE OR REPLACE TABLE syntax+ :create-or-replace-table+;; Does the driver support custom writeback actions. Drivers that support this must;; implement [[execute-write-query!]]:actions/customdiff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex fad16f793e2..cbdadfb2d55 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -18,6 +18,7 @@[metabase.driver.sql.util :as sql.u][metabase.driver.util :as driver.u][metabase.util.humanization :as u.humanization]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.performance :refer [some]][potemkin :as p]))@@ -122,11 +123,73 @@;;; | Transforms |;;; +----------------------------------------------------------------------------------------------------------------++(defn- create-table-and-insert-data!+ [driver transform-details conn-spec]+ (let [create-query (driver/compile-transform driver transform-details)+ rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]+ rows-affected))++(defn- run-with-rename-tables-strategy!+ [driver database output-table transform-details conn-spec]+ (let [new-temp (driver.u/temp-table-name driver output-table)+ old-temp (driver.u/temp-table-name driver output-table)]+ (try+ (let [new-temp-details (assoc transform-details :output-table new-temp)+ rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]+ (driver/rename-tables! driver (:id database) {output-table old-temp+ new-temp output-table})+ (driver/drop-table! driver (:id database) old-temp)+ {:rows-affected rows-affected})+ (catch Exception e+ (log/error e "Failed to run transform using rename-tables strategy")+ (try (driver/drop-table! driver (:id database) new-temp) (catch Exception _))+ (throw e)))))++(defn- run-with-create-drop-rename-strategy!+ [driver database output-table transform-details conn-spec]+ (let [tmp-table (driver.u/temp-table-name driver output-table)]+ (try+ (let [tmp-table-details (assoc transform-details :output-table tmp-table)+ rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]+ (driver/drop-table! driver (:id database) output-table)+ (driver/rename-table! driver (:id database) tmp-table output-table)+ {:rows-affected rows-affected})+ (catch Exception e+ (log/error e "Failed to run transform using create-drop-rename strategy")+ (try (driver/drop-table! driver (:id database) tmp-table) (catch Exception _))+ (throw e)))))++(defn- run-with-drop-create-fallback-strategy!+ [driver database output-table transform-details conn-spec]+ (try+ (driver/drop-table! driver (:id database) output-table)+ {:rows-affected (create-table-and-insert-data! driver transform-details conn-spec)}+ (catch Exception e+ (log/error e "Failed to run transform using drop-create strategy")+ (throw e))))++;; Follows similar logic to `transfer-file-to-db :table`(defmethod driver/run-transform! [:sql :table]- [driver {:keys [conn-spec output-table] :as transform-details} {:keys [overwrite?]}]- (let [queries (cond->> [(driver/compile-transform driver transform-details)]- overwrite? (cons (driver/compile-drop-table driver output-table)))]- {:rows-affected (last (driver/execute-raw-queries! driver conn-spec queries))}))+ [driver {:keys [conn-spec output-table database] :as transform-details} _opts]+ (let [table-exists? (driver/table-exists? driver database+ {:schema (namespace output-table)+ :name (name output-table)})]+ (cond+ (or (not table-exists?)+ (driver/database-supports? driver :create-or-replace-table database))+ (create-table-and-insert-data! driver transform-details conn-spec)++ ;; Atomic renames fully supported+ (driver/database-supports? driver :atomic-renames database)+ (run-with-rename-tables-strategy! driver database output-table transform-details conn-spec)++ ;; Single rename supported, partial atomicity+ (driver/database-supports? driver :rename database)+ (run-with-create-drop-rename-strategy! driver database output-table transform-details conn-spec)++ ;; Drop then create, no atomicity+ :else+ (run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))(defmethod driver/run-transform! [:sql :table-incremental][driver {:keys [conn-spec database output-table] :as transform-details} _opts]diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex 0224cb3ac36..ead1c976c5d 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -822,3 +822,18 @@{:macaw-error (:error result)}(-> result :context :cause))))result))++(def ^:const transform-temp-table-prefix+ "Prefix used for temporary tables created during transform execution."+ "mb_transform_temp_table")++(defn temp-table-name+ "Generate a temporary table name with a random suffix for uniqueness.++ Takes a `table` keyword like `:schema/table_name` where the namespace is the schema.++ Format: <prefix>_<random_suffix>"+ [_driver table]+ (let [schema (some-> table namespace)+ rand (subs (str (random-uuid)) 0 8)]+ (keyword schema (str transform-temp-table-prefix "_" rand))))