Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add dataset option to disable fk validation on batch insert (#68646)

Timothy S. Dean59fd52ac43b6
diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.clj
index 8560fdab38a..4c8c887f3f8 100644
--- a/test/metabase/query_processor/remapping_test.clj
+++ b/test/metabase/query_processor/remapping_test.clj
@@ -14,7 +14,6 @@
[metabase.query-processor.test-util :as qp.test-util]
[metabase.test :as mt]
[metabase.test.data.dataset-definitions :as defs]
- [metabase.test.data.sql-jdbc.load-data :as load-data]
[metabase.util.date-2 :as u.date]))
(deftest ^:parallel basic-internal-remapping-test
@@ -219,32 +218,29 @@
;; `created_by` column which references the PK column in that same table. This tests that remapping table aliases are
;; handled correctly
(mt/test-drivers (mt/normal-drivers-with-feature :left-join ::self-referencing-fks)
- ;; MySQL 9.6+ enforces FK constraints row-by-row during bulk INSERT, so we need to disable FK checks
- ;; when loading data for this self-referencing dataset
- (binding [load-data/*disable-fk-checks* true]
- (mt/dataset test-data-self-referencing-user
- (qp.store/with-metadata-provider (-> (mt/metadata-provider)
- (lib.tu/remap-metadata-provider (mt/id :users :created_by)
- (mt/id :users :name))
+ (mt/dataset test-data-self-referencing-user
+ (qp.store/with-metadata-provider (-> (mt/metadata-provider)
+ (lib.tu/remap-metadata-provider (mt/id :users :created_by)
+ (mt/id :users :name))
;; simulate this being a real FK so implicit joins work
- (lib.tu/merged-mock-metadata-provider
- {:fields [{:id (mt/id :users :created_by)
- :fk-target-field-id (mt/id :users :id)}]}))
- (let [results (mt/run-mbql-query users
- {:order-by [[:asc $name]]
- :limit 4})]
- (when (= driver/*driver* :h2)
- (is (= ["ID"
- "NAME"
- "LAST_LOGIN"
- "CREATED_BY"
- "USERS__via__CREATED_BY__NAME"] ; <- remapped column
- (map :lib/desired-column-alias (mt/cols results)))))
- (is (= [[14 "Broen Olujimi" "2014-10-03T13:45:00Z" 13 "Dwight Gresham"]
- [7 "Conchúr Tihomir" "2014-08-02T09:30:00Z" 6 "Shad Ferdynand"]
- [13 "Dwight Gresham" "2014-08-01T10:30:00Z" 12 "Kfir Caj"]
- [2 "Felipinho Asklepios" "2014-12-05T15:15:00Z" 1 "Plato Yeshua"]]
- (mt/rows results)))))))))
+ (lib.tu/merged-mock-metadata-provider
+ {:fields [{:id (mt/id :users :created_by)
+ :fk-target-field-id (mt/id :users :id)}]}))
+ (let [results (mt/run-mbql-query users
+ {:order-by [[:asc $name]]
+ :limit 4})]
+ (when (= driver/*driver* :h2)
+ (is (= ["ID"
+ "NAME"
+ "LAST_LOGIN"
+ "CREATED_BY"
+ "USERS__via__CREATED_BY__NAME"] ; <- remapped column
+ (map :lib/desired-column-alias (mt/cols results)))))
+ (is (= [[14 "Broen Olujimi" "2014-10-03T13:45:00Z" 13 "Dwight Gresham"]
+ [7 "Conchúr Tihomir" "2014-08-02T09:30:00Z" 6 "Shad Ferdynand"]
+ [13 "Dwight Gresham" "2014-08-01T10:30:00Z" 12 "Kfir Caj"]
+ [2 "Felipinho Asklepios" "2014-12-05T15:15:00Z" 1 "Plato Yeshua"]]
+ (mt/rows results))))))))
(defn- remappings-with-metadata
[metadata]
diff --git a/test/metabase/test/data/dataset_definitions.clj b/test/metabase/test/data/dataset_definitions.clj
index 2c1807434e8..a74d9372d57 100644
--- a/test/metabase/test/data/dataset_definitions.clj
+++ b/test/metabase/test/data/dataset_definitions.clj
@@ -166,7 +166,11 @@
(for [[idx [username last-login password-text]] (m/indexed rows)]
[username last-login password-text (if (zero? idx)
1
- idx)])))))
+ idx)])))
+ ;; Self-referencing FKs require disabling FK checks during data loading on MySQL 9.6+,
+ ;; which enforces FK constraints row-by-row during bulk INSERT.
+ (fn [dbdef]
+ (assoc-in dbdef [:options :disable-fk-checks] true))))
(tx/defdataset attempted-murders
"A dataset for testing temporal values with and without timezones. Records of number of crow counts spoted and the
diff --git a/test/metabase/test/data/interface.clj b/test/metabase/test/data/interface.clj
index 6bc052498ed..3ee0f237b48 100644
--- a/test/metabase/test/data/interface.clj
+++ b/test/metabase/test/data/interface.clj
@@ -110,7 +110,10 @@
[:database-name ms/NonBlankString] ; this must be unique
[:table-definitions [:sequential ValidTableDefinition]]
[:options [:map {:closed true}
- [:native-ddl {:optional true} [:sequential :any]]]]]
+ [:native-ddl {:optional true} [:sequential :any]]
+ ;; When true, drivers that support it (e.g., MySQL) will disable FK checks during data loading.
+ ;; Useful for datasets with self-referencing FKs that need to be inserted in a single batch.
+ [:disable-fk-checks {:optional true} :boolean]]]]
(ms/InstanceOfClass DatabaseDefinition)])
;; TODO - this should probably be a protocol instead
diff --git a/test/metabase/test/data/sql_jdbc/load_data.clj b/test/metabase/test/data/sql_jdbc/load_data.clj
index 3a7ca5b0561..001b96bfa1e 100644
--- a/test/metabase/test/data/sql_jdbc/load_data.clj
+++ b/test/metabase/test/data/sql_jdbc/load_data.clj
@@ -255,20 +255,21 @@
(defn- create-db-load-data!
"Load the data for each Table."
- [driver ^java.sql.Connection conn {:keys [table-definitions] :as dbdef}]
- (doseq [tabledef table-definitions
- :let [reference-duration (or (some-> (get @reference-load-durations [(:database-name dbdef) (:table-name tabledef)])
- u/format-nanoseconds)
- "NONE")]]
- (u/profile (format "load-data for %s %s %s (reference H2 duration: %s)"
- (name driver) (:database-name dbdef) (:table-name tabledef) reference-duration)
- (try
- (load-data-for-table-definition! driver conn dbdef tabledef)
- (catch Throwable e
- (throw (ex-info (format "Error loading data: %s" (ex-message e))
- {:driver driver, :tabledef (update tabledef :rows (fn [rows]
- (concat (take 10 rows) ['...])))}
- e)))))))
+ [driver ^java.sql.Connection conn {:keys [table-definitions options] :as dbdef}]
+ (binding [*disable-fk-checks* (get options :disable-fk-checks false)]
+ (doseq [tabledef table-definitions
+ :let [reference-duration (or (some-> (get @reference-load-durations [(:database-name dbdef) (:table-name tabledef)])
+ u/format-nanoseconds)
+ "NONE")]]
+ (u/profile (format "load-data for %s %s %s (reference H2 duration: %s)"
+ (name driver) (:database-name dbdef) (:table-name tabledef) reference-duration)
+ (try
+ (load-data-for-table-definition! driver conn dbdef tabledef)
+ (catch Throwable e
+ (throw (ex-info (format "Error loading data: %s" (ex-message e))
+ {:driver driver, :tabledef (update tabledef :rows (fn [rows]
+ (concat (take 10 rows) ['...])))}
+ e))))))))
(defn create-db!
"Default implementation of [[tx/create-db!]] for SQL drivers. Loads test data into a data