Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Skip H2 databases during serdes extract (GHY-3633) (#74429)

Eric Normand36a1c834d2c4
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index d460eb3da53..ee08740fee9 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -125,7 +125,10 @@
(mt/with-premium-features #{:serialization}
(testing "POST /api/ee/serialization/export"
(mt/with-empty-h2-app-db!
- (mt/with-temp [:model/Collection coll {:name "API Collection"}
+ ;; The Card defaults wire database_id to an H2 test database, which is now skipped during
+ ;; serdes extract. Create a non-H2 database so the data-model export still contains a Database.
+ (mt/with-temp [:model/Database _ {:name "Non-H2 DB" :engine :postgres}
+ :model/Collection coll {:name "API Collection"}
:model/Dashboard _ {:collection_id (:id coll)}
:model/Card card {:collection_id (:id coll)}
:model/Collection coll2 {:name "Other Collection"}
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index c002ab6254f..a3d35c9653f 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -30,12 +30,15 @@
(set! *warn-on-reflection* true)
-;; `reindex!` below is ok in a parallel test since it's not actually executing anything
+;; `reindex!` below is ok in a parallel test since it's not actually executing anything.
+;; Many tests here rely on the H2 test-data database via Card defaults, so we keep the H2 guard
+;; off and re-enable the H2 path in the extract (production keeps it filtered).
#_{:clj-kondo/ignore [:metabase/validate-deftest]}
(use-fixtures :each (fn [thunk]
(mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
models.database/assert-not-h2! (constantly nil)]
- (thunk))))
+ (binding [models.database/*include-h2-in-extract?* true]
+ (thunk)))))
(defn- dir->contents-set [p ^File dir]
(->> dir
@@ -143,7 +146,8 @@
:collection [[100 {:refs {:personal_owner_id ::rs/omit}}]
[10 {:refs {:personal_owner_id ::rs/omit}
:spec-gen {:namespace :snippets}}]]
- :database [[10]]
+ ;; serdes skips H2 databases at extract, so force :postgres here
+ :database [[10 {:spec-gen {:engine :postgres}}]]
;; Tables are special - we define table 0-9 under db0, 10-19 under db1, etc. The :card spec below
;; depends on this relationship.
:table (into [] (for [db [:db0 :db1 :db2 :db3 :db4 :db5 :db6 :db7 :db8 :db9]]
@@ -394,7 +398,7 @@
;; preparation
(mt/test-helpers-set-global-values!
(mt/with-temp
- [:model/Database db1s {:name "my-db"}
+ [:model/Database db1s {:name "my-db" :engine :postgres}
:model/Collection coll1s {:name "My Collection"}
:model/Table table1s {:name "CUSTOMERS"
:db_id (:id db1s)}
@@ -500,7 +504,8 @@
:description "Linked Collection"}
:model/Database {db-id :id
db-name :name} {:name "Linked database"
- :description "Linked database desc"}
+ :description "Linked database desc"
+ :engine :postgres}
:model/Table {table-id :id
table-name :name} {:db_id db-id
:schema "Public"
@@ -754,10 +759,11 @@
(ts/with-dbs [source-db dest-db]
(ts/with-db source-db
(mt/with-temp
- [:model/Database {router-db-id :id} {:name "Router"}
+ [:model/Database {router-db-id :id} {:name "Router" :engine :postgres}
:model/DatabaseRouter _ {:database_id router-db-id :user_attribute "foobar"}
:model/Database _ {:router_database_id router-db-id
- :name "Destination"}]
+ :name "Destination"
+ :engine :postgres}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
(storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index ae961df1e4c..12fd6c4e527 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1727,7 +1727,8 @@
:model/Collection {clean-coll-id :id
clean-coll-eid :entity_id} {:name "Clean Collection"}
:model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}
- :model/Database {db-id :id} {}
+ ;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
:model/Card {card1-id :id} {:name "Some Card", :database_id db-id}
:model/Card {clean-card-eid :entity_id} {:name "Clean Card"
:collection_id clean-coll-id
@@ -1837,6 +1838,15 @@
:no-data-model true})]
(is (= #{} (ids-by-model "Collection" ser)))))))
+(deftest skip-h2-databases-test
+ (testing "H2 databases must not be extracted because import rejects them (see GHY-3633)"
+ (mt/with-empty-h2-app-db!
+ (mt/with-temp [:model/Database _h2-db {:name "H2 DB" :engine :h2}
+ :model/Database _non-h2 {:name "Postgres DB" :engine :postgres}]
+ (let [ser (extract/extract {:no-settings true})]
+ (is (= #{"Postgres DB"} (ids-by-model "Database" ser))
+ "Only non-H2 databases should appear in the extract"))))))
+
(deftest xray-of-analytics-model-export-test
(testing "X-rays of analytics models can be exported without errors"
(mt/with-empty-h2-app-db!
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index bc2b579854e..8b0e9cda7f6 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -23,12 +23,15 @@
[metabase.warehouses.models.database :as models.database]
[toucan2.core :as t2]))
-;; `reindex!` below is ok in a parallel test since it's not actually executing anything
+;; `reindex!` below is ok in a parallel test since it's not actually executing anything.
+;; Many tests here use the H2 test-data database (Card defaults), so we keep the H2 guard off
+;; and re-enable H2 in the extract (production keeps it filtered).
#_{:clj-kondo/ignore [:metabase/validate-deftest]}
(use-fixtures :each (fn [thunk]
(mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)
models.database/assert-not-h2! (constantly nil)]
- (thunk))))
+ (binding [models.database/*include-h2-in-extract?* true]
+ (thunk)))))
(defn- no-labels [path]
(mapv #(dissoc % :label) path))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index 0f72f469387..ea6085e1b92 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -25,7 +25,7 @@
[clojure.java.io :as io]
[clojure.set :as set]
[clojure.string :as str]
- [clojure.test :refer [deftest is testing use-fixtures]]
+ [clojure.test :refer [deftest is testing]]
[clojure.walk :as walk]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as ingest]
@@ -38,18 +38,12 @@
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.util.log :as log]
- [metabase.util.yaml :as yaml]
- [metabase.warehouses.models.database :as models.database])
+ [metabase.util.yaml :as yaml])
(:import
(java.io File)
(java.nio.file Files Path StandardCopyOption)
(java.nio.file.attribute FileAttribute)))
-#_{:clj-kondo/ignore [:metabase/validate-deftest]}
-(use-fixtures :each (fn [thunk]
- (mt/with-dynamic-fn-redefs [models.database/assert-not-h2! (constantly nil)]
- (thunk))))
-
(set! *warn-on-reflection* true)
(def ^File source-dir (io/as-file (io/resource "serialization_baseline")))
@@ -190,15 +184,14 @@
(defn- update-baseline!
"Run this if you've examined the output of [[directory-comparison-test]], and are happy to accept the changes."
[]
- (mt/with-dynamic-fn-redefs [models.database/assert-not-h2! (constantly nil)]
- (let [temp-dir (temp-dir "serialization_test")
- output-dir (.toFile (.resolve temp-dir "serialization_output"))]
- (mt/with-empty-h2-app-db!
- (load-extract! source-dir output-dir))
- (delete-dir-contents! source-dir)
- (Files/move (.toPath output-dir)
- (.toPath source-dir)
- (into-array [StandardCopyOption/REPLACE_EXISTING])))))
+ (let [temp-dir (temp-dir "serialization_test")
+ output-dir (.toFile (.resolve temp-dir "serialization_output"))]
+ (mt/with-empty-h2-app-db!
+ (load-extract! source-dir output-dir))
+ (delete-dir-contents! source-dir)
+ (Files/move (.toPath output-dir)
+ (.toPath source-dir)
+ (into-array [StandardCopyOption/REPLACE_EXISTING]))))
(comment
(delete-dir-contents! source-dir)
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 5f740821440..2ddb7c60c42 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -630,12 +630,20 @@
:is_sample false
:uploads_enabled false}})
+(def ^:dynamic *include-h2-in-extract?*
+ "When false (the default), [[serdes/extract-query]] skips H2 databases because they are rejected at import time
+ by [[assert-not-h2!]]. Round-trip tests that exercise H2 throughout — and rebind `assert-not-h2!` accordingly —
+ may rebind this to `true` to keep the H2 databases in the extract."
+ false)
+
(defmethod serdes/extract-query "Database"
[model-name {:keys [where]}]
(t2/reducible-select (keyword "model" model-name)
- {:where [:and
- (or where true)
- [:= :router_database_id nil]]}))
+ {:where (cond-> [:and
+ (or where true)
+ [:= :router_database_id nil]]
+ (not *include-h2-in-extract?*)
+ (conj [:not= :engine "h2"]))}))
(defmethod serdes/entity-id "Database"
[_ {:keys [name]}]
diff --git a/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml b/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml
index 33119db4275..a5da8667b03 100644
--- a/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml
@@ -1,11 +1,11 @@
name: Sample Database
-engine: h2
+engine: postgres
dbms_version:
- flavor: H2
- version: 2.1.214 (2022-06-13)
+ flavor: PostgreSQL
+ version: '14.0'
semantic-version:
- - 2
- - 1
+ - 14
+ - 0
created_at: '2024-08-28T14:38:42.764234Z'
timezone: UTC
metadata_sync_schedule: 0 22 * * * ? *