The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Skip H2 databases during serdes extract (GHY-3633) (#74429)
Eric Normand36a1c834d2c4
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex d460eb3da53..ee08740fee9 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -125,7 +125,10 @@(mt/with-premium-features #{:serialization}(testing "POST /api/ee/serialization/export"(mt/with-empty-h2-app-db!- (mt/with-temp [:model/Collection coll {:name "API Collection"}+ ;; The Card defaults wire database_id to an H2 test database, which is now skipped during+ ;; serdes extract. Create a non-H2 database so the data-model export still contains a Database.+ (mt/with-temp [:model/Database _ {:name "Non-H2 DB" :engine :postgres}+ :model/Collection coll {:name "API Collection"}:model/Dashboard _ {:collection_id (:id coll)}:model/Card card {:collection_id (:id coll)}:model/Collection coll2 {:name "Other Collection"}diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex c002ab6254f..a3d35c9653f 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -30,12 +30,15 @@(set! *warn-on-reflection* true)-;; `reindex!` below is ok in a parallel test since it's not actually executing anything+;; `reindex!` below is ok in a parallel test since it's not actually executing anything.+;; Many tests here rely on the H2 test-data database via Card defaults, so we keep the H2 guard+;; off and re-enable the H2 path in the extract (production keeps it filtered).#_{:clj-kondo/ignore [:metabase/validate-deftest]}(use-fixtures :each (fn [thunk](mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)models.database/assert-not-h2! (constantly nil)]- (thunk))))+ (binding [models.database/*include-h2-in-extract?* true]+ (thunk)))))(defn- dir->contents-set [p ^File dir](->> dir@@ -143,7 +146,8 @@:collection [[100 {:refs {:personal_owner_id ::rs/omit}}][10 {:refs {:personal_owner_id ::rs/omit}:spec-gen {:namespace :snippets}}]]- :database [[10]]+ ;; serdes skips H2 databases at extract, so force :postgres here+ :database [[10 {:spec-gen {:engine :postgres}}]];; Tables are special - we define table 0-9 under db0, 10-19 under db1, etc. The :card spec below;; depends on this relationship.:table (into [] (for [db [:db0 :db1 :db2 :db3 :db4 :db5 :db6 :db7 :db8 :db9]]@@ -394,7 +398,7 @@;; preparation(mt/test-helpers-set-global-values!(mt/with-temp- [:model/Database db1s {:name "my-db"}+ [:model/Database db1s {:name "my-db" :engine :postgres}:model/Collection coll1s {:name "My Collection"}:model/Table table1s {:name "CUSTOMERS":db_id (:id db1s)}@@ -500,7 +504,8 @@:description "Linked Collection"}:model/Database {db-id :iddb-name :name} {:name "Linked database"- :description "Linked database desc"}+ :description "Linked database desc"+ :engine :postgres}:model/Table {table-id :idtable-name :name} {:db_id db-id:schema "Public"@@ -754,10 +759,11 @@(ts/with-dbs [source-db dest-db](ts/with-db source-db(mt/with-temp- [:model/Database {router-db-id :id} {:name "Router"}+ [:model/Database {router-db-id :id} {:name "Router" :engine :postgres}:model/DatabaseRouter _ {:database_id router-db-id :user_attribute "foobar"}:model/Database _ {:router_database_id router-db-id- :name "Destination"}]+ :name "Destination"+ :engine :postgres}](let [extraction (serdes/with-cache (into [] (extract/extract {})))](storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex ae961df1e4c..12fd6c4e527 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1727,7 +1727,8 @@:model/Collection {clean-coll-id :idclean-coll-eid :entity_id} {:name "Clean Collection"}:model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}- :model/Database {db-id :id} {}+ ;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}:model/Card {card1-id :id} {:name "Some Card", :database_id db-id}:model/Card {clean-card-eid :entity_id} {:name "Clean Card":collection_id clean-coll-id@@ -1837,6 +1838,15 @@:no-data-model true})](is (= #{} (ids-by-model "Collection" ser)))))))+(deftest skip-h2-databases-test+ (testing "H2 databases must not be extracted because import rejects them (see GHY-3633)"+ (mt/with-empty-h2-app-db!+ (mt/with-temp [:model/Database _h2-db {:name "H2 DB" :engine :h2}+ :model/Database _non-h2 {:name "Postgres DB" :engine :postgres}]+ (let [ser (extract/extract {:no-settings true})]+ (is (= #{"Postgres DB"} (ids-by-model "Database" ser))+ "Only non-H2 databases should appear in the extract"))))))+(deftest xray-of-analytics-model-export-test(testing "X-rays of analytics models can be exported without errors"(mt/with-empty-h2-app-db!diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex bc2b579854e..8b0e9cda7f6 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -23,12 +23,15 @@[metabase.warehouses.models.database :as models.database][toucan2.core :as t2]))-;; `reindex!` below is ok in a parallel test since it's not actually executing anything+;; `reindex!` below is ok in a parallel test since it's not actually executing anything.+;; Many tests here use the H2 test-data database (Card defaults), so we keep the H2 guard off+;; and re-enable H2 in the extract (production keeps it filtered).#_{:clj-kondo/ignore [:metabase/validate-deftest]}(use-fixtures :each (fn [thunk](mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)models.database/assert-not-h2! (constantly nil)]- (thunk))))+ (binding [models.database/*include-h2-in-extract?* true]+ (thunk)))))(defn- no-labels [path](mapv #(dissoc % :label) path))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex 0f72f469387..ea6085e1b92 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -25,7 +25,7 @@[clojure.java.io :as io][clojure.set :as set][clojure.string :as str]- [clojure.test :refer [deftest is testing use-fixtures]]+ [clojure.test :refer [deftest is testing]][clojure.walk :as walk][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as ingest]@@ -38,18 +38,12 @@[metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.util.log :as log]- [metabase.util.yaml :as yaml]- [metabase.warehouses.models.database :as models.database])+ [metabase.util.yaml :as yaml])(:import(java.io File)(java.nio.file Files Path StandardCopyOption)(java.nio.file.attribute FileAttribute)))-#_{:clj-kondo/ignore [:metabase/validate-deftest]}-(use-fixtures :each (fn [thunk]- (mt/with-dynamic-fn-redefs [models.database/assert-not-h2! (constantly nil)]- (thunk))))-(set! *warn-on-reflection* true)(def ^File source-dir (io/as-file (io/resource "serialization_baseline")))@@ -190,15 +184,14 @@(defn- update-baseline!"Run this if you've examined the output of [[directory-comparison-test]], and are happy to accept the changes."[]- (mt/with-dynamic-fn-redefs [models.database/assert-not-h2! (constantly nil)]- (let [temp-dir (temp-dir "serialization_test")- output-dir (.toFile (.resolve temp-dir "serialization_output"))]- (mt/with-empty-h2-app-db!- (load-extract! source-dir output-dir))- (delete-dir-contents! source-dir)- (Files/move (.toPath output-dir)- (.toPath source-dir)- (into-array [StandardCopyOption/REPLACE_EXISTING])))))+ (let [temp-dir (temp-dir "serialization_test")+ output-dir (.toFile (.resolve temp-dir "serialization_output"))]+ (mt/with-empty-h2-app-db!+ (load-extract! source-dir output-dir))+ (delete-dir-contents! source-dir)+ (Files/move (.toPath output-dir)+ (.toPath source-dir)+ (into-array [StandardCopyOption/REPLACE_EXISTING]))))(comment(delete-dir-contents! source-dir)diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 5f740821440..2ddb7c60c42 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -630,12 +630,20 @@:is_sample false:uploads_enabled false}})+(def ^:dynamic *include-h2-in-extract?*+ "When false (the default), [[serdes/extract-query]] skips H2 databases because they are rejected at import time+ by [[assert-not-h2!]]. Round-trip tests that exercise H2 throughout — and rebind `assert-not-h2!` accordingly —+ may rebind this to `true` to keep the H2 databases in the extract."+ false)+(defmethod serdes/extract-query "Database"[model-name {:keys [where]}](t2/reducible-select (keyword "model" model-name)- {:where [:and- (or where true)- [:= :router_database_id nil]]}))+ {:where (cond-> [:and+ (or where true)+ [:= :router_database_id nil]]+ (not *include-h2-in-extract?*)+ (conj [:not= :engine "h2"]))}))(defmethod serdes/entity-id "Database"[_ {:keys [name]}]diff --git a/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml b/test_resources/serialization_baseline/databases/sample_database/sample_database.yamlindex 33119db4275..a5da8667b03 100644--- a/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml+++ b/test_resources/serialization_baseline/databases/sample_database/sample_database.yaml@@ -1,11 +1,11 @@name: Sample Database-engine: h2+engine: postgresdbms_version:- flavor: H2- version: 2.1.214 (2022-06-13)+ flavor: PostgreSQL+ version: '14.0'semantic-version:- - 2- - 1+ - 14+ - 0created_at: '2024-08-28T14:38:42.764234Z'timezone: UTCmetadata_sync_schedule: 0 22 * * * ? *