Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Drop metabase_version from serialization (GHY-4013) (#76760)

Eric Normand2d635fadf978
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 6bed3024718..d4a984b32c8 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3514,7 +3514,6 @@
app-db
appearance
collections
- config
events
lib
logger
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 7b288938a1f..97fbe7c828e 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -8,7 +8,6 @@
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.collections.models.collection :as collection]
- [metabase.config.core :as config]
[metabase.models.serialization :as serdes]
[metabase.util :as u]
[metabase.util.log :as log]
@@ -233,22 +232,11 @@
;; extract all non-content entities like data model and settings if necessary
(eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))]))))
-(defn- needs-version?
- "True for extracted entities that should carry a `:metabase_version` stamp."
- [entity]
- (and (not (instance? Exception entity))
- (not= "Setting" (-> entity :serdes/meta last :model))))
-
-(defn- stamp-version [entity]
- (if (needs-version? entity)
- (assoc entity :metabase_version config/mb-version-string)
- entity))
-
(defn extract
"Returns a reducible stream of entities to serialize"
[opts]
(serdes.backfill/backfill-ids!)
- (eduction (map stamp-version) (extract-subtrees opts)))
+ (extract-subtrees opts))
(comment
(def nodes (let [colls (mapv vector (repeat "Collection") (collection-set-for-user nil))]
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 294f761f4ae..ca6642beb17 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -9,7 +9,6 @@
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.app-db.core :as mdb]
[metabase.app-db.transient-error :as transient-error]
- [metabase.config.core :as config]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.util :as u]
@@ -48,10 +47,6 @@
"Document" #{:document}
"Card" #{:dashboard_id :document_id :parameters}})
-(def ^:private ^:dynamic *warned-version-mismatch*
- "Used to avoid double-logging on version mismatches. Warns if nil or set to true"
- nil)
-
(defn- keys-to-strip [ingested]
(let [model (-> ingested :serdes/meta last :model)]
(or (model->circular-dependency-keys model)
@@ -65,22 +60,21 @@
If [[load-one]] throws because it can't find that entity in the filesystem, check if it's already loaded in
our database."
[ctx deps]
- (binding [*warned-version-mismatch* (if (nil? *warned-version-mismatch*) (atom false) *warned-version-mismatch*)]
- (if (empty? deps)
- ctx
- (letfn [(loader [ctx dep]
- (try
- (load-one! ctx dep)
- (catch Exception e
- (cond
- ;; It was missing, but we found it locally, so just return the context.
- (and (= (:error (ex-data e)) ::not-found)
- (serdes/load-find-local dep))
- ctx
+ (if (empty? deps)
+ ctx
+ (letfn [(loader [ctx dep]
+ (try
+ (load-one! ctx dep)
+ (catch Exception e
+ (cond
+ ;; It was missing, but we found it locally, so just return the context.
+ (and (= (:error (ex-data e)) ::not-found)
+ (serdes/load-find-local dep))
+ ctx
- :else
- (throw e)))))]
- (reduce loader ctx deps)))))
+ :else
+ (throw e)))))]
+ (reduce loader ctx deps))))
(defn- safe-local-id
"Looks up the local primary key for `path`, swallowing any exception from the DB lookup.
@@ -120,23 +114,6 @@
(let [model (t2.model/resolve-model (symbol model-name))]
(serdes.backfill/has-entity-id? model))))
-(defn- warn-if-version-mismatch
- "Checks if the version in the exported entity differs from the current Metabase version.
- Logs a warning if there is a mismatch. Entities without a `:metabase_version` (eg. Settings,
- which are bundled into settings.yaml without per-entity metadata) are skipped."
- [ingested path]
- (when (and (or (nil? *warned-version-mismatch*) (not @*warned-version-mismatch*))
- (:metabase_version ingested))
- (let [current-version config/mb-version-string
- exported-version (:metabase_version ingested)]
- (when (not= exported-version current-version)
- (log/warnf "Version mismatch loading %s: exported with: %s, current version: %s"
- path
- exported-version
- current-version)
- (when (not (nil? *warned-version-mismatch*))
- (reset! *warned-version-mismatch* true))))))
-
(defn- load-one!
"Loads a single entity, specified by its `:serdes/meta` abstract path, into the appdb, doing some bookkeeping to
avoid cycles.
@@ -212,7 +189,6 @@
:level (count expanding)}))
local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]
(try
- (warn-if-version-mismatch ingested path)
(with-retries 3 200
(fn []
(t2/with-transaction [_tx]
@@ -255,41 +231,40 @@
:or {backfill? true
continue-on-error false
reindex? true}}]
- (binding [*warned-version-mismatch* (atom false)]
- (u/prog1
- ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient
- ;; failure on one entity doesn't abort the entire import. See #74412.
- (do
- (when backfill?
- (t2/with-transaction [_tx]
- (serdes.backfill/backfill-ids!)))
- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared
- ;; dependencies guide the import, and make sure all containers are imported before contents, etc.
- (let [contents (serdes.ingest/ingest-list ingestion)
- ingest-errors (serdes.ingest/ingest-errors ingestion)
- ctx (cond-> (new-context ingestion)
- (seq ingest-errors) (update :errors into ingest-errors))]
- (when (and (seq ingest-errors) (not continue-on-error))
- (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]
- (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"
- (count ingest-errors)
- (str/join ", " file-names))
- {:ingest-errors ingest-errors
- :files file-names}
- (first ingest-errors)))))
- (log/infof "Starting deserialization, total %s documents" (count contents))
- (reduce (fn [ctx item]
- (try
- (load-one! ctx item)
- (catch Exception e
- (when-not continue-on-error
- (throw e))
- ;; eschew big and scary stacktrace
- (log/warnf (u/strip-error e "Skipping deserialization error"))
- (update ctx :errors conj e))))
- ctx
- contents)))
- (when reindex?
- ;; Reindex after all entities are loaded. Individual entity commits may have produced stale
- ;; search index entries; this ensures the index reflects the final state.
- (search/reindex!)))))
+ (u/prog1
+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient
+ ;; failure on one entity doesn't abort the entire import. See #74412.
+ (do
+ (when backfill?
+ (t2/with-transaction [_tx]
+ (serdes.backfill/backfill-ids!)))
+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared
+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.
+ (let [contents (serdes.ingest/ingest-list ingestion)
+ ingest-errors (serdes.ingest/ingest-errors ingestion)
+ ctx (cond-> (new-context ingestion)
+ (seq ingest-errors) (update :errors into ingest-errors))]
+ (when (and (seq ingest-errors) (not continue-on-error))
+ (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]
+ (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"
+ (count ingest-errors)
+ (str/join ", " file-names))
+ {:ingest-errors ingest-errors
+ :files file-names}
+ (first ingest-errors)))))
+ (log/infof "Starting deserialization, total %s documents" (count contents))
+ (reduce (fn [ctx item]
+ (try
+ (load-one! ctx item)
+ (catch Exception e
+ (when-not continue-on-error
+ (throw e))
+ ;; eschew big and scary stacktrace
+ (log/warnf (u/strip-error e "Skipping deserialization error"))
+ (update ctx :errors conj e))))
+ ctx
+ contents)))
+ (when reindex?
+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale
+ ;; search index entries; this ensures the index reflects the final state.
+ (search/reindex!))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 57358755ae0..ddfd4b25037 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -99,7 +99,6 @@ serdes/meta:
model: Card
archived_directly: false
dashboard_id: null
-metabase_version: v1.54.4-SNAPSHOT (c6780bb)
source_card_id: null
type: %s
document_id: null
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 191fd68fbba..71d5ca04052 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -107,8 +107,7 @@
(defn- clean-entity
"Removes any comparison-confounding fields, like `:created_at`."
[entity]
- (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule
- :metabase_version))
+ (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule))
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
(deftest e2e-storage-ingestion-test
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index 98a8a7dcaa8..3f5f6c8daab 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -12,7 +12,6 @@
[metabase-enterprise.serialization.v2.round-trip-test :as round-trip-test]
[metabase.actions.models :as action]
[metabase.audit-app.core :as audit]
- [metabase.config.core :as config]
[metabase.core.core :as mbc]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
@@ -2847,19 +2846,14 @@
(is (= #{light-eid dark-eid}
(ids-by-model "EmbeddingTheme" (extract/extract {}))))))))
-(deftest stamp-metabase-version-test
- (testing "extract stamps :metabase_version on entities (so load can detect version mismatches)"
+(deftest no-metabase-version-stamp-test
+ (testing "extract does not stamp :metabase_version on entities (GHY-4013: it caused spurious remote-sync diffs)"
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "My Collection"}
:model/Card _ {:name "My Card" :collection_id coll-id}]
(let [extracted (into [] (extract/extract {}))
by-model (group-by (comp :model last :serdes/meta) extracted)]
- (testing "Collections and Cards get the current Metabase version"
- (doseq [m ["Collection" "Card"]
- entity (get by-model m)]
- (is (= config/mb-version-string (:metabase_version entity))
- (str m " should be stamped with the current version"))))
- (testing "Settings are not stamped — settings.yaml only persists :key and :value"
- (doseq [setting (get by-model "Setting")]
- (is (not (contains? setting :metabase_version))
- "Setting should not be stamped"))))))))
+ (doseq [m ["Collection" "Card"]
+ entity (get by-model m)]
+ (is (not (contains? entity :metabase_version))
+ (str m " should not carry a :metabase_version"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index cca41b76fbd..aa0f7a93c9d 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -19,7 +19,6 @@
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.json :as json]
- [metabase.util.log.capture :as log.capture]
[metabase.warehouses.models.database :as models.database]
[toucan2.core :as t2]))
@@ -1995,32 +1994,6 @@
(serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))
(is (= 4 (coll-count)))))))
-(deftest warn-if-version-mismatch-test
- (ts/with-dbs [source-db dest-db dest-db2 dest-db3]
- (ts/with-db source-db
- (mt/with-temp [:model/Collection _ {:name "col-1"}]
- (let [extract (into [] (serdes.extract/extract {:no-settings true}))]
- (ts/with-db dest-db
- (testing "logs a warning when version in serdes/meta differs from current version"
- (let [old-version-extract (map #(assoc % :metabase_version "v1.0.0 (oldcommit)") extract)]
- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]
- (serdes.load/load-metabase! (ingestion-in-memory old-version-extract))
- (is (some #(str/includes? % "Version mismatch loading") (messages)))
- (is (= 1 (count (filter #(str/includes? % "Version mismatch loading") (messages))))
- "Should log a version mismatch warning only once per load")))))
- (ts/with-db dest-db2
- (testing "No warnings when version in serdes/meta matches current version"
- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]
- (serdes.load/load-metabase! (ingestion-in-memory extract))
- (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages))))))))
- (ts/with-db dest-db3
- (testing "No warnings when entities have no :metabase_version (eg. legacy exports or Settings)"
- (let [no-version-extract (map #(dissoc % :metabase_version) extract)]
- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]
- (serdes.load/load-metabase! (ingestion-in-memory no-version-extract))
- (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages))))
- "Missing :metabase_version should be treated as unknown, not as a mismatch"))))))))))
-
(deftest import-published-table-with-existing-database-test
(testing "Importing a published table works when database already exists on target"
(let [serialized (atom nil)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index 183f9f28f75..3acb93d70cd 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -54,6 +54,8 @@
;; Is worth considering when adding entries here, whether they shouldn't just be skipped in extraction.
#{:cache_field_values_schedule
:metadata_sync_schedule
+ ;; instance-specific build version, no longer serialized (GHY-4013). Legacy baseline fixtures still
+ ;; carry it, which exercises that such exports still import cleanly now that it's skipped.
:metabase_version
;; result_metadata is non-deterministic for dashboard/document cards because the Card before-update hook
;; re-computes it without :verified-result-metadata? set. Fixing this properly requires making serdes
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 596490af388..5c6c72db672 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -42,11 +42,11 @@
(testing "the Collections properly exported"
(let [yaml-parent (-> (yaml/from-file (io/file dump-dir "collections" "main"
"some_collection.yaml"))
- (dissoc :serdes/meta :metabase_version)
+ (dissoc :serdes/meta)
(update :created_at t/offset-date-time))
yaml-child (-> (yaml/from-file (io/file dump-dir "collections" "main"
"some_collection" "child_collection.yaml"))
- (dissoc :serdes/meta :metabase_version)
+ (dissoc :serdes/meta)
(update :created_at t/offset-date-time))]
(is (= (-> (into {} (t2/select-one :model/Collection :id (:id parent)))
(dissoc :id :location)
@@ -142,7 +142,6 @@
"databases" "my_company_data"
"tables" "customers"
"fields" "company__SLASH__organization_website.yaml"))
- (dissoc :metabase_version)
(update :visibility_type keyword)
(update :base_type keyword))))))))))
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 1a50efd84f0..a9ed68c343e 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1369,9 +1369,12 @@
(defmethod serdes/make-spec "Card"
[_model-name _opts]
{:copy [:archived :archived_directly :collection_position :collection_preview :description :display
- :embedding_params :enable_embedding :embedding_type :entity_id :metabase_version :public_uuid :type :name
+ :embedding_params :enable_embedding :embedding_type :entity_id :public_uuid :type :name
:card_schema]
- :skip [;; cache invalidation is instance-specific
+ :skip [;; instance-specific build version; serializing it produces spurious remote-sync diffs, and the
+ ;; serialized representation is versioned by :card_schema instead
+ :metabase_version
+ ;; cache invalidation is instance-specific
:cache_invalidated_at
;; those are instance-specific analytic columns
:view_count :last_used_at :initially_published_at