The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Drop metabase_version from serialization (GHY-4013) (#76760)
Eric Normand2d635fadf978
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 6bed3024718..d4a984b32c8 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3514,7 +3514,6 @@app-dbappearancecollections- configeventslibloggerdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 7b288938a1f..97fbe7c828e 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -8,7 +8,6 @@[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill][metabase-enterprise.serialization.v2.models :as serdes.models][metabase.collections.models.collection :as collection]- [metabase.config.core :as config][metabase.models.serialization :as serdes][metabase.util :as u][metabase.util.log :as log]@@ -233,22 +232,11 @@;; extract all non-content entities like data model and settings if necessary(eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))]))))-(defn- needs-version?- "True for extracted entities that should carry a `:metabase_version` stamp."- [entity]- (and (not (instance? Exception entity))- (not= "Setting" (-> entity :serdes/meta last :model))))--(defn- stamp-version [entity]- (if (needs-version? entity)- (assoc entity :metabase_version config/mb-version-string)- entity))-(defn extract"Returns a reducible stream of entities to serialize"[opts](serdes.backfill/backfill-ids!)- (eduction (map stamp-version) (extract-subtrees opts)))+ (extract-subtrees opts))(comment(def nodes (let [colls (mapv vector (repeat "Collection") (collection-set-for-user nil))]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 294f761f4ae..ca6642beb17 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -9,7 +9,6 @@[metabase-enterprise.serialization.v2.models :as serdes.models][metabase.app-db.core :as mdb][metabase.app-db.transient-error :as transient-error]- [metabase.config.core :as config][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.util :as u]@@ -48,10 +47,6 @@"Document" #{:document}"Card" #{:dashboard_id :document_id :parameters}})-(def ^:private ^:dynamic *warned-version-mismatch*- "Used to avoid double-logging on version mismatches. Warns if nil or set to true"- nil)-(defn- keys-to-strip [ingested](let [model (-> ingested :serdes/meta last :model)](or (model->circular-dependency-keys model)@@ -65,22 +60,21 @@If [[load-one]] throws because it can't find that entity in the filesystem, check if it's already loaded inour database."[ctx deps]- (binding [*warned-version-mismatch* (if (nil? *warned-version-mismatch*) (atom false) *warned-version-mismatch*)]- (if (empty? deps)- ctx- (letfn [(loader [ctx dep]- (try- (load-one! ctx dep)- (catch Exception e- (cond- ;; It was missing, but we found it locally, so just return the context.- (and (= (:error (ex-data e)) ::not-found)- (serdes/load-find-local dep))- ctx+ (if (empty? deps)+ ctx+ (letfn [(loader [ctx dep]+ (try+ (load-one! ctx dep)+ (catch Exception e+ (cond+ ;; It was missing, but we found it locally, so just return the context.+ (and (= (:error (ex-data e)) ::not-found)+ (serdes/load-find-local dep))+ ctx- :else- (throw e)))))]- (reduce loader ctx deps)))))+ :else+ (throw e)))))]+ (reduce loader ctx deps))))(defn- safe-local-id"Looks up the local primary key for `path`, swallowing any exception from the DB lookup.@@ -120,23 +114,6 @@(let [model (t2.model/resolve-model (symbol model-name))](serdes.backfill/has-entity-id? model))))-(defn- warn-if-version-mismatch- "Checks if the version in the exported entity differs from the current Metabase version.- Logs a warning if there is a mismatch. Entities without a `:metabase_version` (eg. Settings,- which are bundled into settings.yaml without per-entity metadata) are skipped."- [ingested path]- (when (and (or (nil? *warned-version-mismatch*) (not @*warned-version-mismatch*))- (:metabase_version ingested))- (let [current-version config/mb-version-string- exported-version (:metabase_version ingested)]- (when (not= exported-version current-version)- (log/warnf "Version mismatch loading %s: exported with: %s, current version: %s"- path- exported-version- current-version)- (when (not (nil? *warned-version-mismatch*))- (reset! *warned-version-mismatch* true))))))-(defn- load-one!"Loads a single entity, specified by its `:serdes/meta` abstract path, into the appdb, doing some bookkeeping toavoid cycles.@@ -212,7 +189,6 @@:level (count expanding)}))local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))](try- (warn-if-version-mismatch ingested path)(with-retries 3 200(fn [](t2/with-transaction [_tx]@@ -255,41 +231,40 @@:or {backfill? truecontinue-on-error falsereindex? true}}]- (binding [*warned-version-mismatch* (atom false)]- (u/prog1- ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient- ;; failure on one entity doesn't abort the entire import. See #74412.- (do- (when backfill?- (t2/with-transaction [_tx]- (serdes.backfill/backfill-ids!)))- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared- ;; dependencies guide the import, and make sure all containers are imported before contents, etc.- (let [contents (serdes.ingest/ingest-list ingestion)- ingest-errors (serdes.ingest/ingest-errors ingestion)- ctx (cond-> (new-context ingestion)- (seq ingest-errors) (update :errors into ingest-errors))]- (when (and (seq ingest-errors) (not continue-on-error))- (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]- (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"- (count ingest-errors)- (str/join ", " file-names))- {:ingest-errors ingest-errors- :files file-names}- (first ingest-errors)))))- (log/infof "Starting deserialization, total %s documents" (count contents))- (reduce (fn [ctx item]- (try- (load-one! ctx item)- (catch Exception e- (when-not continue-on-error- (throw e))- ;; eschew big and scary stacktrace- (log/warnf (u/strip-error e "Skipping deserialization error"))- (update ctx :errors conj e))))- ctx- contents)))- (when reindex?- ;; Reindex after all entities are loaded. Individual entity commits may have produced stale- ;; search index entries; this ensures the index reflects the final state.- (search/reindex!)))))+ (u/prog1+ ;; Each entity is loaded in its own transaction (inside load-one!), so a deadlock or transient+ ;; failure on one entity doesn't abort the entire import. See #74412.+ (do+ (when backfill?+ (t2/with-transaction [_tx]+ (serdes.backfill/backfill-ids!)))+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared+ ;; dependencies guide the import, and make sure all containers are imported before contents, etc.+ (let [contents (serdes.ingest/ingest-list ingestion)+ ingest-errors (serdes.ingest/ingest-errors ingestion)+ ctx (cond-> (new-context ingestion)+ (seq ingest-errors) (update :errors into ingest-errors))]+ (when (and (seq ingest-errors) (not continue-on-error))+ (let [file-names (mapv #(or (:file (ex-data %)) (ex-message %)) ingest-errors)]+ (throw (ex-info (format "Failed to read %d file(s) during ingestion: %s"+ (count ingest-errors)+ (str/join ", " file-names))+ {:ingest-errors ingest-errors+ :files file-names}+ (first ingest-errors)))))+ (log/infof "Starting deserialization, total %s documents" (count contents))+ (reduce (fn [ctx item]+ (try+ (load-one! ctx item)+ (catch Exception e+ (when-not continue-on-error+ (throw e))+ ;; eschew big and scary stacktrace+ (log/warnf (u/strip-error e "Skipping deserialization error"))+ (update ctx :errors conj e))))+ ctx+ contents)))+ (when reindex?+ ;; Reindex after all entities are loaded. Individual entity commits may have produced stale+ ;; search index entries; this ensures the index reflects the final state.+ (search/reindex!))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 57358755ae0..ddfd4b25037 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -99,7 +99,6 @@ serdes/meta:model: Cardarchived_directly: falsedashboard_id: null-metabase_version: v1.54.4-SNAPSHOT (c6780bb)source_card_id: nulltype: %sdocument_id: nulldiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 191fd68fbba..71d5ca04052 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -107,8 +107,7 @@(defn- clean-entity"Removes any comparison-confounding fields, like `:created_at`."[entity]- (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule- :metabase_version))+ (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule))#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}(deftest e2e-storage-ingestion-testdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 98a8a7dcaa8..3f5f6c8daab 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -12,7 +12,6 @@[metabase-enterprise.serialization.v2.round-trip-test :as round-trip-test][metabase.actions.models :as action][metabase.audit-app.core :as audit]- [metabase.config.core :as config][metabase.core.core :as mbc][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -2847,19 +2846,14 @@(is (= #{light-eid dark-eid}(ids-by-model "EmbeddingTheme" (extract/extract {}))))))))-(deftest stamp-metabase-version-test- (testing "extract stamps :metabase_version on entities (so load can detect version mismatches)"+(deftest no-metabase-version-stamp-test+ (testing "extract does not stamp :metabase_version on entities (GHY-4013: it caused spurious remote-sync diffs)"(mt/with-empty-h2-app-db!(ts/with-temp-dpc [:model/Collection {coll-id :id} {:name "My Collection"}:model/Card _ {:name "My Card" :collection_id coll-id}](let [extracted (into [] (extract/extract {}))by-model (group-by (comp :model last :serdes/meta) extracted)]- (testing "Collections and Cards get the current Metabase version"- (doseq [m ["Collection" "Card"]- entity (get by-model m)]- (is (= config/mb-version-string (:metabase_version entity))- (str m " should be stamped with the current version"))))- (testing "Settings are not stamped — settings.yaml only persists :key and :value"- (doseq [setting (get by-model "Setting")]- (is (not (contains? setting :metabase_version))- "Setting should not be stamped"))))))))+ (doseq [m ["Collection" "Card"]+ entity (get by-model m)]+ (is (not (contains? entity :metabase_version))+ (str m " should not carry a :metabase_version"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex cca41b76fbd..aa0f7a93c9d 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -19,7 +19,6 @@[metabase.test :as mt][metabase.util :as u][metabase.util.json :as json]- [metabase.util.log.capture :as log.capture][metabase.warehouses.models.database :as models.database][toucan2.core :as t2]))@@ -1995,32 +1994,6 @@(serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))(is (= 4 (coll-count)))))))-(deftest warn-if-version-mismatch-test- (ts/with-dbs [source-db dest-db dest-db2 dest-db3]- (ts/with-db source-db- (mt/with-temp [:model/Collection _ {:name "col-1"}]- (let [extract (into [] (serdes.extract/extract {:no-settings true}))]- (ts/with-db dest-db- (testing "logs a warning when version in serdes/meta differs from current version"- (let [old-version-extract (map #(assoc % :metabase_version "v1.0.0 (oldcommit)") extract)]- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]- (serdes.load/load-metabase! (ingestion-in-memory old-version-extract))- (is (some #(str/includes? % "Version mismatch loading") (messages)))- (is (= 1 (count (filter #(str/includes? % "Version mismatch loading") (messages))))- "Should log a version mismatch warning only once per load")))))- (ts/with-db dest-db2- (testing "No warnings when version in serdes/meta matches current version"- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]- (serdes.load/load-metabase! (ingestion-in-memory extract))- (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages))))))))- (ts/with-db dest-db3- (testing "No warnings when entities have no :metabase_version (eg. legacy exports or Settings)"- (let [no-version-extract (map #(dissoc % :metabase_version) extract)]- (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]- (serdes.load/load-metabase! (ingestion-in-memory no-version-extract))- (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages))))- "Missing :metabase_version should be treated as unknown, not as a mismatch"))))))))))-(deftest import-published-table-with-existing-database-test(testing "Importing a published table works when database already exists on target"(let [serialized (atom nil)diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex 183f9f28f75..3acb93d70cd 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -54,6 +54,8 @@;; Is worth considering when adding entries here, whether they shouldn't just be skipped in extraction.#{:cache_field_values_schedule:metadata_sync_schedule+ ;; instance-specific build version, no longer serialized (GHY-4013). Legacy baseline fixtures still+ ;; carry it, which exercises that such exports still import cleanly now that it's skipped.:metabase_version;; result_metadata is non-deterministic for dashboard/document cards because the Card before-update hook;; re-computes it without :verified-result-metadata? set. Fixing this properly requires making serdesdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.cljindex 596490af388..5c6c72db672 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj@@ -42,11 +42,11 @@(testing "the Collections properly exported"(let [yaml-parent (-> (yaml/from-file (io/file dump-dir "collections" "main""some_collection.yaml"))- (dissoc :serdes/meta :metabase_version)+ (dissoc :serdes/meta)(update :created_at t/offset-date-time))yaml-child (-> (yaml/from-file (io/file dump-dir "collections" "main""some_collection" "child_collection.yaml"))- (dissoc :serdes/meta :metabase_version)+ (dissoc :serdes/meta)(update :created_at t/offset-date-time))](is (= (-> (into {} (t2/select-one :model/Collection :id (:id parent)))(dissoc :id :location)@@ -142,7 +142,6 @@"databases" "my_company_data""tables" "customers""fields" "company__SLASH__organization_website.yaml"))- (dissoc :metabase_version)(update :visibility_type keyword)(update :base_type keyword))))))))))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 1a50efd84f0..a9ed68c343e 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1369,9 +1369,12 @@(defmethod serdes/make-spec "Card"[_model-name _opts]{:copy [:archived :archived_directly :collection_position :collection_preview :description :display- :embedding_params :enable_embedding :embedding_type :entity_id :metabase_version :public_uuid :type :name+ :embedding_params :enable_embedding :embedding_type :entity_id :public_uuid :type :name:card_schema]- :skip [;; cache invalidation is instance-specific+ :skip [;; instance-specific build version; serializing it produces spurious remote-sync diffs, and the+ ;; serialized representation is versioned by :card_schema instead+ :metabase_version+ ;; cache invalidation is instance-specific:cache_invalidated_at;; those are instance-specific analytic columns:view_count :last_used_at :initially_published_at