The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add warning for when mismatch in serdes version during loading (#66140)
Nathan Voxlandf2e8e294f82b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex dae0b12bb74..0d70159f6ec 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2175,6 +2175,7 @@app-dbappearancecollections+ configloggermodelspluginsdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 17659a679a6..58873fde21c 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -7,6 +7,7 @@[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill][metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.models :as serdes.models]+ [metabase.config.core :as config][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.util :as u]@@ -28,6 +29,10 @@"Document" #{:document}"Card" #{:dashboard_id :document_id}})+(def ^:private ^:dynamic *warned-version-mismatch*+ "Used to avoid double-logging on version mismatches. Warns if nil or set to true"+ nil)+(defn- keys-to-strip [ingested](let [model (-> ingested :serdes/meta last :model)](or (model->circular-dependency-keys model)@@ -41,21 +46,22 @@If [[load-one]] throws because it can't find that entity in the filesystem, check if it's already loaded inour database."[ctx deps]- (if (empty? deps)- ctx- (letfn [(loader [ctx dep]- (try- (load-one! ctx dep)- (catch Exception e- (cond- ;; It was missing, but we found it locally, so just return the context.- (and (= (:error (ex-data e)) ::not-found)- (serdes/load-find-local dep))- ctx-- :else- (throw e)))))]- (reduce loader ctx deps))))+ (binding [*warned-version-mismatch* (if (nil? *warned-version-mismatch*) (atom false) *warned-version-mismatch*)]+ (if (empty? deps)+ ctx+ (letfn [(loader [ctx dep]+ (try+ (load-one! ctx dep)+ (catch Exception e+ (cond+ ;; It was missing, but we found it locally, so just return the context.+ (and (= (:error (ex-data e)) ::not-found)+ (serdes/load-find-local dep))+ ctx++ :else+ (throw e)))))]+ (reduce loader ctx deps)))))(defn- path-error-data [error-type expanding path](let [last-model (:model (last path))]@@ -81,6 +87,21 @@(let [model (t2.model/resolve-model (symbol model-name))](serdes.backfill/has-entity-id? model))))+(defn- warn-if-version-mismatch+ "Checks if the version in the exported entity's serdes/meta differs from the current Metabase version.+ Logs a warning if there is a mismatch."+ [ingested path]+ (when (or (nil? *warned-version-mismatch*) (not @*warned-version-mismatch*))+ (let [current-version config/mb-version-string+ exported-version (or (:metabase_version ingested) "UNKNOWN")]+ (when (not= exported-version current-version)+ (log/warnf "Version mismatch loading %s: exported with: %s, current version: %s"+ path+ exported-version+ current-version)+ (when (not (nil? *warned-version-mismatch*))+ (reset! *warned-version-mismatch* true))))))+(defn- load-one!"Loads a single entity, specified by its `:serdes/meta` abstract path, into the appdb, doing some bookkeeping toavoid cycles.@@ -149,6 +170,7 @@:level (count expanding)}))local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))](try+ (warn-if-version-mismatch ingested path)(serdes/load-one! ingested local-or-nil)ctx(catch Exception e@@ -180,29 +202,30 @@:or {backfill? truecontinue-on-error falsereindex? true}}]- (u/prog1- (t2/with-transaction [_tx]- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies- ;; guide the import, and make sure all containers are imported before contents, etc.- (when backfill?- (serdes.backfill/backfill-ids!))- (let [contents (serdes.ingest/ingest-list ingestion)- ctx (new-context ingestion)]- (log/infof "Starting deserialization, total %s documents" (count contents))- (reduce (fn [ctx item]- (try- (load-one! ctx item)- (catch Exception e- (when-not continue-on-error- (throw e))- ;; eschew big and scary stacktrace- (log/warnf (u/strip-error e "Skipping deserialization error"))- (update ctx :errors conj e))))- ctx- contents)))- (when reindex?- ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.- ;; this means that the corresponding entries would have been missing or stale when we indexed them.- ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.- ;; while we're figuring that out, here's a crude stopgap.- (search/reindex!))))+ (binding [*warned-version-mismatch* (atom false)]+ (u/prog1+ (t2/with-transaction [_tx]+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies+ ;; guide the import, and make sure all containers are imported before contents, etc.+ (when backfill?+ (serdes.backfill/backfill-ids!))+ (let [contents (serdes.ingest/ingest-list ingestion)+ ctx (new-context ingestion)]+ (log/infof "Starting deserialization, total %s documents" (count contents))+ (reduce (fn [ctx item]+ (try+ (load-one! ctx item)+ (catch Exception e+ (when-not continue-on-error+ (throw e))+ ;; eschew big and scary stacktrace+ (log/warnf (u/strip-error e "Skipping deserialization error"))+ (update ctx :errors conj e))))+ ctx+ contents)))+ (when reindex?+ ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.+ ;; this means that the corresponding entries would have been missing or stale when we indexed them.+ ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.+ ;; while we're figuring that out, here's a crude stopgap.+ (search/reindex!)))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex cba7019d8f6..5b857a560e0 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -14,6 +14,7 @@[metabase.test :as mt][metabase.util :as u][metabase.util.json :as json]+ [metabase.util.log.capture :as log.capture][toucan2.core :as t2]));; `reindex!` below is ok in a parallel test since it's not actually executing anything@@ -1291,7 +1292,7 @@(vec (for [entity ser](merge entity (get changes (:entity_id entity))))))logs-extract (fn [re logs]- (keep #(rest (re-find re %))+ (keep #(not-empty (rest (re-find re %)))(map :message logs)))](mt/with-empty-h2-app-db!(mt/with-temp [:model/Collection coll {:name "coll"}@@ -1480,3 +1481,22 @@(testing "absent :entity_id also works"(serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))(is (= 4 (coll-count)))))))++(deftest warn-if-version-mismatch-test+ (ts/with-dbs [source-db dest-db dest-db2]+ (ts/with-db source-db+ (mt/with-temp [:model/Collection _ {:name "col-1"}]+ (let [extract (into [] (serdes.extract/extract {:no-settings true}))]+ (ts/with-db dest-db+ (testing "logs a warning when version in serdes/meta differs from current version"+ (let [old-version-extract (map #(assoc % :metabase_version "v1.0.0 (oldcommit)") extract)]+ (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]+ (serdes.load/load-metabase! (ingestion-in-memory old-version-extract))+ (is (some #(str/includes? % "Version mismatch loading") (messages)))+ (is (= 1 (count (filter #(str/includes? % "Version mismatch loading") (messages))))+ "Should log a version mismatch warning only once per load")))))+ (ts/with-db dest-db2+ (testing "No warnings when version in serdes/meta matches current version"+ (log.capture/with-log-messages-for-level [messages :warn]+ (serdes.load/load-metabase! (ingestion-in-memory extract))+ (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages)))))))))))))