Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add warning for when mismatch in serdes version during loading (#66140)

Nathan Voxlandf2e8e294f82b
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index dae0b12bb74..0d70159f6ec 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2175,6 +2175,7 @@
app-db
appearance
collections
+ config
logger
models
plugins
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 17659a679a6..58873fde21c 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -7,6 +7,7 @@
[metabase-enterprise.serialization.v2.backfill-ids :as serdes.backfill]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase-enterprise.serialization.v2.models :as serdes.models]
+ [metabase.config.core :as config]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.util :as u]
@@ -28,6 +29,10 @@
"Document" #{:document}
"Card" #{:dashboard_id :document_id}})
+(def ^:private ^:dynamic *warned-version-mismatch*
+ "Used to avoid double-logging on version mismatches. Warns if nil or set to true"
+ nil)
+
(defn- keys-to-strip [ingested]
(let [model (-> ingested :serdes/meta last :model)]
(or (model->circular-dependency-keys model)
@@ -41,21 +46,22 @@
If [[load-one]] throws because it can't find that entity in the filesystem, check if it's already loaded in
our database."
[ctx deps]
- (if (empty? deps)
- ctx
- (letfn [(loader [ctx dep]
- (try
- (load-one! ctx dep)
- (catch Exception e
- (cond
- ;; It was missing, but we found it locally, so just return the context.
- (and (= (:error (ex-data e)) ::not-found)
- (serdes/load-find-local dep))
- ctx
-
- :else
- (throw e)))))]
- (reduce loader ctx deps))))
+ (binding [*warned-version-mismatch* (if (nil? *warned-version-mismatch*) (atom false) *warned-version-mismatch*)]
+ (if (empty? deps)
+ ctx
+ (letfn [(loader [ctx dep]
+ (try
+ (load-one! ctx dep)
+ (catch Exception e
+ (cond
+ ;; It was missing, but we found it locally, so just return the context.
+ (and (= (:error (ex-data e)) ::not-found)
+ (serdes/load-find-local dep))
+ ctx
+
+ :else
+ (throw e)))))]
+ (reduce loader ctx deps)))))
(defn- path-error-data [error-type expanding path]
(let [last-model (:model (last path))]
@@ -81,6 +87,21 @@
(let [model (t2.model/resolve-model (symbol model-name))]
(serdes.backfill/has-entity-id? model))))
+(defn- warn-if-version-mismatch
+ "Checks if the version in the exported entity's serdes/meta differs from the current Metabase version.
+ Logs a warning if there is a mismatch."
+ [ingested path]
+ (when (or (nil? *warned-version-mismatch*) (not @*warned-version-mismatch*))
+ (let [current-version config/mb-version-string
+ exported-version (or (:metabase_version ingested) "UNKNOWN")]
+ (when (not= exported-version current-version)
+ (log/warnf "Version mismatch loading %s: exported with: %s, current version: %s"
+ path
+ exported-version
+ current-version)
+ (when (not (nil? *warned-version-mismatch*))
+ (reset! *warned-version-mismatch* true))))))
+
(defn- load-one!
"Loads a single entity, specified by its `:serdes/meta` abstract path, into the appdb, doing some bookkeeping to
avoid cycles.
@@ -149,6 +170,7 @@
:level (count expanding)}))
local-or-nil (when-not require-new-entity (serdes/load-find-local rebuilt-path))]
(try
+ (warn-if-version-mismatch ingested path)
(serdes/load-one! ingested local-or-nil)
ctx
(catch Exception e
@@ -180,29 +202,30 @@
:or {backfill? true
continue-on-error false
reindex? true}}]
- (u/prog1
- (t2/with-transaction [_tx]
- ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies
- ;; guide the import, and make sure all containers are imported before contents, etc.
- (when backfill?
- (serdes.backfill/backfill-ids!))
- (let [contents (serdes.ingest/ingest-list ingestion)
- ctx (new-context ingestion)]
- (log/infof "Starting deserialization, total %s documents" (count contents))
- (reduce (fn [ctx item]
- (try
- (load-one! ctx item)
- (catch Exception e
- (when-not continue-on-error
- (throw e))
- ;; eschew big and scary stacktrace
- (log/warnf (u/strip-error e "Skipping deserialization error"))
- (update ctx :errors conj e))))
- ctx
- contents)))
- (when reindex?
- ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.
- ;; this means that the corresponding entries would have been missing or stale when we indexed them.
- ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.
- ;; while we're figuring that out, here's a crude stopgap.
- (search/reindex!))))
+ (binding [*warned-version-mismatch* (atom false)]
+ (u/prog1
+ (t2/with-transaction [_tx]
+ ;; We proceed in the arbitrary order of ingest-list, deserializing all the files. Their declared dependencies
+ ;; guide the import, and make sure all containers are imported before contents, etc.
+ (when backfill?
+ (serdes.backfill/backfill-ids!))
+ (let [contents (serdes.ingest/ingest-list ingestion)
+ ctx (new-context ingestion)]
+ (log/infof "Starting deserialization, total %s documents" (count contents))
+ (reduce (fn [ctx item]
+ (try
+ (load-one! ctx item)
+ (catch Exception e
+ (when-not continue-on-error
+ (throw e))
+ ;; eschew big and scary stacktrace
+ (log/warnf (u/strip-error e "Skipping deserialization error"))
+ (update ctx :errors conj e))))
+ ctx
+ contents)))
+ (when reindex?
+ ;; Hack: the transaction above typically takes much longer than our delay on the search indexing queue.
+ ;; this means that the corresponding entries would have been missing or stale when we indexed them.
+ ;; ideally, we would delay the indexing somehow, or only reindex what we've loaded.
+ ;; while we're figuring that out, here's a crude stopgap.
+ (search/reindex!)))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index cba7019d8f6..5b857a560e0 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -14,6 +14,7 @@
[metabase.test :as mt]
[metabase.util :as u]
[metabase.util.json :as json]
+ [metabase.util.log.capture :as log.capture]
[toucan2.core :as t2]))
;; `reindex!` below is ok in a parallel test since it's not actually executing anything
@@ -1291,7 +1292,7 @@
(vec (for [entity ser]
(merge entity (get changes (:entity_id entity))))))
logs-extract (fn [re logs]
- (keep #(rest (re-find re %))
+ (keep #(not-empty (rest (re-find re %)))
(map :message logs)))]
(mt/with-empty-h2-app-db!
(mt/with-temp [:model/Collection coll {:name "coll"}
@@ -1480,3 +1481,22 @@
(testing "absent :entity_id also works"
(serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))
(is (= 4 (coll-count)))))))
+
+(deftest warn-if-version-mismatch-test
+ (ts/with-dbs [source-db dest-db dest-db2]
+ (ts/with-db source-db
+ (mt/with-temp [:model/Collection _ {:name "col-1"}]
+ (let [extract (into [] (serdes.extract/extract {:no-settings true}))]
+ (ts/with-db dest-db
+ (testing "logs a warning when version in serdes/meta differs from current version"
+ (let [old-version-extract (map #(assoc % :metabase_version "v1.0.0 (oldcommit)") extract)]
+ (log.capture/with-log-messages-for-level [messages [metabase-enterprise.serialization.v2.load :warn]]
+ (serdes.load/load-metabase! (ingestion-in-memory old-version-extract))
+ (is (some #(str/includes? % "Version mismatch loading") (messages)))
+ (is (= 1 (count (filter #(str/includes? % "Version mismatch loading") (messages))))
+ "Should log a version mismatch warning only once per load")))))
+ (ts/with-db dest-db2
+ (testing "No warnings when version in serdes/meta matches current version"
+ (log.capture/with-log-messages-for-level [messages :warn]
+ (serdes.load/load-metabase! (ingestion-in-memory extract))
+ (is (= 0 (count (filter #(str/includes? % "Version mismatch loading") (messages)))))))))))))