Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fail serialization export when cards escape requested collections (#75176) (#76741)

Eric Normand008bf29ce04f
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 120114582d6..793b2859128 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -134,7 +134,13 @@
:id id})))))
:analytics-card-ids escaped-in-analytics})))))
-(defn- log-escape-report! [escaped]
+(defn- handle-escapees!
+ "Reports cards that are referenced from inside the requested collections but saved outside them.
+
+ Logs a per-entity warning for each escapee.
+
+ Throws if `continue-on-error?` is false."
+ [escaped continue-on-error?]
(let [dashboards (group-by #(get % "Dashboard") escaped)]
(doseq [[dash-id escapes] (dissoc dashboards nil)]
(log/warnf "Failed to export Dashboard %d (%s) containing Card saved outside requested collections: %s"
@@ -148,7 +154,14 @@
(if (get item "Card")
(entity-label {:model :model/Card :id (get item "Card")})
(dissoc item :escapee))
- (entity-label (:escapee item)))))))))
+ (entity-label (:escapee item))))))))
+ (when-not continue-on-error?
+ (throw (ex-info (format (str "Serialization failed: %d card(s) referenced by the requested collections "
+ "are saved outside them, which would produce an incomplete export. See the "
+ "warnings above for the affected entities. Pass continue-on-error to export "
+ "anyway, skipping the affected dashboards and cards.")
+ (count (distinct (map (comp :id :escapee) escaped))))
+ {:status-code 400}))))
(defn- resolve-targets
"Returns all targets (for either supplied initial `targets` or for supplied `user-id`)."
@@ -188,37 +201,33 @@
;; cards referenced by dashboards live outside the target collections.
{:keys [reportable-escaped analytics-card-ids]} (when has-content?
(escape-analysis by-model nodes))]
+ ;; A card referenced from inside the requested collections but living outside them would produce an incomplete
+ ;; export.
(when (seq reportable-escaped)
- (log-escape-report! reportable-escaped))
- ;; By default a card referenced from inside the requested collections but living outside them aborts
- ;; the whole export. With continue-on-error we don't abort: the escaped cards are outside the
- ;; collection set so they're filtered out of extraction anyway, and the dashboards/cards that
- ;; reference them are skipped on import (which also honors continue-on-error).
- (when (or (empty? reportable-escaped)
- (:continue-on-error opts))
- (let [coll-set (get by-model "Collection")
- ;; When targets are specified, also include Tables found via descendants
- ;; (published tables in target collections). These are extracted by ID, not all.
- targeted-data-model (when (seq targets)
- (select-keys by-model serdes.models/data-model-in-collection))
- by-model (cond-> (select-keys by-model models)
- ;; Add Tables back if they were found in descendants
- (seq targeted-data-model) (merge targeted-data-model)
- ;; Remove analytics cards from extraction - they have stable entity_ids across instances
- ;; so cards that reference them can still be exported and imported correctly
- (and analytics-card-ids (contains? by-model "Card"))
- (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))
- extract-by-ids (fn [[model ids]]
- (serdes/extract-all model (merge opts {:collection-set coll-set
- :where [:in :id ids]})))
- extract-all (fn [model]
- (serdes/extract-all model (assoc opts :collection-set coll-set)))]
- (eduction cat
- [(if (seq targets)
- (eduction (map extract-by-ids) cat by-model)
- (eduction (map extract-all) cat (set/intersection (set serdes.models/content) models)))
- ;; extract all non-content entities like data model and settings if necessary
- (eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))])))))
+ (handle-escapees! reportable-escaped (:continue-on-error opts))) ;; may throw
+ (let [coll-set (get by-model "Collection")
+ ;; When targets are specified, also include Tables found via descendants
+ ;; (published tables in target collections). These are extracted by ID, not all.
+ targeted-data-model (when (seq targets)
+ (select-keys by-model serdes.models/data-model-in-collection))
+ by-model (cond-> (select-keys by-model models)
+ ;; Add Tables back if they were found in descendants
+ (seq targeted-data-model) (merge targeted-data-model)
+ ;; Remove analytics cards from extraction - they have stable entity_ids across instances
+ ;; so cards that reference them can still be exported and imported correctly
+ (and analytics-card-ids (contains? by-model "Card"))
+ (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))
+ extract-by-ids (fn [[model ids]]
+ (serdes/extract-all model (merge opts {:collection-set coll-set
+ :where [:in :id ids]})))
+ extract-all (fn [model]
+ (serdes/extract-all model (assoc opts :collection-set coll-set)))]
+ (eduction cat
+ [(if (seq targets)
+ (eduction (map extract-by-ids) cat by-model)
+ (eduction (map extract-all) cat (set/intersection (set serdes.models/content) models)))
+ ;; extract all non-content entities like data model and settings if necessary
+ (eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))]))))
(defn- needs-version?
"True for extracted entities that should carry a `:metabase_version` stamp."
@@ -243,4 +252,5 @@
(u/traverse colls #(serdes/ascendants (first %) (second %)))
(u/traverse colls #(serdes/descendants (first %) (second %) {})))))
(def escaped (escape-analysis (u/group-by first second (keys nodes)) nodes))
- (log-escape-report! escaped))
+ ;; continue-on-error? true so this just logs in the REPL instead of throwing
+ (handle-escapees! (:reportable-escaped escaped) true))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 72bfb6e8eaf..b7d5e225c67 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -241,8 +241,9 @@
(deftest export-log-captures-extract-warnings-test
(testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)"
;; A dashboard in the exported collection references a card living in a different collection. Escape analysis
- ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. That warning
- ;; must still land in export.log even though extract happens outside the storage logging block.
+ ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. Under
+ ;; continue-on-error the export still completes, and that warning must land in export.log even though extract
+ ;; happens outside the storage logging block.
(mt/with-premium-features #{:serialization}
(mt/with-temp [:model/Collection target {:name "Target Collection"}
:model/Collection other {:name "Other Collection"}
@@ -251,12 +252,31 @@
:model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}]
(let [res (binding [api.serialization/*additive-logging* false]
(mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
- :collection (:id target) :data_model false :settings false))
+ :collection (:id target) :data_model false :settings false
+ :continue_on_error true))
log (read-export-log res)]
(is (some? log) "export.log should be present in the archive")
(is (re-find #"outside requested collections" log)
"export.log should contain the escape-analysis warning emitted during extract"))))))
+(deftest export-aborts-on-escaped-card-test
+ (testing "Export fails loudly with a 4xx instead of silently emitting an empty archive when a referenced card lives outside the requested collections (#75176)"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]
+ (mt/with-temp [:model/Collection target {:name "Target Collection"}
+ :model/Collection other {:name "Other Collection"}
+ :model/Card outside-card {:collection_id (:id other) :name "OutsideCard"}
+ :model/Dashboard dash {:collection_id (:id target) :name "DashWithOutsideCard"}
+ :model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}]
+ (let [body (binding [api.serialization/*additive-logging* false]
+ (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}
+ :collection (:id target) :data_model false :settings false))]
+ (is (re-find #"incomplete export" (str body))
+ "the 4xx body explains that the export would be incomplete")
+ (is (empty? (filter #(str/starts-with? (str (:message %)) "Error during serialization export")
+ (messages)))
+ "the abort surfaces as a client-side 4xx, not logged as a server error")))))))
+
(deftest import-restores-entities-test
(testing "Import restores deleted/renamed entities and updates search index"
(with-serialization-test-data! [coll dash card]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index e55590d16d9..98a8a7dcaa8 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -44,6 +44,13 @@
(map (comp :id last :serdes/meta))
set))
+(defn- extract-aborts!
+ "Realize `(extract/extract opts)`, asserting it aborts with the escape-analysis error (#75176).
+ Escape warnings are logged before the abort, so callers can still assert on the logged messages."
+ [opts]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"incomplete export"
+ (into [] (extract/extract opts)))))
+
(deftest fundamentals-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/Collection
@@ -1455,7 +1462,7 @@
set))))
(testing "depending on data from personal collections results in errors"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract/extract {:targets [["Collection" coll4-id]] :no-settings true :no-data-model true :no-transforms true})
+ (extract-aborts! {:targets [["Collection" coll4-id]] :no-settings true :no-data-model true :no-transforms true})
(let [msgs (into #{}
(map :message)
(messages))]
@@ -1601,7 +1608,7 @@
:values_source_config {:card_id card1-id}}]}]
(testing "Complain about card not available for exporting"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract/extract {:targets [["Collection" coll1-id]]
+ (extract-aborts! {:targets [["Collection" coll1-id]]
:no-settings true
:no-data-model true})
(is (some #(str/starts-with? % "Failed to export Dashboard")
@@ -1611,7 +1618,7 @@
(testing "Complain about card depending on an outside card: "
(testing "when its :source-table"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract/extract {:targets [["Collection" coll2-id]]
+ (extract-aborts! {:targets [["Collection" coll2-id]]
:no-settings true
:no-data-model true})
(is (some #(str/starts-with? % "Failed to export Cards")
@@ -1620,7 +1627,7 @@
(messages))))))
(testing "when it's :parameters"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract/extract {:targets [["Collection" coll2-id]]
+ (extract-aborts! {:targets [["Collection" coll2-id]]
:no-settings true
:no-data-model true})
(is (some #(str/starts-with? % "Failed to export Cards")
@@ -1630,7 +1637,7 @@
(testing "When exporting all collections"
(testing "Complain about dependents in personal collections"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (extract/extract {:no-settings true
+ (extract-aborts! {:no-settings true
:no-data-model true})
(is (some #(str/starts-with? % "Failed to export Cards")
(into #{}
@@ -1667,8 +1674,8 @@
escaped-id :id} {:name "Escaped Card" :database_id db-id}
:model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]
(let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]
- (testing "without the flag, escape analysis aborts the whole export"
- (is (empty? (into [] (extract/extract opts)))))
+ (testing "without the flag, escape analysis aborts the whole export with an error (#75176)"
+ (extract-aborts! opts))
(testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
(let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]