The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fail serialization export when cards escape requested collections (#75176) (#76741)
Eric Normand008bf29ce04f
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 120114582d6..793b2859128 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -134,7 +134,13 @@:id id}))))):analytics-card-ids escaped-in-analytics})))))-(defn- log-escape-report! [escaped]+(defn- handle-escapees!+ "Reports cards that are referenced from inside the requested collections but saved outside them.++ Logs a per-entity warning for each escapee.++ Throws if `continue-on-error?` is false."+ [escaped continue-on-error?](let [dashboards (group-by #(get % "Dashboard") escaped)](doseq [[dash-id escapes] (dissoc dashboards nil)](log/warnf "Failed to export Dashboard %d (%s) containing Card saved outside requested collections: %s"@@ -148,7 +154,14 @@(if (get item "Card")(entity-label {:model :model/Card :id (get item "Card")})(dissoc item :escapee))- (entity-label (:escapee item)))))))))+ (entity-label (:escapee item))))))))+ (when-not continue-on-error?+ (throw (ex-info (format (str "Serialization failed: %d card(s) referenced by the requested collections "+ "are saved outside them, which would produce an incomplete export. See the "+ "warnings above for the affected entities. Pass continue-on-error to export "+ "anyway, skipping the affected dashboards and cards.")+ (count (distinct (map (comp :id :escapee) escaped))))+ {:status-code 400}))))(defn- resolve-targets"Returns all targets (for either supplied initial `targets` or for supplied `user-id`)."@@ -188,37 +201,33 @@;; cards referenced by dashboards live outside the target collections.{:keys [reportable-escaped analytics-card-ids]} (when has-content?(escape-analysis by-model nodes))]+ ;; A card referenced from inside the requested collections but living outside them would produce an incomplete+ ;; export.(when (seq reportable-escaped)- (log-escape-report! reportable-escaped))- ;; By default a card referenced from inside the requested collections but living outside them aborts- ;; the whole export. With continue-on-error we don't abort: the escaped cards are outside the- ;; collection set so they're filtered out of extraction anyway, and the dashboards/cards that- ;; reference them are skipped on import (which also honors continue-on-error).- (when (or (empty? reportable-escaped)- (:continue-on-error opts))- (let [coll-set (get by-model "Collection")- ;; When targets are specified, also include Tables found via descendants- ;; (published tables in target collections). These are extracted by ID, not all.- targeted-data-model (when (seq targets)- (select-keys by-model serdes.models/data-model-in-collection))- by-model (cond-> (select-keys by-model models)- ;; Add Tables back if they were found in descendants- (seq targeted-data-model) (merge targeted-data-model)- ;; Remove analytics cards from extraction - they have stable entity_ids across instances- ;; so cards that reference them can still be exported and imported correctly- (and analytics-card-ids (contains? by-model "Card"))- (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))- extract-by-ids (fn [[model ids]]- (serdes/extract-all model (merge opts {:collection-set coll-set- :where [:in :id ids]})))- extract-all (fn [model]- (serdes/extract-all model (assoc opts :collection-set coll-set)))]- (eduction cat- [(if (seq targets)- (eduction (map extract-by-ids) cat by-model)- (eduction (map extract-all) cat (set/intersection (set serdes.models/content) models)))- ;; extract all non-content entities like data model and settings if necessary- (eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))])))))+ (handle-escapees! reportable-escaped (:continue-on-error opts))) ;; may throw+ (let [coll-set (get by-model "Collection")+ ;; When targets are specified, also include Tables found via descendants+ ;; (published tables in target collections). These are extracted by ID, not all.+ targeted-data-model (when (seq targets)+ (select-keys by-model serdes.models/data-model-in-collection))+ by-model (cond-> (select-keys by-model models)+ ;; Add Tables back if they were found in descendants+ (seq targeted-data-model) (merge targeted-data-model)+ ;; Remove analytics cards from extraction - they have stable entity_ids across instances+ ;; so cards that reference them can still be exported and imported correctly+ (and analytics-card-ids (contains? by-model "Card"))+ (update "Card" (fn [ids] (vec (remove analytics-card-ids ids)))))+ extract-by-ids (fn [[model ids]]+ (serdes/extract-all model (merge opts {:collection-set coll-set+ :where [:in :id ids]})))+ extract-all (fn [model]+ (serdes/extract-all model (assoc opts :collection-set coll-set)))]+ (eduction cat+ [(if (seq targets)+ (eduction (map extract-by-ids) cat by-model)+ (eduction (map extract-all) cat (set/intersection (set serdes.models/content) models)))+ ;; extract all non-content entities like data model and settings if necessary+ (eduction (map #(serdes/extract-all % opts)) cat (remove (set serdes.models/content) models))]))))(defn- needs-version?"True for extracted entities that should carry a `:metabase_version` stamp."@@ -243,4 +252,5 @@(u/traverse colls #(serdes/ascendants (first %) (second %)))(u/traverse colls #(serdes/descendants (first %) (second %) {})))))(def escaped (escape-analysis (u/group-by first second (keys nodes)) nodes))- (log-escape-report! escaped))+ ;; continue-on-error? true so this just logs in the REPL instead of throwing+ (handle-escapees! (:reportable-escaped escaped) true))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 72bfb6e8eaf..b7d5e225c67 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -241,8 +241,9 @@(deftest export-log-captures-extract-warnings-test(testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)";; A dashboard in the exported collection references a card living in a different collection. Escape analysis- ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. That warning- ;; must still land in export.log even though extract happens outside the storage logging block.+ ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. Under+ ;; continue-on-error the export still completes, and that warning must land in export.log even though extract+ ;; happens outside the storage logging block.(mt/with-premium-features #{:serialization}(mt/with-temp [:model/Collection target {:name "Target Collection"}:model/Collection other {:name "Other Collection"}@@ -251,12 +252,31 @@:model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}](let [res (binding [api.serialization/*additive-logging* false](mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}- :collection (:id target) :data_model false :settings false))+ :collection (:id target) :data_model false :settings false+ :continue_on_error true))log (read-export-log res)](is (some? log) "export.log should be present in the archive")(is (re-find #"outside requested collections" log)"export.log should contain the escape-analysis warning emitted during extract"))))))+(deftest export-aborts-on-escaped-card-test+ (testing "Export fails loudly with a 4xx instead of silently emitting an empty archive when a referenced card lives outside the requested collections (#75176)"+ (mt/with-premium-features #{:serialization}+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]+ (mt/with-temp [:model/Collection target {:name "Target Collection"}+ :model/Collection other {:name "Other Collection"}+ :model/Card outside-card {:collection_id (:id other) :name "OutsideCard"}+ :model/Dashboard dash {:collection_id (:id target) :name "DashWithOutsideCard"}+ :model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}]+ (let [body (binding [api.serialization/*additive-logging* false]+ (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}+ :collection (:id target) :data_model false :settings false))]+ (is (re-find #"incomplete export" (str body))+ "the 4xx body explains that the export would be incomplete")+ (is (empty? (filter #(str/starts-with? (str (:message %)) "Error during serialization export")+ (messages)))+ "the abort surfaces as a client-side 4xx, not logged as a server error")))))))+(deftest import-restores-entities-test(testing "Import restores deleted/renamed entities and updates search index"(with-serialization-test-data! [coll dash card]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex e55590d16d9..98a8a7dcaa8 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -44,6 +44,13 @@(map (comp :id last :serdes/meta))set))+(defn- extract-aborts!+ "Realize `(extract/extract opts)`, asserting it aborts with the escape-analysis error (#75176).+ Escape warnings are logged before the abort, so callers can still assert on the logged messages."+ [opts]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"incomplete export"+ (into [] (extract/extract opts)))))+(deftest fundamentals-test(mt/with-empty-h2-app-db!(ts/with-temp-dpc [:model/Collection@@ -1455,7 +1462,7 @@set))))(testing "depending on data from personal collections results in errors"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract/extract {:targets [["Collection" coll4-id]] :no-settings true :no-data-model true :no-transforms true})+ (extract-aborts! {:targets [["Collection" coll4-id]] :no-settings true :no-data-model true :no-transforms true})(let [msgs (into #{}(map :message)(messages))]@@ -1601,7 +1608,7 @@:values_source_config {:card_id card1-id}}]}](testing "Complain about card not available for exporting"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract/extract {:targets [["Collection" coll1-id]]+ (extract-aborts! {:targets [["Collection" coll1-id]]:no-settings true:no-data-model true})(is (some #(str/starts-with? % "Failed to export Dashboard")@@ -1611,7 +1618,7 @@(testing "Complain about card depending on an outside card: "(testing "when its :source-table"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract/extract {:targets [["Collection" coll2-id]]+ (extract-aborts! {:targets [["Collection" coll2-id]]:no-settings true:no-data-model true})(is (some #(str/starts-with? % "Failed to export Cards")@@ -1620,7 +1627,7 @@(messages))))))(testing "when it's :parameters"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract/extract {:targets [["Collection" coll2-id]]+ (extract-aborts! {:targets [["Collection" coll2-id]]:no-settings true:no-data-model true})(is (some #(str/starts-with? % "Failed to export Cards")@@ -1630,7 +1637,7 @@(testing "When exporting all collections"(testing "Complain about dependents in personal collections"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (extract/extract {:no-settings true+ (extract-aborts! {:no-settings true:no-data-model true})(is (some #(str/starts-with? % "Failed to export Cards")(into #{}@@ -1667,8 +1674,8 @@escaped-id :id} {:name "Escaped Card" :database_id db-id}:model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}](let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]- (testing "without the flag, escape analysis aborts the whole export"- (is (empty? (into [] (extract/extract opts)))))+ (testing "without the flag, escape analysis aborts the whole export with an error (#75176)"+ (extract-aborts! opts))(testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]](let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]