Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Honor continue_on_error and full_stacktrace on serialization export (#75122)

Ngoc Khuataf6ea737a3e4
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 805face4859..bf70eb023e0 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -79,6 +79,30 @@
['metabase-enterprise.serialization
'metabase.models.serialization])
+(defn- log-export-error!
+ "Log a serialization export error, honoring `full-stacktrace` (full trace vs stripped one-liner)."
+ [e full-stacktrace]
+ (if full-stacktrace
+ (log/error e "Error during serialization export")
+ (log/error (u/strip-error e "Error during serialization export"))))
+
+(defn- extract-entities!
+ "Run eager extraction (target resolution, escape analysis) before streaming starts. It must run
+ before the streaming response opens so a failure can still set a non-200 status. Eager logs (e.g.
+ escape-analysis warnings) are captured into `log-output` so they land in export.log alongside the
+ storage logs captured later. `full-stacktrace` is honored for genuine server failures the way the
+ streaming path and the CLI export do; client input errors carry a `:status-code` and pass through to
+ the API layer unlogged, so they surface as a clean 4xx rather than a logged server error."
+ [opts ^ByteArrayOutputStream log-output full-stacktrace]
+ (try
+ (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
+ {:additive *additive-logging*})]
+ (extract/extract opts))
+ (catch Exception e
+ (when-not (:status-code (ex-data e))
+ (log-export-error! e full-stacktrace))
+ (throw e))))
+
(defn- serialize-to-stream!
"Serialize directly to an OutputStream as streaming tar.gz. Returns result map.
@@ -94,9 +118,7 @@
(v2.storage/store! entities writer))
(catch Exception e
(reset! error e)
- (if full-stacktrace
- (log/error e "Error during serialization export")
- (log/error (u/strip-error e "Error during serialization export")))
+ (log-export-error! e full-stacktrace)
nil)))]
;; Read the buffer and write the log after the appender has closed (and thus flushed) so nothing is lost.
(try
@@ -238,14 +260,11 @@
(format "%s-%s"
(u/slugify (appearance/site-name))
(u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))
- ;; extract/extract runs eager setup (target resolution, escape analysis) which can throw
- ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts so the error
- ;; can set a non-200 status. Its eager logs (e.g. escape-analysis warnings) are captured into `log-output`
- ;; so they end up in export.log alongside the storage logs captured later.
+ ;; Eager setup (target resolution, escape analysis) must run before the streaming response opens
+ ;; so a failure can still set a non-200 status. extract-entities! captures its logs into log-output
+ ;; (so escape-analysis warnings land in export.log) and honors full_stacktrace for genuine failures.
log-output (ByteArrayOutputStream.)
- entities (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
- {:additive *additive-logging*})]
- (extract/extract opts))]
+ entities (extract-entities! opts log-output full-stacktrace?)]
(sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan]
(sr/set-header! "Content-Disposition"
(format "attachment; filename=\"%s.tar.gz\"" export-dirname))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 0b00603a1d8..700fcdd57a0 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -185,8 +185,14 @@
;; cards referenced by dashboards live outside the target collections.
{:keys [reportable-escaped analytics-card-ids]} (when has-content?
(escape-analysis by-model nodes))]
- (if (seq reportable-escaped)
- (log-escape-report! reportable-escaped)
+ (when (seq reportable-escaped)
+ (log-escape-report! reportable-escaped))
+ ;; By default a card referenced from inside the requested collections but living outside them aborts
+ ;; the whole export. With continue-on-error we don't abort: the escaped cards are outside the
+ ;; collection set so they're filtered out of extraction anyway, and the dashboards/cards that
+ ;; reference them are skipped on import (which also honors continue-on-error).
+ (when (or (empty? reportable-escaped)
+ (:continue-on-error opts))
(let [coll-set (get by-model "Collection")
;; When targets are specified, also include Tables found via descendants
;; (published tables in target collections). These are extracted by ID, not all.
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 2e19c1de4f2..72bfb6e8eaf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -7,6 +7,7 @@
[medley.core :as m]
[metabase-enterprise.serialization.api :as api.serialization]
[metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]
+ [metabase-enterprise.serialization.v2.extract :as v2.extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase.analytics.snowplow-test :as snowplow-test]
[metabase.models.serialization :as serdes]
@@ -431,6 +432,40 @@
"error_message" nil}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))
+(deftest export-eager-error-honors-full-stacktrace-test
+ (testing "a server-side failure during eager extraction setup (before streaming) honors full_stacktrace (GDGT-2491)"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Collection {coll-id :id} {}]
+ (mt/with-dynamic-fn-redefs [v2.extract/extract (fn [& _] (throw (ex-info "deliberate eager failure" {})))]
+ (testing "stripped one-liner by default"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]
+ (mt/user-http-request :crowberto :post 500 "ee/serialization/export"
+ :collection coll-id :data_model false :settings false)
+ (let [errs (filter #(str/starts-with? (str (:message %)) "Error during serialization export")
+ (messages))]
+ (is (seq errs) "the eager failure is logged")
+ (is (every? (comp nil? :e) errs)
+ "no throwable attached when full_stacktrace is off"))))
+ (testing "full trace when full_stacktrace=true"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]
+ (mt/user-http-request :crowberto :post 500 "ee/serialization/export"
+ :collection coll-id :data_model false :settings false
+ :full_stacktrace true)
+ (is (some :e (messages))
+ "the throwable is attached when full_stacktrace is on"))))))))
+
+(deftest export-eager-input-error-is-not-logged-test
+ (testing "a bad collection id fails eager extraction with a clean 4xx and is not logged as a server error (GDGT-2491)"
+ (mt/with-premium-features #{:serialization}
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]
+ ;; well-formed (21-char) but non-existent entity id: passes endpoint validation, then
+ ;; parse-target throws an ex-info carrying a :status-code, so it surfaces as a 4xx
+ (mt/user-http-request :crowberto :post 400 "ee/serialization/export"
+ :collection "0123456789abcdef01234" :data_model false :settings false)
+ (is (empty? (filter #(str/starts-with? (str (:message %)) "Error during serialization export")
+ (messages)))
+ "client input errors are not logged as server errors")))))
+
(deftest serialization-permissions-test
(testing "Only admins can export/import"
(mt/with-premium-features #{:serialization}
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index bd340a124f9..212ab8a4c3a 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1650,6 +1650,38 @@
(is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))
"Databases should be exported even when cards reference personal collections"))))))
+(deftest escape-continue-on-error-test
+ (testing "continue-on-error lets the export proceed past escape analysis instead of aborting (#74622)"
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering
+ :model/Database {db-id :id} {:engine :postgres}
+ :model/Collection {coll-id :id
+ coll-eid :entity_id} {:name "Target Collection"}
+ :model/Card {clean-eid :entity_id} {:name "Clean Card"
+ :collection_id coll-id
+ :database_id db-id}
+ :model/Dashboard {dash-id :id
+ dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}
+ ;; this card lives outside the target collection, so it "escapes"
+ :model/Card {escaped-eid :entity_id
+ escaped-id :id} {:name "Escaped Card" :database_id db-id}
+ :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]
+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]
+ (testing "without the flag, escape analysis aborts the whole export"
+ (is (empty? (into [] (extract/extract opts)))))
+ (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"
+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
+ (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]
+ (is (= #{coll-eid} (ids-by-model "Collection" extracted)))
+ (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))
+ (is (contains? (ids-by-model "Card" extracted) clean-eid)
+ "the clean card inside the target collection is exported")
+ (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))
+ "the escaped card outside the target collection is not exported")
+ (is (some #(str/starts-with? % "Failed to export Dashboard")
+ (map :message (messages)))
+ "the escape report is still logged as a warning")))))))))
+
(deftest recursive-colls-test
(mt/with-empty-h2-app-db!
(mt/with-temp [:model/Collection {parent-id :id
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index c9fd54e816f..b3cb83947c7 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -90,6 +90,57 @@
(is (= "Basic Collection" (:name (first colls))))
(is (= eid1 (:entity_id (first colls)))))))))))
+(deftest escape-continue-on-error-roundtrip-test
+ (testing "archive exported past escape analysis imports under continue-on-error without crashing (#74622)"
+ (let [serialized (atom nil)
+ coll-eid (atom nil)
+ clean-eid (atom nil)
+ dash-eid (atom nil)
+ escaped-eid (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (let [db (ts/create! :model/Database :name "rt-db")
+ table (ts/create! :model/Table :name "t" :db_id (:id db))
+ user (ts/create! :model/User :first_name "A" :last_name "B" :email "[email protected]")
+ target (ts/create! :model/Collection :name "Target")
+ outside (ts/create! :model/Collection :name "Outside (not exported)")
+ q {:type :query :database (:id db) :query {:source-table (:id table)}}
+ clean (ts/create! :model/Card :name "Clean" :collection_id (:id target)
+ :database_id (:id db) :table_id (:id table)
+ :query_type :query :dataset_query q :creator_id (:id user))
+ ;; lives outside the target collection, so it "escapes"
+ escaped (ts/create! :model/Card :name "Escaped" :collection_id (:id outside)
+ :database_id (:id db) :table_id (:id table)
+ :query_type :query :dataset_query q :creator_id (:id user))
+ dash (ts/create! :model/Dashboard :name "Dash" :collection_id (:id target) :creator_id (:id user))]
+ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id clean))
+ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id escaped))
+ (reset! coll-eid (:entity_id target))
+ (reset! clean-eid (:entity_id clean))
+ (reset! dash-eid (:entity_id dash))
+ (reset! escaped-eid (:entity_id escaped))
+ (reset! serialized
+ (into [] (serdes.extract/extract {:targets [["Collection" (:id target)]]
+ :no-settings true
+ :continue-on-error true})))))
+ (testing "export keeps the collection, clean card and dashboard, leaves the escaped card out"
+ (is (contains? (ids-by-model @serialized "Collection") @coll-eid))
+ (is (contains? (ids-by-model @serialized "Card") @clean-eid))
+ (is (not (contains? (ids-by-model @serialized "Card") @escaped-eid)))
+ (is (contains? (ids-by-model @serialized "Dashboard") @dash-eid)))
+ ;; The dashboard carries a dashcard pointing at the escaped card, which isn't in the archive. Under
+ ;; continue-on-error the import skips that dashboard (load-metabase! wraps each entity in try/catch)
+ ;; rather than aborting, and everything else lands. We do NOT promise a strict (default) import works.
+ (testing "import under continue-on-error doesn't crash; the dangling dashboard is skipped, the rest lands"
+ (ts/with-db dest-db
+ (let [report (serdes.load/load-metabase! (ingestion-in-memory @serialized) {:continue-on-error true})]
+ (is (some? (t2/select-one :model/Collection :entity_id @coll-eid)) "target collection imported")
+ (is (some? (t2/select-one :model/Card :entity_id @clean-eid)) "clean card imported")
+ (is (nil? (t2/select-one :model/Card :entity_id @escaped-eid)) "escaped card not imported")
+ (is (nil? (t2/select-one :model/Dashboard :entity_id @dash-eid))
+ "the dashboard that needs the escaped card is skipped, not imported with a dangling ref")
+ (is (seq (:errors report)) "the skipped dashboard is recorded as an import error"))))))))
+
(deftest deserialization-nested-collections-test
(testing "with a three-level nesting of collections"
(let [serialized (atom nil)