The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Honor continue_on_error and full_stacktrace on serialization export (#75122)
Ngoc Khuataf6ea737a3e4
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 805face4859..bf70eb023e0 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -79,6 +79,30 @@['metabase-enterprise.serialization'metabase.models.serialization])+(defn- log-export-error!+ "Log a serialization export error, honoring `full-stacktrace` (full trace vs stripped one-liner)."+ [e full-stacktrace]+ (if full-stacktrace+ (log/error e "Error during serialization export")+ (log/error (u/strip-error e "Error during serialization export"))))++(defn- extract-entities!+ "Run eager extraction (target resolution, escape analysis) before streaming starts. It must run+ before the streaming response opens so a failure can still set a non-200 status. Eager logs (e.g.+ escape-analysis warnings) are captured into `log-output` so they land in export.log alongside the+ storage logs captured later. `full-stacktrace` is honored for genuine server failures the way the+ streaming path and the CLI export do; client input errors carry a `:status-code` and pass through to+ the API layer unlogged, so they surface as a clean 4xx rather than a logged server error."+ [opts ^ByteArrayOutputStream log-output full-stacktrace]+ (try+ (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes+ {:additive *additive-logging*})]+ (extract/extract opts))+ (catch Exception e+ (when-not (:status-code (ex-data e))+ (log-export-error! e full-stacktrace))+ (throw e))))+(defn- serialize-to-stream!"Serialize directly to an OutputStream as streaming tar.gz. Returns result map.@@ -94,9 +118,7 @@(v2.storage/store! entities writer))(catch Exception e(reset! error e)- (if full-stacktrace- (log/error e "Error during serialization export")- (log/error (u/strip-error e "Error during serialization export")))+ (log-export-error! e full-stacktrace)nil)))];; Read the buffer and write the log after the appender has closed (and thus flushed) so nothing is lost.(try@@ -238,14 +260,11 @@(format "%s-%s"(u/slugify (appearance/site-name))(u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))- ;; extract/extract runs eager setup (target resolution, escape analysis) which can throw- ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts so the error- ;; can set a non-200 status. Its eager logs (e.g. escape-analysis warnings) are captured into `log-output`- ;; so they end up in export.log alongside the storage logs captured later.+ ;; Eager setup (target resolution, escape analysis) must run before the streaming response opens+ ;; so a failure can still set a non-200 status. extract-entities! captures its logs into log-output+ ;; (so escape-analysis warnings land in export.log) and honors full_stacktrace for genuine failures.log-output (ByteArrayOutputStream.)- entities (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes- {:additive *additive-logging*})]- (extract/extract opts))]+ entities (extract-entities! opts log-output full-stacktrace?)](sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan](sr/set-header! "Content-Disposition"(format "attachment; filename=\"%s.tar.gz\"" export-dirname))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 0b00603a1d8..700fcdd57a0 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -185,8 +185,14 @@;; cards referenced by dashboards live outside the target collections.{:keys [reportable-escaped analytics-card-ids]} (when has-content?(escape-analysis by-model nodes))]- (if (seq reportable-escaped)- (log-escape-report! reportable-escaped)+ (when (seq reportable-escaped)+ (log-escape-report! reportable-escaped))+ ;; By default a card referenced from inside the requested collections but living outside them aborts+ ;; the whole export. With continue-on-error we don't abort: the escaped cards are outside the+ ;; collection set so they're filtered out of extraction anyway, and the dashboards/cards that+ ;; reference them are skipped on import (which also honors continue-on-error).+ (when (or (empty? reportable-escaped)+ (:continue-on-error opts))(let [coll-set (get by-model "Collection");; When targets are specified, also include Tables found via descendants;; (published tables in target collections). These are extracted by ID, not all.diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 2e19c1de4f2..72bfb6e8eaf 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -7,6 +7,7 @@[medley.core :as m][metabase-enterprise.serialization.api :as api.serialization][metabase-enterprise.serialization.metadata-file-import :as metadata-file-import]+ [metabase-enterprise.serialization.v2.extract :as v2.extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase.analytics.snowplow-test :as snowplow-test][metabase.models.serialization :as serdes]@@ -431,6 +432,40 @@"error_message" nil}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))+(deftest export-eager-error-honors-full-stacktrace-test+ (testing "a server-side failure during eager extraction setup (before streaming) honors full_stacktrace (GDGT-2491)"+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Collection {coll-id :id} {}]+ (mt/with-dynamic-fn-redefs [v2.extract/extract (fn [& _] (throw (ex-info "deliberate eager failure" {})))]+ (testing "stripped one-liner by default"+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]+ (mt/user-http-request :crowberto :post 500 "ee/serialization/export"+ :collection coll-id :data_model false :settings false)+ (let [errs (filter #(str/starts-with? (str (:message %)) "Error during serialization export")+ (messages))]+ (is (seq errs) "the eager failure is logged")+ (is (every? (comp nil? :e) errs)+ "no throwable attached when full_stacktrace is off"))))+ (testing "full trace when full_stacktrace=true"+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]+ (mt/user-http-request :crowberto :post 500 "ee/serialization/export"+ :collection coll-id :data_model false :settings false+ :full_stacktrace true)+ (is (some :e (messages))+ "the throwable is attached when full_stacktrace is on"))))))))++(deftest export-eager-input-error-is-not-logged-test+ (testing "a bad collection id fails eager extraction with a clean 4xx and is not logged as a server error (GDGT-2491)"+ (mt/with-premium-features #{:serialization}+ (mt/with-log-messages-for-level [messages [metabase-enterprise.serialization :error]]+ ;; well-formed (21-char) but non-existent entity id: passes endpoint validation, then+ ;; parse-target throws an ex-info carrying a :status-code, so it surfaces as a 4xx+ (mt/user-http-request :crowberto :post 400 "ee/serialization/export"+ :collection "0123456789abcdef01234" :data_model false :settings false)+ (is (empty? (filter #(str/starts-with? (str (:message %)) "Error during serialization export")+ (messages)))+ "client input errors are not logged as server errors")))))+(deftest serialization-permissions-test(testing "Only admins can export/import"(mt/with-premium-features #{:serialization}diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex bd340a124f9..212ab8a4c3a 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1650,6 +1650,38 @@(is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))"Databases should be exported even when cards reference personal collections"))))))+(deftest escape-continue-on-error-test+ (testing "continue-on-error lets the export proceed past escape analysis instead of aborting (#74622)"+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [;; non-H2 engine so the database survives serdes extract filtering+ :model/Database {db-id :id} {:engine :postgres}+ :model/Collection {coll-id :id+ coll-eid :entity_id} {:name "Target Collection"}+ :model/Card {clean-eid :entity_id} {:name "Clean Card"+ :collection_id coll-id+ :database_id db-id}+ :model/Dashboard {dash-id :id+ dash-eid :entity_id} {:name "A Dashboard" :collection_id coll-id}+ ;; this card lives outside the target collection, so it "escapes"+ :model/Card {escaped-eid :entity_id+ escaped-id :id} {:name "Escaped Card" :database_id db-id}+ :model/DashboardCard _ {:card_id escaped-id :dashboard_id dash-id}]+ (let [opts {:targets [["Collection" coll-id]] :no-settings true :no-data-model true}]+ (testing "without the flag, escape analysis aborts the whole export"+ (is (empty? (into [] (extract/extract opts)))))+ (testing "with continue-on-error, everything in the target collection is exported and the escaped card is left out"+ (mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]+ (let [extracted (into [] (extract/extract (assoc opts :continue-on-error true)))]+ (is (= #{coll-eid} (ids-by-model "Collection" extracted)))+ (is (= #{dash-eid} (ids-by-model "Dashboard" extracted)))+ (is (contains? (ids-by-model "Card" extracted) clean-eid)+ "the clean card inside the target collection is exported")+ (is (not (contains? (ids-by-model "Card" extracted) escaped-eid))+ "the escaped card outside the target collection is not exported")+ (is (some #(str/starts-with? % "Failed to export Dashboard")+ (map :message (messages)))+ "the escape report is still logged as a warning")))))))))+(deftest recursive-colls-test(mt/with-empty-h2-app-db!(mt/with-temp [:model/Collection {parent-id :iddiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex c9fd54e816f..b3cb83947c7 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -90,6 +90,57 @@(is (= "Basic Collection" (:name (first colls))))(is (= eid1 (:entity_id (first colls)))))))))))+(deftest escape-continue-on-error-roundtrip-test+ (testing "archive exported past escape analysis imports under continue-on-error without crashing (#74622)"+ (let [serialized (atom nil)+ coll-eid (atom nil)+ clean-eid (atom nil)+ dash-eid (atom nil)+ escaped-eid (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (let [db (ts/create! :model/Database :name "rt-db")+ table (ts/create! :model/Table :name "t" :db_id (:id db))+ user (ts/create! :model/User :first_name "A" :last_name "B" :email "[email protected]")+ target (ts/create! :model/Collection :name "Target")+ outside (ts/create! :model/Collection :name "Outside (not exported)")+ q {:type :query :database (:id db) :query {:source-table (:id table)}}+ clean (ts/create! :model/Card :name "Clean" :collection_id (:id target)+ :database_id (:id db) :table_id (:id table)+ :query_type :query :dataset_query q :creator_id (:id user))+ ;; lives outside the target collection, so it "escapes"+ escaped (ts/create! :model/Card :name "Escaped" :collection_id (:id outside)+ :database_id (:id db) :table_id (:id table)+ :query_type :query :dataset_query q :creator_id (:id user))+ dash (ts/create! :model/Dashboard :name "Dash" :collection_id (:id target) :creator_id (:id user))]+ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id clean))+ (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id escaped))+ (reset! coll-eid (:entity_id target))+ (reset! clean-eid (:entity_id clean))+ (reset! dash-eid (:entity_id dash))+ (reset! escaped-eid (:entity_id escaped))+ (reset! serialized+ (into [] (serdes.extract/extract {:targets [["Collection" (:id target)]]+ :no-settings true+ :continue-on-error true})))))+ (testing "export keeps the collection, clean card and dashboard, leaves the escaped card out"+ (is (contains? (ids-by-model @serialized "Collection") @coll-eid))+ (is (contains? (ids-by-model @serialized "Card") @clean-eid))+ (is (not (contains? (ids-by-model @serialized "Card") @escaped-eid)))+ (is (contains? (ids-by-model @serialized "Dashboard") @dash-eid)))+ ;; The dashboard carries a dashcard pointing at the escaped card, which isn't in the archive. Under+ ;; continue-on-error the import skips that dashboard (load-metabase! wraps each entity in try/catch)+ ;; rather than aborting, and everything else lands. We do NOT promise a strict (default) import works.+ (testing "import under continue-on-error doesn't crash; the dangling dashboard is skipped, the rest lands"+ (ts/with-db dest-db+ (let [report (serdes.load/load-metabase! (ingestion-in-memory @serialized) {:continue-on-error true})]+ (is (some? (t2/select-one :model/Collection :entity_id @coll-eid)) "target collection imported")+ (is (some? (t2/select-one :model/Card :entity_id @clean-eid)) "clean card imported")+ (is (nil? (t2/select-one :model/Card :entity_id @escaped-eid)) "escaped card not imported")+ (is (nil? (t2/select-one :model/Dashboard :entity_id @dash-eid))+ "the dashboard that needs the escaped card is skipped, not imported with a dangling ref")+ (is (seq (:errors report)) "the skipped dashboard is recorded as an import error"))))))))+(deftest deserialization-nested-collections-test(testing "with a three-level nesting of collections"(let [serialized (atom nil)