The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3802 capture extract-phase logs in serialization export.log (#75255)
Eric Normand706c4c444b26
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex 39a5e8d1d4c..805face4859 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -72,30 +72,37 @@;;; Logic+(def ^:private serialization-logger-prefixes+ "log4j2 logger-name prefixes whose logs are forked into the `export.log`/`import.log` files inside the archive.+ These are not loaded namespaces; each prefix captures every logger nested under it (e.g.+ `metabase-enterprise.serialization` captures `metabase-enterprise.serialization.v2.extract`)."+ ['metabase-enterprise.serialization+ 'metabase.models.serialization])+(defn- serialize-to-stream!- "Serialize directly to an OutputStream as streaming tar.gz. Returns result map."- [^java.io.OutputStream output ^String dirname entities {:keys [full-stacktrace]}]- (let [log-output (ByteArrayOutputStream.)- writer (v2.storage.tar/tar-writer output dirname)- error (atom nil)- report (with-open [_logger (logger/for-ns log-output ['metabase-enterprise.serialization- 'metabase.models.serialization]- {:additive *additive-logging*})]- (try- (let [report (serdes/with-cache- (v2.storage/store! entities writer))]- (v2.protocols/store-log! writer (.toByteArray log-output))- (v2.protocols/finish! writer)- report)- (catch Exception e- (reset! error e)- (if full-stacktrace- (log/error e "Error during serialization export")- (log/error (u/strip-error e "Error during serialization export")))- (try- (v2.protocols/store-log! writer (.toByteArray log-output))- (v2.protocols/finish! writer)- (catch Exception _)))))]+ "Serialize directly to an OutputStream as streaming tar.gz. Returns result map.++ Storage logs are appended to `log-output`, whose full contents are then written to `export.log` inside the+ archive."+ [^java.io.OutputStream output ^String dirname entities ^ByteArrayOutputStream log-output {:keys [full-stacktrace]}]+ (let [writer (v2.storage.tar/tar-writer output dirname)+ error (atom nil)+ report (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes+ {:additive *additive-logging*})]+ (try+ (serdes/with-cache+ (v2.storage/store! entities writer))+ (catch Exception e+ (reset! error e)+ (if full-stacktrace+ (log/error e "Error during serialization export")+ (log/error (u/strip-error e "Error during serialization export")))+ nil)))]+ ;; Read the buffer and write the log after the appender has closed (and thus flushed) so nothing is lost.+ (try+ (v2.protocols/store-log! writer (.toByteArray log-output))+ (v2.protocols/finish! writer)+ (catch Exception _)){:report report:success (nil? @error):error-message (when @error@@ -121,8 +128,7 @@log-file (io/file dst "import.log")err (atom nil)reindex? (if (nil? reindex?) true reindex?)- report (with-open [_logger (logger/for-ns log-file ['metabase-enterprise.serialization- 'metabase.models.serialization]+ report (with-open [_logger (logger/for-ns log-file serialization-logger-prefixes{:additive *additive-logging*})](try ; try/catch inside logging to log errors(log/infof "Serdes import, size %s" size)@@ -233,13 +239,18 @@(u/slugify (appearance/site-name))(u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))));; extract/extract runs eager setup (target resolution, escape analysis) which can throw- ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts.- entities (extract/extract opts)]+ ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts so the error+ ;; can set a non-200 status. Its eager logs (e.g. escape-analysis warnings) are captured into `log-output`+ ;; so they end up in export.log alongside the storage logs captured later.+ log-output (ByteArrayOutputStream.)+ entities (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes+ {:additive *additive-logging*})]+ (extract/extract opts))](sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan](sr/set-header! "Content-Disposition"(format "attachment; filename=\"%s.tar.gz\"" export-dirname))(let [start (System/nanoTime)- result (serialize-to-stream! output export-dirname entities opts)]+ result (serialize-to-stream! output export-dirname entities log-output opts)](track-export-event! collection opts start result)))));; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the restdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex 20ba28feaa9..2e19c1de4f2 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -237,6 +237,25 @@"error_message" nil}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))+(deftest export-log-captures-extract-warnings-test+ (testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)"+ ;; A dashboard in the exported collection references a card living in a different collection. Escape analysis+ ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. That warning+ ;; must still land in export.log even though extract happens outside the storage logging block.+ (mt/with-premium-features #{:serialization}+ (mt/with-temp [:model/Collection target {:name "Target Collection"}+ :model/Collection other {:name "Other Collection"}+ :model/Card outside-card {:collection_id (:id other) :name "OutsideCard"}+ :model/Dashboard dash {:collection_id (:id target) :name "DashWithOutsideCard"}+ :model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}]+ (let [res (binding [api.serialization/*additive-logging* false]+ (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}+ :collection (:id target) :data_model false :settings false))+ log (read-export-log res)]+ (is (some? log) "export.log should be present in the archive")+ (is (re-find #"outside requested collections" log)+ "export.log should contain the escape-analysis warning emitted during extract"))))))+(deftest import-restores-entities-test(testing "Import restores deleted/renamed entities and updates search index"(with-serialization-test-data! [coll dash card]