Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3802 capture extract-phase logs in serialization export.log (#75255)

Eric Normand706c4c444b26
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index 39a5e8d1d4c..805face4859 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -72,30 +72,37 @@
;;; Logic
+(def ^:private serialization-logger-prefixes
+ "log4j2 logger-name prefixes whose logs are forked into the `export.log`/`import.log` files inside the archive.
+ These are not loaded namespaces; each prefix captures every logger nested under it (e.g.
+ `metabase-enterprise.serialization` captures `metabase-enterprise.serialization.v2.extract`)."
+ ['metabase-enterprise.serialization
+ 'metabase.models.serialization])
+
(defn- serialize-to-stream!
- "Serialize directly to an OutputStream as streaming tar.gz. Returns result map."
- [^java.io.OutputStream output ^String dirname entities {:keys [full-stacktrace]}]
- (let [log-output (ByteArrayOutputStream.)
- writer (v2.storage.tar/tar-writer output dirname)
- error (atom nil)
- report (with-open [_logger (logger/for-ns log-output ['metabase-enterprise.serialization
- 'metabase.models.serialization]
- {:additive *additive-logging*})]
- (try
- (let [report (serdes/with-cache
- (v2.storage/store! entities writer))]
- (v2.protocols/store-log! writer (.toByteArray log-output))
- (v2.protocols/finish! writer)
- report)
- (catch Exception e
- (reset! error e)
- (if full-stacktrace
- (log/error e "Error during serialization export")
- (log/error (u/strip-error e "Error during serialization export")))
- (try
- (v2.protocols/store-log! writer (.toByteArray log-output))
- (v2.protocols/finish! writer)
- (catch Exception _)))))]
+ "Serialize directly to an OutputStream as streaming tar.gz. Returns result map.
+
+ Storage logs are appended to `log-output`, whose full contents are then written to `export.log` inside the
+ archive."
+ [^java.io.OutputStream output ^String dirname entities ^ByteArrayOutputStream log-output {:keys [full-stacktrace]}]
+ (let [writer (v2.storage.tar/tar-writer output dirname)
+ error (atom nil)
+ report (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
+ {:additive *additive-logging*})]
+ (try
+ (serdes/with-cache
+ (v2.storage/store! entities writer))
+ (catch Exception e
+ (reset! error e)
+ (if full-stacktrace
+ (log/error e "Error during serialization export")
+ (log/error (u/strip-error e "Error during serialization export")))
+ nil)))]
+ ;; Read the buffer and write the log after the appender has closed (and thus flushed) so nothing is lost.
+ (try
+ (v2.protocols/store-log! writer (.toByteArray log-output))
+ (v2.protocols/finish! writer)
+ (catch Exception _))
{:report report
:success (nil? @error)
:error-message (when @error
@@ -121,8 +128,7 @@
log-file (io/file dst "import.log")
err (atom nil)
reindex? (if (nil? reindex?) true reindex?)
- report (with-open [_logger (logger/for-ns log-file ['metabase-enterprise.serialization
- 'metabase.models.serialization]
+ report (with-open [_logger (logger/for-ns log-file serialization-logger-prefixes
{:additive *additive-logging*})]
(try ; try/catch inside logging to log errors
(log/infof "Serdes import, size %s" size)
@@ -233,13 +239,18 @@
(u/slugify (appearance/site-name))
(u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))
;; extract/extract runs eager setup (target resolution, escape analysis) which can throw
- ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts.
- entities (extract/extract opts)]
+ ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts so the error
+ ;; can set a non-200 status. Its eager logs (e.g. escape-analysis warnings) are captured into `log-output`
+ ;; so they end up in export.log alongside the storage logs captured later.
+ log-output (ByteArrayOutputStream.)
+ entities (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
+ {:additive *additive-logging*})]
+ (extract/extract opts))]
(sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan]
(sr/set-header! "Content-Disposition"
(format "attachment; filename=\"%s.tar.gz\"" export-dirname))
(let [start (System/nanoTime)
- result (serialize-to-stream! output export-dirname entities opts)]
+ result (serialize-to-stream! output export-dirname entities log-output opts)]
(track-export-event! collection opts start result)))))
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index 20ba28feaa9..2e19c1de4f2 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -237,6 +237,25 @@
"error_message" nil}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))))))
+(deftest export-log-captures-extract-warnings-test
+ (testing "export.log captures escape-analysis warnings emitted during the eager extract phase (GHY-3802)"
+ ;; A dashboard in the exported collection references a card living in a different collection. Escape analysis
+ ;; runs eagerly inside extract/extract (before storage streaming) and warns about the escaped card. That warning
+ ;; must still land in export.log even though extract happens outside the storage logging block.
+ (mt/with-premium-features #{:serialization}
+ (mt/with-temp [:model/Collection target {:name "Target Collection"}
+ :model/Collection other {:name "Other Collection"}
+ :model/Card outside-card {:collection_id (:id other) :name "OutsideCard"}
+ :model/Dashboard dash {:collection_id (:id target) :name "DashWithOutsideCard"}
+ :model/DashboardCard _ {:dashboard_id (:id dash) :card_id (:id outside-card)}]
+ (let [res (binding [api.serialization/*additive-logging* false]
+ (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
+ :collection (:id target) :data_model false :settings false))
+ log (read-export-log res)]
+ (is (some? log) "export.log should be present in the archive")
+ (is (re-find #"outside requested collections" log)
+ "export.log should contain the escape-analysis warning emitted during extract"))))))
+
(deftest import-restores-entities-test
(testing "Import restores deleted/renamed entities and updates search index"
(with-serialization-test-data! [coll dash card]