The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
(ns metabase-enterprise.serialization.api
(:require
[clojure.java.io :as io]
[clojure.string :as str]
[java-time.api :as t]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase-enterprise.serialization.v2.load :as v2.load]
[metabase-enterprise.serialization.v2.protocols :as v2.protocols]
[metabase-enterprise.serialization.v2.storage :as v2.storage]
[metabase-enterprise.serialization.v2.storage.tar :as v2.storage.tar]
[metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.appearance.core :as appearance]
[metabase.logger.core :as logger]
[metabase.models.serialization :as serdes]
[metabase.server.streaming-response :as sr]
[metabase.util :as u]
[metabase.util.compress :as u.compress]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.util.random :as u.random]
[ring.core.protocols :as ring.protocols])
(:import
(java.io ByteArrayOutputStream File)))
(set! *warn-on-reflection* true)
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *additive-logging*
"If custom loggers should pass logs to parent loggers (to system Metabase logs), used to clean up test output."
true)
;;; Storage
(def parent-dir "Dir for storing serialization API export-in-progress and archives."
(let [f (io/file (System/getProperty "java.io.tmpdir") (str "serdesv2-" (u.random/random-name)))]
(.mkdirs f)
(.deleteOnExit f)
(.getPath f)))
;;; Request callbacks
(defn- ba-copy [f]
(with-open [baos (ByteArrayOutputStream.)]
(io/copy f baos)
(.toByteArray baos)))
(defn- on-response! [data callback]
(reify
;; Real HTTP requests and mt/user-real-request go here
ring.protocols/StreamableResponseBody
(write-body-to-stream [_ response out]
(ring.protocols/write-body-to-stream data response out)
(future (callback)))
;; mt/user-http-request goes here
clojure.java.io.IOFactory
(make-input-stream [_ _]
(let [res (io/input-stream (if (instance? File data)
(ba-copy data)
data))]
(callback)
res))))
;;; Logic
(def ^:private serialization-logger-prefixes
"log4j2 logger-name prefixes whose logs are forked into the `export.log`/`import.log` files inside the archive.
These are not loaded namespaces; each prefix captures every logger nested under it (e.g.
`metabase-enterprise.serialization` captures `metabase-enterprise.serialization.v2.extract`)."
['metabase-enterprise.serialization
'metabase.models.serialization])
(defn- log-export-error!
"Log a serialization export error, honoring `full-stacktrace` (full trace vs stripped one-liner)."
[e full-stacktrace]
(if full-stacktrace
(log/error e "Error during serialization export")
(log/error (u/strip-error e "Error during serialization export"))))
(defn- extract-entities!
"Run eager extraction (target resolution, escape analysis) before streaming starts. It must run
before the streaming response opens so a failure can still set a non-200 status. Eager logs (e.g.
escape-analysis warnings) are captured into `log-output` so they land in export.log alongside the
storage logs captured later. `full-stacktrace` is honored for genuine server failures the way the
streaming path and the CLI export do; client input errors carry a `:status-code` and pass through to
the API layer unlogged, so they surface as a clean 4xx rather than a logged server error."
[opts ^ByteArrayOutputStream log-output full-stacktrace]
(try
(with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
{:additive *additive-logging*})]
(extract/extract opts))
(catch Exception e
(when-not (:status-code (ex-data e))
(log-export-error! e full-stacktrace))
(throw e))))
(defn- serialize-to-stream!
"Serialize directly to an OutputStream as streaming tar.gz. Returns result map.
Storage logs are appended to `log-output`, whose full contents are then written to `export.log` inside the
archive."
[^java.io.OutputStream output ^String dirname entities ^ByteArrayOutputStream log-output {:keys [full-stacktrace]}]
(let [writer (v2.storage.tar/tar-writer output dirname)
error (atom nil)
report (with-open [_logger (logger/for-ns log-output serialization-logger-prefixes
{:additive *additive-logging*})]
(try
(serdes/with-cache
(v2.storage/store! entities writer))
(catch Exception e
(reset! error e)
(log-export-error! e full-stacktrace)
nil)))]
;; Read the buffer and write the log after the appender has closed (and thus flushed) so nothing is lost.
(try
(v2.protocols/store-log! writer (.toByteArray log-output))
(v2.protocols/finish! writer)
(catch Exception _))
{:report report
:success (nil? @error)
:error-message (when @error
(u/strip-error @error nil))}))
(defn- find-serialization-dir
"Find an actual top-level dir with serialization data inside, instead of picking up various .DS_Store and similar
things."
^File [^File parent]
(let [check-dir (fn [^File f]
(and (.isDirectory f)
(some v2.ingest/legal-top-level-paths (.list f))))]
(if (check-dir parent)
parent
(->> (.listFiles parent)
(u/seek check-dir)))))
(defn- unpack&import [^File file & [{:keys [size
continue-on-error
full-stacktrace
reindex?]}]]
(let [dst (io/file parent-dir (u.random/random-name))
log-file (io/file dst "import.log")
err (atom nil)
reindex? (if (nil? reindex?) true reindex?)
report (with-open [_logger (logger/for-ns log-file serialization-logger-prefixes
{:additive *additive-logging*})]
(try ; try/catch inside logging to log errors
(log/infof "Serdes import, size %s" size)
(let [cnt (try (u.compress/untgz file dst)
(catch Exception e
(throw (ex-info "Cannot unpack archive" {:status 422} e))))
path (find-serialization-dir dst)]
(when-not path
(throw (ex-info "No source dir detected. Please make sure the serialization files are in the top level dir."
{:status 400
:dst (.getPath dst)
:count cnt
:files (.listFiles dst)})))
(log/infof "In total %s entries unpacked, source dir detected" cnt)
(serdes/with-cache
(-> (v2.ingest/ingest-yaml (.getPath path))
(v2.load/load-metabase! {:continue-on-error continue-on-error
:reindex? reindex?}))))
(catch Exception e
(reset! err e)
(if full-stacktrace
(log/error e "Error during serialization import")
(log/error (u/strip-error e "Error during serialization import"))))))]
{:log-file log-file
:status (:status (ex-data @err))
:error-message (when @err
(u/strip-error @err nil))
:report report
:callback #(when (.exists dst)
(run! io/delete-file (reverse (file-seq dst))))}))
(defn- track-export-event! [collection opts start {:keys [report success error-message]}]
(analytics/track-event! :snowplow/serialization
{:event :serialization
:direction "export"
:source "api"
:duration_ms (int (/ (- (System/nanoTime) start) 1e6))
:count (reduce + 0 (vals (:entity-counts report)))
:error_count (count (:errors report))
:collection (str/join "," (map str collection))
:all_collections (and (empty? collection)
(not (:no-collections opts)))
:data_model (not (:no-data-model opts))
:settings (not (:no-settings opts))
:field_values (:include-field-values opts)
:secrets false
:success (boolean success)
:error_message error-message}))
;;; HTTP API
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
;; of the REST API
;;
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-query-params-use-kebab-case
:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/export"
"Serialize and retrieve Metabase instance.
Outputs `.tar.gz` file with serialization results and an `export.log` file.
On error outputs serialization logs directly."
[_route-params
{:keys [collection dirname]
include-field-values? :field_values
all-collections? :all_collections
data-model? :data_model
settings? :settings
continue-on-error? :continue_on_error
full-stacktrace? :full_stacktrace
:as _query-params}
:- [:map {:closed true}
[:dirname {:optional true} [:maybe
{:description (deferred-tru "name of directory and archive file (default: `<instance-name>-<YYYY-MM-dd_HH-mm>`): 1-255 characters from A-Z a-z 0-9 _ - . %, not starting with . or -")}
(mu/with-api-error-message
;; `%`: the default name is a slug, which percent-encodes non-ASCII
[:re #"^[A-Za-z0-9_%][A-Za-z0-9._%-]{0,254}$"]
(deferred-tru "name of directory and archive file (default: `<instance-name>-<YYYY-MM-dd_HH-mm>`): 1-255 characters from A-Z a-z 0-9 _ - . %, not starting with . or -")
(deferred-tru "1-255 characters from A-Z a-z 0-9 _ - . %, not starting with . or -"))]]
[:collection {:optional true} [:maybe
{:description "collections' db ids/entity-ids to serialize"}
(ms/QueryVectorOf
[:or
ms/PositiveInt
[:re {:error/message "if you are passing entity_id, it should be exactly 21 chars long"}
#"^.{21}$"]
[:re {:error/message "value must be string with `eid:<...>` prefix"}
#"^eid:.{21}$"]])]]
[:all_collections {:default true} (mu/with ms/BooleanValue {:description "Serialize all collections (`true` unless you specify `collection`)"})]
[:settings {:default true} (mu/with ms/BooleanValue {:description "Serialize Metabase settings"})]
[:data_model {:default true} (mu/with ms/BooleanValue {:description "Serialize Metabase data model"})]
[:field_values {:default false} (mu/with ms/BooleanValue {:description "Serialize cached field values"})]
[:continue_on_error {:default false} (mu/with ms/BooleanValue {:description "Do not break execution on errors"})]
[:full_stacktrace {:default false} (mu/with ms/BooleanValue {:description "Show full stacktraces in the logs"})]]]
(api/check-superuser)
(let [opts {:targets (mapv #(vector "Collection" %)
collection)
:no-collections (and (empty? collection)
(not all-collections?))
:no-data-model (not data-model?)
:no-settings (not settings?)
:include-field-values include-field-values?
:continue-on-error continue-on-error?
:full-stacktrace full-stacktrace?}
export-dirname (or dirname
(format "%s-%s"
;; + 17 for the timestamp = 255, so the default passes the `dirname` schema
(u/slugify (appearance/site-name) {:max-length 238})
(u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))
;; Eager setup (target resolution, escape analysis) must run before the streaming response opens
;; so a failure can still set a non-200 status. extract-entities! captures its logs into log-output
;; (so escape-analysis warnings land in export.log) and honors full_stacktrace for genuine failures.
log-output (ByteArrayOutputStream.)
entities (extract-entities! opts log-output full-stacktrace?)]
(sr/streaming-response {:content-type "application/gzip"
:status 200
:headers {"Content-Disposition" (format "attachment; filename=\"%s.tar.gz\""
export-dirname)}} [output _cancel-chan]
(let [start (System/nanoTime)
result (serialize-to-stream! output export-dirname entities log-output opts)]
(track-export-event! collection opts start result)))))
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
;; of the REST API
;;
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
;;
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-query-params-use-kebab-case
:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/import"
"Deserialize Metabase instance from an archive generated by /export.
Parameters:
- `file`: archive encoded as `multipart/form-data` (required).
Returns logs of deserialization."
{:multipart true}
[_route-params
{continue-on-error? :continue_on_error
full-stacktrace? :full_stacktrace
reindex-search? :reindex
:as _query-params}
:- [:map {:closed true}
[:continue_on_error {:default false} (mu/with ms/BooleanValue {:description "Do not break execution on errors"})]
[:full_stacktrace {:default false} (mu/with ms/BooleanValue {:description "Show full stacktraces in the logs"})]
;; TODO this parameter is a kludge to fix https://linear.app/metabase/issue/GDGT-573
;; ideally we'd fix the underlying issue (by delaying realtime indexing updates until the tx closes)
;; for now, we let users opt out, in case they're indexing a lot, so they can only reindex on the last step
[:reindex {:default true} (mu/with ms/BooleanValue {:description "Rebuild the search index afterwards"})]]
{:keys [file]} :- [:map {:closed true}
[:file (mu/with ms/File {:description ".tgz with serialization data"})]]]
(api/check-superuser)
(try
(let [start (System/nanoTime)
{:keys [log-file
status
error-message
report
callback]} (unpack&import (:tempfile file)
{:size (:size file)
:continue-on-error continue-on-error?
:full-stacktrace full-stacktrace?
:reindex? reindex-search?})
imported (into (sorted-set) (map (comp :model last)) (:seen report))]
(analytics/track-event! :snowplow/serialization
{:event :serialization
:direction "import"
:source "api"
:duration_ms (int (/ (- (System/nanoTime) start) 1e6))
:models (str/join "," imported)
:count (if (contains? imported "Setting")
(inc (count (remove #(= "Setting" (:model (first %))) (:seen report))))
(count (:seen report)))
:error_count (count (:errors report))
:success (not error-message)
:error_message error-message})
(if error-message
{:status (or status 500)
:headers {"Content-Type" "text/plain"}
:body (on-response! log-file callback)}
{:status 200
:headers {"Content-Type" "text/plain"}
:body (on-response! log-file callback)}))
(finally
(io/delete-file (:tempfile file)))))
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/serialization` routes."
(api.macros/ns-handler *ns* +auth))