Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Trivial serdes export fixes for metadata (#72434)

Alexander Polyankin28f1915be372
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
index fbc5aea69e8..a51cb93d7e3 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj
@@ -1,13 +1,14 @@
(ns metabase-enterprise.serialization.api
(:require
- [clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.string :as str]
[java-time.api :as t]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase-enterprise.serialization.v2.load :as v2.load]
- [metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.protocols :as v2.protocols]
+ [metabase-enterprise.serialization.v2.storage :as v2.storage]
+ [metabase-enterprise.serialization.v2.storage.tar :as v2.storage.tar]
[metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
@@ -19,7 +20,6 @@
[metabase.util :as u]
[metabase.util.compress :as u.compress]
[metabase.util.date-2 :as u.date]
- [metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -68,43 +68,34 @@
;;; Logic
-(defn- serialize&pack ^File [{:keys [dirname full-stacktrace] :as opts}]
- (let [dirname (or dirname
- (format "%s-%s"
- (u/slugify (appearance/site-name))
- (u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))
- path (io/file parent-dir dirname)
- dst (io/file (str (.getPath path) ".tar.gz"))
- log-file (io/file path "export.log")
- err (atom nil)
- report (with-open [_logger (logger/for-ns log-file ['metabase-enterprise.serialization
- 'metabase.models.serialization]
- {:additive *additive-logging*})]
- (try ; try/catch inside logging to log errors
- (let [report (serdes/with-cache
- (-> (extract/extract opts)
- (storage/store! path)))]
- ;; not removing dumped yamls immediately to save some time before response
- (u.compress/tgz path dst)
- report)
- (catch Exception e
- (reset! err e)
- (if full-stacktrace
- (log/error e "Error during serialization export")
- (log/error (u/strip-error e "Error during serialization export"))))))]
- {:archive (when (.exists dst)
- dst)
- :log-file (when (.exists log-file)
- log-file)
- :report report
- :status (:status-code (ex-data @err))
- :error-message (when @err
- (u/strip-error @err nil))
- :callback (fn []
- (when (.exists path)
- (run! io/delete-file (reverse (file-seq path))))
- (when (.exists dst)
- (io/delete-file dst)))}))
+(defn- serialize-to-stream!
+ "Serialize directly to an OutputStream as streaming tar.gz. Returns result map."
+ [^java.io.OutputStream output ^String dirname entities {:keys [full-stacktrace]}]
+ (let [log-output (ByteArrayOutputStream.)
+ writer (v2.storage.tar/tar-writer output dirname)
+ error (atom nil)
+ report (with-open [_logger (logger/for-ns log-output ['metabase-enterprise.serialization
+ 'metabase.models.serialization]
+ {:additive *additive-logging*})]
+ (try
+ (let [report (serdes/with-cache
+ (v2.storage/store! entities writer))]
+ (v2.protocols/store-log! writer (.toByteArray log-output))
+ (v2.protocols/finish! writer)
+ report)
+ (catch Exception e
+ (reset! error e)
+ (if full-stacktrace
+ (log/error e "Error during serialization export")
+ (log/error (u/strip-error e "Error during serialization export")))
+ (try
+ (v2.protocols/store-log! writer (.toByteArray log-output))
+ (v2.protocols/finish! writer)
+ (catch Exception _)))))]
+ {:report report
+ :success (nil? @error)
+ :error-message (when @error
+ (u/strip-error @error nil))}))
(defn- find-serialization-dir
"Find an actual top-level dir with serialization data inside, instead of picking up various .DS_Store and similar
@@ -159,31 +150,23 @@
:callback #(when (.exists dst)
(run! io/delete-file (reverse (file-seq dst))))}))
-(defn- serialization-api-handler
- [collection result-atom opts]
- (let [done-chan (a/chan)]
- [(u.jvm/in-virtual-thread*
- (let [start (System/nanoTime)
- {:keys [archive report error-message] :as pack-result} (serialize&pack opts)]
- (analytics/track-event! :snowplow/serialization
- {:event :serialization
- :direction "export"
- :source "api"
- :duration_ms (int (/ (- (System/nanoTime) start) 1e6))
- :count (count (:seen report))
- :error_count (count (:errors report))
- :collection (str/join "," (map str collection))
- :all_collections (and (empty? collection)
- (not (:no-collections opts)))
- :data_model (not (:no-data-model opts))
- :settings (not (:no-settings opts))
- :field_values (:include-field-values opts)
- :secrets (:include-database-secrets opts)
- :success (boolean archive)
- :error_message error-message})
- (reset! result-atom pack-result)
- (a/>!! done-chan :done)))
- done-chan]))
+(defn- track-export-event! [collection opts start {:keys [report success error-message]}]
+ (analytics/track-event! :snowplow/serialization
+ {:event :serialization
+ :direction "export"
+ :source "api"
+ :duration_ms (int (/ (- (System/nanoTime) start) 1e6))
+ :count (count (:seen report))
+ :error_count (count (:errors report))
+ :collection (str/join "," (map str collection))
+ :all_collections (and (empty? collection)
+ (not (:no-collections opts)))
+ :data_model (not (:no-data-model opts))
+ :settings (not (:no-settings opts))
+ :field_values (:include-field-values opts)
+ :secrets (:include-database-secrets opts)
+ :success (boolean success)
+ :error_message error-message}))
;;; HTTP API
@@ -239,33 +222,21 @@
:no-settings (not settings?)
:include-field-values include-field-values?
:include-database-secrets include-database-secrets?
- :dirname dirname
:continue-on-error continue-on-error?
:full-stacktrace full-stacktrace?}
- result (atom nil)
- [archive-thread finished-chan] (serialization-api-handler collection result opts)]
- (sr/streaming-response {:content-type "application/gzip" :status 200} [os cancel-chan]
- (a/<!!
- (a/go
- (case (a/alt! cancel-chan :cancel finished-chan :done)
- :cancel (do
- (future-cancel archive-thread)
- (when-let [cb (:callback @result)]
- (cb))
- (.close os))
- :done (let [{:keys [archive status log-file callback]} @result]
- (try
- (if archive
- (do
- (sr/set-header! "Content-Disposition"
- (format "attachment; filename=\"%s\"" (.getName ^File archive)))
- (io/copy archive os))
- (do
- (sr/set-status! (or status 500))
- (sr/set-content-type! "text/plain")
- (io/copy log-file os)))
- (finally
- (callback))))))))))
+ export-dirname (or dirname
+ (format "%s-%s"
+ (u/slugify (appearance/site-name))
+ (u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))
+ ;; extract/extract runs eager setup (target resolution, escape analysis) which can throw
+ ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts.
+ entities (extract/extract opts)]
+ (sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan]
+ (sr/set-header! "Content-Disposition"
+ (format "attachment; filename=\"%s.tar.gz\"" export-dirname))
+ (let [start (System/nanoTime)
+ result (serialize-to-stream! output export-dirname entities opts)]
+ (track-export-event! collection opts start result)))))
;; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest
;; of the REST API
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
index 5bf58202df6..9f3615724ed 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
@@ -8,6 +8,7 @@
[metabase-enterprise.serialization.v2.ingest :as v2.ingest]
[metabase-enterprise.serialization.v2.load :as v2.load]
[metabase-enterprise.serialization.v2.storage :as v2.storage]
+ [metabase-enterprise.serialization.v2.storage.files :as v2.storage.files]
[metabase.analytics.core :as analytics]
[metabase.app-db.core :as mdb]
[metabase.events.core :as events]
@@ -113,7 +114,7 @@
report (try
(serdes/with-cache
(-> (v2.extract/extract opts)
- (v2.storage/store! path)))
+ (v2.storage/store! (v2.storage.files/file-writer path))))
;; we could publish :event/serdes-dump to go with :event/serdes-load above, but
;; nothing would listen to it currently
(catch Exception e
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
index 137e48171c7..f8d3f49c3ed 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj
@@ -5,6 +5,7 @@
[metabase-enterprise.serialization.v2.ingest]
[metabase-enterprise.serialization.v2.load]
[metabase-enterprise.serialization.v2.storage]
+ [metabase-enterprise.serialization.v2.storage.util]
[potemkin :as p]))
(comment
@@ -12,7 +13,8 @@
metabase-enterprise.serialization.v2.extract/keep-me
metabase-enterprise.serialization.v2.ingest/keep-me
metabase-enterprise.serialization.v2.load/keep-me
- metabase-enterprise.serialization.v2.storage/keep-me)
+ metabase-enterprise.serialization.v2.storage/keep-me
+ metabase-enterprise.serialization.v2.storage.util/keep-me)
(p/import-vars
[metabase-enterprise.serialization.dump
@@ -33,6 +35,7 @@
[metabase-enterprise.serialization.v2.load
load-metabase!]
[metabase-enterprise.serialization.v2.storage
+ store!]
+ [metabase-enterprise.serialization.v2.storage.util
resolve-storage-path
- slugify-name
- store!])
+ slugify-name])
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 151beae0a39..b5d594238b8 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -143,8 +143,8 @@
:error ::not-found}))))
(log/debug "Local" {:path (serdes/log-path-str path)})
ctx)
- (let [_ (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}
- (circular path) (assoc :stripped true)))
+ (let [_ (log/trace "Loading" (cond-> {:path (serdes/log-path-str path)}
+ (circular path) (assoc :stripped true)))
;; Use the abstract path as attached by the ingestion process, not the original one we were passed.
rebuilt-path (serdes/path ingested)
;; If nil or absent :entity_id is taken as a signal to create a new entity
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.clj
new file mode 100644
index 00000000000..de5d04ef05d
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.clj
@@ -0,0 +1,14 @@
+(ns metabase-enterprise.serialization.v2.protocols
+ (:require
+ [potemkin.types :as p]))
+
+(p/defprotocol+ ExportWriter
+ "Protocol for serialization export writer."
+ (store-entity! [this entity]
+ "Store a single serialized entity. Returns its `:serdes/meta`.")
+ (store-settings! [this settings]
+ "Store accumulated settings as a single settings.yaml entry.")
+ (store-log! [this ^bytes content]
+ "Store export.log content.")
+ (finish! [this]
+ "Finalize the backend (e.g. close tar stream). Called after all entities are written."))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
index 503305e6a72..9723303e81b 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj
@@ -1,90 +1,15 @@
(ns metabase-enterprise.serialization.v2.storage
(:require
- [clojure.java.io :as io]
- [clojure.string :as str]
- [metabase-enterprise.serialization.dump :refer [spit-yaml!]]
- [metabase.lib.core :as lib]
- [metabase.models.serialization :as serdes]
- [metabase.util :as u]
- [metabase.util.log :as log]
- [metabase.util.string :as u.str])
- (:import
- [java.io File]
- [java.nio.file Path]))
+ [metabase-enterprise.serialization.v2.protocols :as protocols]))
(set! *warn-on-reflection* true)
-(def ^:private max-label-length 100)
-(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4
-
-(defn slugify-name
- "Slugify a name for use as a file or directory name: lowercase, replace special chars with underscores,
- preserve dots and unicode, escape slashes. Truncated for filesystem safety."
- [^String s]
- (when (seq s)
- (-> s
- u/lower-case-en
- (str/replace "\\" "__BACKSLASH__")
- (str/replace "/" "__SLASH__")
- (str/replace #"[^\p{L}\p{N}_.]" "_")
- (u.str/limit-bytes max-label-bytes)
- (u.str/limit-chars max-label-length))))
-
-(defn- resolve-path
- "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings
- with deduplication per folder. Uses `unique-name-fns` atom `{resolved-parent-path -> unique-name-fn}` where
- each fn is a `lib/non-truncating-unique-name-generator`.
- Keyed on the full resolved parent path to avoid cross-contamination between unrelated directories
- (e.g. `collections/transforms/` vs `databases/.../schemas/transforms/`)."
- [unique-name-fns path]
- (loop [remaining path
- resolved []]
- (if (empty? remaining)
- resolved
- (let [{:keys [label key]} (first remaining)
- slug (slugify-name label)
- gen (or (get @unique-name-fns resolved)
- (let [g (lib/non-truncating-unique-name-generator)]
- (swap! unique-name-fns assoc resolved g)
- g))
- unique-name (gen key slug)]
- (recur (rest remaining)
- (conj resolved unique-name))))))
-
-(defn resolve-storage-path
- "Given ctx and entity, returns a vector of resolved (slugified, deduplicated) path strings.
- The last element is the filename (without extension)."
- [ctx entity]
- (resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))
-
-(defn- file ^File [ctx entity]
- (let [resolved (resolve-storage-path ctx entity)
- dirnames (drop-last resolved)
- basename (str (last resolved) ".yaml")]
- (apply io/file (:root-dir ctx) (concat dirnames [basename]))))
-
-(defn- store-entity! [opts entity]
- (let [f (file opts entity)]
- (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))
- :file (str (.relativize (Path/of (str (:root-dir opts)) (make-array String 0))
- (.toPath f)))})
- (spit-yaml! f entity)
- (:serdes/meta entity)))
-
-(defn- store-settings! [{:keys [root-dir]} settings]
- (when (seq settings)
- (let [as-map (into (sorted-map)
- (for [{:keys [key value]} settings]
- [key value]))]
- (spit-yaml! (io/file root-dir "settings.yaml") as-map))))
-
(defn store!
- "Helper for storing a serialized database to a tree of YAML files."
- [stream root-dir]
+ "Consume the entity `stream` and store each entity via the given `writer`.
+ Returns a report map with `:seen` and `:errors` vectors."
+ [stream writer]
(let [settings (atom [])
- report (atom {:seen [] :errors []})
- opts (-> (serdes/storage-base-context)
- (assoc :root-dir root-dir))]
+ report (atom {:seen [] :errors []})]
(doseq [entity stream]
(cond
(instance? Exception entity)
@@ -94,8 +19,8 @@
(swap! settings conj entity)
:else
- (swap! report update :seen conj (store-entity! opts entity))))
+ (swap! report update :seen conj (protocols/store-entity! writer entity))))
(when (seq @settings)
- (store-settings! opts @settings)
+ (protocols/store-settings! writer @settings)
(swap! report update :seen conj [{:model "Setting"}]))
@report))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj
new file mode 100644
index 00000000000..92bdff665de
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj
@@ -0,0 +1,46 @@
+(ns metabase-enterprise.serialization.v2.storage.files
+ "Filesystem storage backend — writes YAML files to a directory tree."
+ (:require
+ [clojure.java.io :as io]
+ [metabase-enterprise.serialization.dump :refer [spit-yaml!]]
+ [metabase-enterprise.serialization.v2.protocols :as protocols]
+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]
+ [metabase.models.serialization :as serdes]
+ [metabase.util.log :as log])
+ (:import
+ [java.io File]
+ [java.nio.file Path]))
+
+(set! *warn-on-reflection* true)
+
+(defn- file ^File [ctx root-dir entity]
+ (let [resolved (storage.util/resolve-storage-path ctx entity)
+ dirnames (drop-last resolved)
+ basename (str (last resolved) ".yaml")]
+ (apply io/file root-dir (concat dirnames [basename]))))
+
+(defn file-writer
+ "Create a filesystem storage backend rooted at `root-dir`."
+ [root-dir]
+ (let [ctx (serdes/storage-base-context)]
+ (reify protocols/ExportWriter
+ (store-entity! [_ entity]
+ (let [f (file ctx root-dir entity)]
+ (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))
+ :file (str (.relativize (Path/of (str root-dir) (make-array String 0))
+ (.toPath f)))})
+ (spit-yaml! f entity)
+ (:serdes/meta entity)))
+
+ (store-settings! [_ settings]
+ (when (seq settings)
+ (let [as-map (into (sorted-map)
+ (for [{:keys [key value]} settings]
+ [key value]))]
+ (spit-yaml! (io/file root-dir "settings.yaml") as-map))))
+
+ (store-log! [_ _content]
+ nil)
+
+ (finish! [_]
+ nil))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj
new file mode 100644
index 00000000000..25494d75383
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj
@@ -0,0 +1,63 @@
+(ns metabase-enterprise.serialization.v2.storage.tar
+ "Streaming tar.gz storage backend — writes entries directly to a TarArchiveOutputStream."
+ (:require
+ [clojure.string :as str]
+ [metabase-enterprise.serialization.dump :refer [yaml-content]]
+ [metabase-enterprise.serialization.v2.protocols :as protocols]
+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]
+ [metabase.models.serialization :as serdes]
+ [metabase.util.log :as log])
+ (:import
+ [java.io OutputStream]
+ [org.apache.commons.compress.archivers.tar TarArchiveEntry TarArchiveOutputStream]
+ [org.apache.commons.compress.compressors.gzip GzipCompressorOutputStream GzipParameters]))
+
+(set! *warn-on-reflection* true)
+
+(defn- put-entry!
+ "Write a single entry to a TarArchiveOutputStream."
+ [^TarArchiveOutputStream tar-out ^String entry-path ^bytes content]
+ (let [entry (doto (TarArchiveEntry. entry-path)
+ (.setSize (alength content)))]
+ (.putArchiveEntry tar-out entry)
+ (.write tar-out content)
+ (.closeArchiveEntry tar-out)))
+
+(defn- entry-path
+ "Build the tar entry path: dirname/resolved.../name.yaml"
+ ^String [dirname resolved]
+ (str dirname "/" (str/join "/" (drop-last resolved)) "/" (last resolved) ".yaml"))
+
+(defn tar-writer
+ "Create a streaming tar.gz storage backend writing to `output`."
+ [^OutputStream output ^String dirname]
+ (let [gzip (GzipCompressorOutputStream. output (doto (GzipParameters.)
+ (.setModificationTime (System/currentTimeMillis))))
+ tar (TarArchiveOutputStream. gzip 512 "UTF-8")
+ ctx (serdes/storage-base-context)]
+ (.setLongFileMode tar TarArchiveOutputStream/LONGFILE_POSIX)
+ (.setBigNumberMode tar TarArchiveOutputStream/BIGNUMBER_POSIX)
+ (reify protocols/ExportWriter
+ (store-entity! [_ entity]
+ (let [resolved (storage.util/resolve-storage-path ctx entity)
+ path (entry-path dirname resolved)
+ content (.getBytes ^String (yaml-content entity) "UTF-8")]
+ (log/trace "Storing" {:path (serdes/log-path-str (:serdes/meta entity))
+ :file path})
+ (put-entry! tar path content)
+ (:serdes/meta entity)))
+
+ (store-settings! [_ settings]
+ (when (seq settings)
+ (let [as-map (into (sorted-map)
+ (for [{:keys [key value]} settings]
+ [key value]))
+ content (.getBytes ^String (yaml-content as-map) "UTF-8")]
+ (put-entry! tar (str dirname "/settings.yaml") content))))
+
+ (store-log! [_ content]
+ (put-entry! tar (str dirname "/export.log") content))
+
+ (finish! [_]
+ (.finish tar)
+ (.finish gzip)))))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj
new file mode 100644
index 00000000000..0774eff3303
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj
@@ -0,0 +1,49 @@
+(ns metabase-enterprise.serialization.v2.storage.util
+ (:require
+ [clojure.string :as str]
+ [metabase.lib.core :as lib]
+ [metabase.models.serialization :as serdes]
+ [metabase.util :as u]
+ [metabase.util.string :as u.str]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private max-label-length 100)
+(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4
+
+(defn slugify-name
+ "Slugify a name for use as a file or directory name: lowercase, replace special chars with underscores,
+ preserve dots and unicode, escape slashes. Truncated for filesystem safety."
+ [^String s]
+ (when (seq s)
+ (-> s
+ u/lower-case-en
+ (str/replace "\\" "__BACKSLASH__")
+ (str/replace "/" "__SLASH__")
+ (str/replace #"[^\p{L}\p{N}_.]" "_")
+ (u.str/limit-bytes max-label-bytes)
+ (u.str/limit-chars max-label-length))))
+
+(defn- resolve-path
+ "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings
+ with deduplication per folder."
+ [unique-name-fns path]
+ (loop [remaining path
+ resolved []]
+ (if (empty? remaining)
+ resolved
+ (let [{:keys [label key]} (first remaining)
+ slug (slugify-name label)
+ gen (or (get @unique-name-fns resolved)
+ (let [g (lib/non-truncating-unique-name-generator)]
+ (swap! unique-name-fns assoc resolved g)
+ g))
+ unique-name (gen key slug)]
+ (recur (rest remaining)
+ (conj resolved unique-name))))))
+
+(defn resolve-storage-path
+ "Given ctx and entity, returns a vector of resolved (slugified, deduplicated) path strings.
+ The last element is the filename (without extension)."
+ [ctx entity]
+ (resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
index b68909f93a7..884da4a7a22 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj
@@ -26,6 +26,27 @@
(GzipCompressorInputStream.)
(TarArchiveInputStream.)))
+(defn- entry-names
+ "Return a seq of all entry names in the given tar.gz response."
+ [f]
+ (with-open [tar (open-tar f)]
+ (mapv (fn [^TarArchiveEntry e] (.getName e)) (u.compress/entries tar))))
+
+(defn- first-entry-name
+ "Return the name of the first entry in the given tar.gz response."
+ [f]
+ (first (entry-names f)))
+
+(defn- read-export-log
+ "Extract the export.log content from a tar.gz response."
+ [f]
+ (with-open [tar (open-tar f)]
+ (loop []
+ (when-let [e (.getNextEntry tar)]
+ (if (str/ends-with? (.getName ^TarArchiveEntry e) "export.log")
+ (slurp (io/reader tar))
+ (recur))))))
+
(def ^:private file-types
[#"([^/]+)?/$" :dir
#"/settings.yaml$" :settings
@@ -44,13 +65,6 @@
[ftype (when (vector? m) (second m))]))
(partition 2 file-types)))
-(defn- log-types
- "Find out entity type by log message"
- [lines]
- (->> lines
- (keep #(second (re-find #"(?:Extracting|Loading|Storing) \{:path (\w+)" %)))
- set))
-
(defn- tar-file-types [f & [raw?]]
(with-open [tar (open-tar f)]
(cond->> (u.compress/entries tar)
@@ -117,13 +131,13 @@
(testing "API respects parameters"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:all_collections false :data_model false :settings true)]
- (is (= #{:log :dir :settings :transform :python-library}
+ (is (= #{:log :settings :transform :python-library}
(tar-file-types f)))))
(testing "We can export just a single collection"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :data_model false :settings false)]
- (is (= #{:log :dir :collection-entity :transform :python-library}
+ (is (= #{:log :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "We can export two collections"
@@ -138,52 +152,44 @@
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
;; eid:... syntax is kept for backward compat
:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]
- (is (= #{:log :dir :collection-entity :transform :python-library}
+ (is (= #{:log :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "We can export that collection using entity id"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:entity_id coll) :data_model false :settings false)]
- (is (= #{:log :dir :collection-entity :transform :python-library}
+ (is (= #{:log :collection-entity :transform :python-library}
(tar-file-types f)))))
(testing "Default export: all-collections, data-model, settings"
(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]
- (is (= #{:transform :log :dir :collection-entity :settings :schema :database :python-library}
+ (is (= #{:transform :log :collection-entity :settings :schema :database :python-library}
(tar-file-types f)))))
- (testing "On exception API returns log"
+ (testing "On exception API returns tar.gz with error in export.log"
(mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)
(mt/dynamic-value serdes/extract-one))]
(let [res (binding [api.serialization/*additive-logging* false]
- (mt/user-http-request :crowberto :post 500 "ee/serialization/export" {}
+ (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
:collection (:id coll) :data_model false :settings false))
- log (slurp (io/input-stream res))]
- (testing "In logs we get an entry for the dashboard, then card, and then an error"
- (is (= #{"Dashboard" "Card"}
- (log-types (str/split-lines log))))
- (is (re-find #"deliberate error message" log))
- (is (= {:id "**ID**",
- :entity_id "**ID**",
- :model "Card",
- :table :report_card
- :cause "[test] deliberate error message"}
- (extract-and-sanitize-exception-map log)))))))
-
- (testing "You can pass specific directory name"
- (let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
- :dirname "check" :all_collections false :data_model false :settings false)]
- (is (= "check/"
- (with-open [tar (open-tar f)]
- (.getName ^TarArchiveEntry (first (u.compress/entries tar))))))))
-
- (testing "Invalid entity ID returns an error instead of falling back to root collection"
- (let [fake-eid "abcdefghijklmnopqrstu"
- res (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}
- :collection fake-eid :data_model false :settings false)
- log (slurp (io/input-stream res))]
- (is (re-find #"Could not find Collection with entity ID" log))
- (is (re-find #"abcdefghijklmnopqrstu" log))))))))
+ log (read-export-log res)]
+ (testing "export.log inside the archive contains error details"
+ (is (some? log) "export.log should be present in the archive")
+ (is (re-find #"deliberate error message" log)))))))
+
+ (testing "You can pass specific directory name"
+ (let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}
+ :dirname "check" :all_collections false :data_model false :settings false)]
+ (is (str/starts-with? (first-entry-name f) "check/")))))
+
+ (testing "Invalid entity ID returns an error instead of falling back to root collection"
+ (let [fake-eid "abcdefghijklmnopqrstu"
+ res (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}
+ :collection fake-eid :data_model false :settings false)]
+ (is (re-find #"Could not find Collection with entity ID"
+ (str (:message res))))
+ (is (re-find #"abcdefghijklmnopqrstu"
+ (str (:message res))))))))
(testing "We've left no new files, every request is cleaned up"
;; if this breaks, check if you consumed every response with io/input-stream
(is (= known-files
@@ -214,19 +220,8 @@
:collection (:id coll) :data_model false :settings false)
io/input-stream)
ba (#'api.serialization/ba-copy res)]
- (testing "Archive contains correct number of files with proper log entries"
- (is (= 13
- (with-open [tar (open-tar ba)]
- (count
- (for [^TarArchiveEntry e (u.compress/entries tar)
- :when (.isFile e)]
- (do
- (condp re-find (.getName e)
- #"/export.log$" (testing "Log contains extract and store entries"
- (is (= (+ #_extract 12 #_store 12)
- (count (line-seq (io/reader tar))))))
- nil)
- (.getName e))))))))
+ (testing "Archive contains correct number of files"
+ (is (= 13 (count (filter #(not (str/ends-with? % "/")) (entry-names ba))))))
(testing "Snowplow export event was sent"
(is (=? {"event" "serialization"
@@ -262,14 +257,10 @@
(t2/delete! :model/Card (:id card))
(let [re-indexed? (atom false)
- res (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
+ _res (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]
(mt/user-http-request :crowberto :post 200 "ee/serialization/import?reindex=false"
{:request-options {:headers {"content-type" "multipart/form-data"}}}
{:file ba}))]
- (testing "Log contains imported entity types"
- (is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}
- (log-types (line-seq (io/reader (io/input-stream res)))))))
-
(testing "Entities are restored in the database"
(is (= (:name dash) (t2/select-one-fn :name :model/Dashboard :entity_id (:entity_id dash))))
(is (= (:name card) (t2/select-one-fn :name :model/Card :entity_id (:entity_id card)))))
@@ -342,9 +333,7 @@
{:file ba}
:continue_on_error true)
log (slurp (io/input-stream res))]
- (testing "Log shows loaded entities and error"
- (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}
- (log-types (str/split-lines log))))
+ (testing "Log contains the missing-collection error"
(is (re-find #"Collection 'DoesNotExist' was not found" log)))
(testing "Snowplow event shows partial success with error count"
@@ -368,15 +357,16 @@
(is (re-find #"Cannot unpack archive" log))))))
(deftest export-extraction-error-test
- (testing "Export fails with 500 when entity extraction fails"
+ (testing "Export with error still returns tar.gz with export.log containing error details"
(with-serialization-test-data! [coll _dash card]
(mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)
(mt/dynamic-value serdes/extract-one))]
- (testing "Error response contains exception details"
+ (testing "Error details are in export.log inside the archive"
(binding [api.serialization/*additive-logging* false]
- (let [res (mt/user-http-request :crowberto :post 500 "ee/serialization/export"
+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export"
:collection (:id coll) :data_model false :settings false)
- log (slurp (io/input-stream res))]
+ log (read-export-log res)]
+ (is (some? log) "export.log should be present in the archive")
(is (= {:id "**ID**"
:entity_id "**ID**"
:model "Card"
@@ -400,12 +390,12 @@
"error_message" #"(?s)Error extracting Card \d+ .*"}
(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))
- (testing "full_stacktrace parameter includes full stack trace"
+ (testing "full_stacktrace parameter includes full stack trace in export.log"
(binding [api.serialization/*additive-logging* false]
- (let [res (mt/user-http-request :crowberto :post 500 "ee/serialization/export"
+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export"
:collection (:id coll) :data_model false :settings false
:full_stacktrace true)
- log (slurp (io/input-stream res))]
+ log (read-export-log res)]
(is (< 50 (count (str/split-lines log))))
;; Pop out the error event
(snowplow-test/pop-event-data-and-user-id!))))))))
@@ -415,17 +405,11 @@
(with-serialization-test-data! [coll _dash card]
(mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)
(mt/dynamic-value serdes/extract-one))]
- (let [res (-> (mt/user-http-request :crowberto :post 200 "ee/serialization/export"
- :collection (:id coll) :data_model false :settings false
- :continue_on_error true)
- io/input-stream)]
- (with-open [tar (open-tar res)]
- (doseq [^TarArchiveEntry e (u.compress/entries tar)]
- (condp re-find (.getName e)
- #"/export.log$" (testing "Log shows extract entries, error, and store entries"
- (is (= (+ #_extract 12 #_error 1 #_store 11)
- (count (line-seq (io/reader tar))))))
- nil))))
+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export"
+ :collection (:id coll) :data_model false :settings false
+ :continue_on_error true)]
+ (testing "Log contains the deliberate error"
+ (is (re-find #"deliberate error message" (read-export-log res)))))
(testing "Snowplow event shows partial success with error count"
(is (=? {"event" "serialization"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
index d61019730b3..d06192da72b 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj
@@ -65,8 +65,8 @@
"error_message" nil}
(-> (snowplow-test/pop-event-data-and-user-id!) first :data))))
- (with-redefs [v2.storage/store-settings! (fn [_opts _settings]
- (throw (Exception. "Cannot load settings")))]
+ (with-redefs [v2.storage/store! (fn [_stream _backend]
+ (throw (Exception. "Cannot load settings")))]
(is (thrown? Exception
(cmd/export dump-dir "--collection" (str (:id coll)) "--no-data-model")))
(testing "Snowplow export event about error was sent"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 122e1ed35f1..62e720a28a3 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -10,6 +10,7 @@
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.serialization.v2.load :as serdes.load]
[metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.settings.core :as setting]
@@ -211,7 +212,7 @@
(is (= 110 (-> @entities (get "Collection") count))))
(testing "storage"
- (storage/store! (seq @extraction) dump-dir)
+ (storage/store! (seq @extraction) (storage.files/file-writer dump-dir))
(testing "for Actions"
(is (= 30 (count (dir->file-set (io/file dump-dir "actions"))))))
@@ -447,7 +448,7 @@
:values_source_type :card}]}
(set (map :parameters (by-model extraction "Card")))))
- (storage/store! (seq extraction) dump-dir)))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))
(testing "ingest and load"
(ts/with-db dest-db
@@ -555,7 +556,7 @@
{:model "Table" :id "Linked table"}]}
(set (serdes/dependencies extracted-dashboard))))
- (storage/store! (seq extraction) dump-dir)))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))
(testing "ingest and load"
;; ingest
@@ -615,7 +616,7 @@
hydrated-dashcards))))
(testing "extract and store"
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
- (storage/store! (seq extraction) dump-dir)))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))
(testing "ingest and load"
(ts/with-db dest-db
(testing "doing ingestion"
@@ -672,7 +673,7 @@
:card_id card-id-2
:dashboard_tab_id tab-id-2}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
- (storage/store! (seq extraction) dump-dir))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
@@ -725,7 +726,7 @@
:model/Card {card-eid :entity_id} {:name "Card on the Dashboard"
:dashboard_id dashboard-id}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
- (storage/store! (seq extraction) dump-dir))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
@@ -754,7 +755,7 @@
:model/Database _ {:router_database_id router-db-id
:name "Destination"}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
- (storage/store! (seq extraction) dump-dir))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
(testing "doing ingestion"
@@ -837,7 +838,7 @@
:products.title %products.title
:orders.product_id %orders.product_id})
(reset! card1s card)
- (storage/store! (extract/extract {}) dump-dir)))))
+ (storage/store! (extract/extract {}) (storage.files/file-writer dump-dir))))))
(ts/with-db dest-db
;; ensure there is something in db so that test-data gets different field ids for sure
@@ -876,7 +877,7 @@
(let [coll (ts/create! :model/Collection :name "coll")
_ (ts/create! :model/Card :name "card" :collection_id (:id coll))]
(storage/store! (extract/extract {:no-settings true
- :no-data-model true}) dump-dir)
+ :no-data-model true}) (storage.files/file-writer dump-dir))
(spit (io/file dump-dir "collections" ".hidden.yaml") "serdes/meta: [{do-not: read}]")
@@ -900,7 +901,7 @@
(let [coll (ts/create! :model/Collection :name "coll")
_ (ts/create! :model/Card :name "card" :collection_id (:id coll))]
(storage/store! (extract/extract {:no-settings true
- :no-data-model true}) dump-dir)
+ :no-data-model true}) (storage.files/file-writer dump-dir))
(spit (io/file dump-dir "collections" "corrupt.yaml") "\0")
(testing "continue-on-error false (default) — throws on ingestion errors"
@@ -927,7 +928,7 @@
:type :channel/http
:details {:url "http://example.com"
:auth-method :none}}]
- (storage/store! (seq (serdes/with-cache (into [] (extract/extract {})))) dump-dir)
+ (storage/store! (seq (serdes/with-cache (into [] (extract/extract {})))) (storage.files/file-writer dump-dir))
(ts/with-db dest-db
(testing "doing ingestion"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
@@ -956,7 +957,7 @@
{:aggregation [[:metric metric-id]]
:breakout [[:field %orders.user_id nil]]})}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
- (storage/store! (seq extraction) dump-dir))
+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
(ts/with-db dest-db
(testing "doing ingestion"
@@ -988,7 +989,7 @@
(serdes/with-cache
(-> (extract/extract {:no-settings true
:no-data-model true})
- (storage/store! dump-dir))))
+ (storage/store! (storage.files/file-writer dump-dir)))))
(testing "loads well too"
(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))
"ingested successfully"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index 681426be8b9..71d5a2aa3b1 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -32,6 +32,7 @@
[metabase-enterprise.serialization.v2.load :as load]
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.search.test-util :as search.tu]
@@ -108,7 +109,7 @@
;; Use a separate cache to make sure there is no cross-contamination.
(serdes/with-cache
(-> (extract/extract {:include-field-values true :include-metabot true})
- (storage/store! output-dir))))
+ (storage/store! (storage.files/file-writer output-dir)))))
(defn- delete-dir-contents! [^File dir]
(when (and dir (.exists dir))
@@ -139,7 +140,7 @@
(defn add-to-baseline!
"Use this within v2.extract-test where relevant to add their fixtures to the baseline."
[]
- (storage/store! (into [] (extract/extract {:include-field-values true :include-metabot true})) source-dir))
+ (storage/store! (into [] (extract/extract {:include-field-values true :include-metabot true})) (storage.files/file-writer source-dir)))
;; If this test is failing, read the docstring at the top of this namespace for what to do B-)
(deftest baseline-completeness-test
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 8271fa5dfb3..7dd231c85ba 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -8,6 +8,8 @@
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as extract]
[metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]
+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]
[metabase.models.serialization :as serdes]
[metabase.test :as mt]
[metabase.util.yaml :as yaml]
@@ -28,7 +30,7 @@
(ts/with-temp-dpc [:model/Collection parent {:name "Some Collection"}
:model/Collection child {:name "Child Collection" :location (format "/%d/" (:id parent))}]
(let [export (into [] (extract/extract {:no-transforms true}))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
(testing "the right files in the right places"
(is (= #{["main" "some_collection.yaml"]
["main" "some_collection" "child_collection.yaml"]}
@@ -76,7 +78,7 @@
:model/Card _c4 {:name "child card" :collection_id (:id child)}
:model/Dashboard _d1 {:name "parent dash" :collection_id (:id parent)}]
(let [export (into [] (extract/extract {:no-transforms true}))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
(testing "the right files in the right places"
(is (= #{["main" "grandparent_collection.yaml"]
["main" "grandparent_collection" "parent_collection.yaml"]
@@ -107,7 +109,7 @@
(let [export (into [] (extract/extract {:no-settings true
:no-data-model true
:no-transforms true}))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
(testing "all snippet collections and snippets under collections/snippets/"
(is (= #{["snippets" "grandparent_collection.yaml"]
["snippets" "grandparent_collection" "parent_collection.yaml"]
@@ -127,7 +129,7 @@
:model/FieldValues _ {:field_id (:id website)}
:model/Table _ {:name "Orders/Invoices" :db_id (:id db)}]
(let [export (into [] (extract/extract {:include-field-values true}))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
(testing "the right files in the right places"
(is (= #{["company__SLASH__organization_website.yaml"]
["company__SLASH__organization_website___fieldvalues.yaml"]}
@@ -196,7 +198,7 @@
(if (str/ends-with? fname "settings.yaml")
(descend coll [:settings])
(descend coll)))))]
- (storage/store! export dump-dir)))))))
+ (storage/store! export (storage.files/file-writer dump-dir))))))))
(deftest store-error-test
(mt/with-empty-h2-app-db!
@@ -208,7 +210,7 @@
(.setWritable (io/file parent-dir) false)
(is (thrown-with-msg? Exception #"Destination path is not writeable: "
(storage/store! [{:serdes/meta [{:model "A" :id "B"}]}]
- dump-dir))))
+ (storage.files/file-writer dump-dir)))))
(testing "directory exists but is not writable"
(.setWritable (io/file parent-dir) true)
(.mkdirs (io/file dump-dir))
@@ -216,7 +218,7 @@
(.setWritable (io/file dump-dir) false)
(is (thrown-with-msg? Exception #"Destination path is not writeable: "
(storage/store! [{:serdes/meta [{:model "A" :id "B"}]}]
- dump-dir)))))))))
+ (storage.files/file-writer dump-dir))))))))))
(deftest nested-fields-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
@@ -227,7 +229,7 @@
_f2 (ts/create! :model/Field :name "child" :table_id (:id t) :parent_id (:id f1))]
(serdes/with-cache
(-> (extract/extract {:no-settings true})
- (storage/store! dump-dir)))
+ (storage/store! (storage.files/file-writer dump-dir))))
(testing "we get correct names for nested fields"
(is (= #{["parent.yaml"]
["parent.child.yaml"]}
@@ -239,7 +241,7 @@
(t2/delete! :model/PythonLibrary)
(ts/with-temp-dpc [:model/PythonLibrary _lib {:path "common" :source "def test(): pass"}]
(let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
(testing "python library stored at top-level python_libraries/"
(is (= #{["common.py.yaml"]}
(file-set (io/file dump-dir "python_libraries"))))))))))
@@ -255,7 +257,7 @@
:targets [["Card" (:id card)]]}))
;; 66 is 'char-count * max-bytes / byte-count'
card-filename (str/join (repeat 66 "ป"))]
- (storage/store! export dump-dir)
+ (storage/store! export (storage.files/file-writer dump-dir))
;; we could also test loading here, but file names do not play significant part in how everything's loaded,
;; `:serdes/meta` does and that one is not shortened or anything
(testing "the right files in the right places"
@@ -264,7 +266,7 @@
"collections form a tree, with same-named files")))))))
(deftest ^:parallel resolve-path-test
- (let [resolve-path @#'storage/resolve-path]
+ (let [resolve-path @#'storage.util/resolve-path]
(testing "basic slugification"
(let [fns (atom {})]
(is (= ["my_collection" "some_card"]
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 3e476c235b4..4bd7a0dc436 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -426,7 +426,7 @@
:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.
:legacy_query]
:transform {:action_id (serdes/parent-ref)
- :database_id (serdes/fk :model/Database :name)
+ :database_id (serdes/fk :model/Database)
:dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})
(defmethod serdes/generate-path "HTTPAction" [_ _] nil)
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 26413f32c7a..768fe0f1a26 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -35,7 +35,7 @@
## Existing transformations
- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way
+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way
- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export
- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`
in that case)
@@ -450,6 +450,11 @@
(defmethod make-spec :default [_ _] nil)
+(defn ^:dynamic ^::cache *make-spec*
+ "Cachable wrapper around [[make-spec]] that is memoized inside [[with-cache]]."
+ [model-name opts]
+ (make-spec model-name opts))
+
(defmulti extract-all
"Entry point for extracting all entities of a particular model:
`(extract-all \"ModelName\" {opts...})`
@@ -495,7 +500,7 @@
- Replace any foreign keys with portable values (eg. entity IDs, or a user ID with their email, etc.)"
[model-name opts instance]
(try
- (let [spec (make-spec model-name opts)]
+ (let [spec (*make-spec* model-name opts)]
(assert spec (str "No serialization spec defined for model " model-name))
(-> (into {}
(remove (fn [[k v]] (= v (get-in spec [:defaults k]))))
@@ -527,7 +532,7 @@
(defn log-and-extract-one
"Extracts a single entity; will replace `extract-one` as public interface once `extract-one` overrides are gone."
[model opts instance]
- (log/info "Extracting" {:path (log-path-str (generate-path model instance))})
+ (log/tracef "Extracting %s" (log-path-str (generate-path model instance)))
(try
(extract-one model opts instance)
(catch Exception e
@@ -557,7 +562,7 @@
(group-by backward-fk entities)))
(defn- extract-batch-nested [model-name opts batch]
- (let [spec (make-spec model-name opts)]
+ (let [spec (*make-spec* model-name opts)]
(reduce-kv (fn [batch k transform]
(if-not (::nested transform)
batch
@@ -577,7 +582,7 @@
"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
collection."
[model {:keys [collection-set where] :as opts}]
- (let [spec (make-spec (name model) opts)]
+ (let [spec (*make-spec* (name model) opts)]
(if (or (empty? collection-set)
(nil? (-> spec :transform :collection_id)))
;; either no collections specified or our model has no collection
@@ -591,7 +596,7 @@
where)]}))))
(defmethod extract-query :default [model-name opts]
- (let [spec (make-spec model-name opts)
+ (let [spec (*make-spec* model-name opts)
nested? (some ::nested (vals (:transform spec)))]
(cond->> (extract-query-collections (keyword "model" model-name) opts)
nested? (extract-reducible-nested model-name (dissoc opts :where)))))
@@ -795,7 +800,7 @@
schemas))
(defn- xform-one [model-name ingested]
- (let [spec (make-spec model-name nil)]
+ (let [spec (*make-spec* model-name nil)]
(assert spec (str "No serialization spec defined for model " model-name))
(-> (select-keys ingested (:copy spec))
(into (for [[k transform] (:transform spec)
@@ -815,7 +820,7 @@
(coerce-keys (:coerce spec)))))
(defn- spec-nested! [model-name ingested instance]
- (let [spec (make-spec model-name nil)]
+ (let [spec (*make-spec* model-name nil)]
(doseq [[k transform] (:transform spec)
:when (and (::nested transform)
;; handling circuit-breaking
@@ -1032,7 +1037,8 @@
"Given a numeric database ID, return its name as a portable reference.
[[*import-database-fk*]] is the inverse."
[id]
- (*export-fk-keyed* id :model/Database :name))
+ (when id
+ (t2/select-one-fn :name [:model/Database :id :name] :id id)))
(defn ^:dynamic ^::cache *import-database-fk*
"Given a portable database name, resolve it back to a numeric ID.
@@ -1049,8 +1055,8 @@
[[import-table-fk]] is the inverse."
[table-id :- [:maybe ::lib.schema.id/table]]
(when table-id
- (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)
- db-name (t2/select-one-fn :name :model/Database :id db_id)]
+ (let [{:keys [db_id name schema]} (t2/select-one [:model/Table :id :db_id :name :schema] :id table-id)
+ db-name (*export-database-fk* db_id)]
[db-name schema name])))
(mu/defn ^:dynamic ^::cache *import-table-fk*
@@ -1128,9 +1134,12 @@
[[*import-field-fk*]] is the inverse."
[field-id :- [:maybe ::lib.schema.id/field]]
(when field-id
- (let [fields (field-hierarchy field-id)
- [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]
- (into [db-name schema field-name] (map :name fields)))))
+ (let [{field-name :name :keys [parent_id table_id]} (t2/select-one [:model/Field :name :parent_id :table_id] :id field-id)]
+ (if (nil? parent_id)
+ (conj (*export-table-fk* table_id) field-name)
+ (let [fields (field-hierarchy field-id)
+ [db-name schema table-name] (*export-table-fk* table_id)]
+ (into [db-name schema table-name] (map :name fields)))))))
(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]
"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 520ab07ceb3..8e1122c8408 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1326,7 +1326,7 @@
:legacy_query]
:transform
{:created_at (serdes/date)
- :database_id (serdes/fk :model/Database :name)
+ :database_id (serdes/fk :model/Database)
:table_id (serdes/fk :model/Table)
:source_card_id (serdes/fk :model/Card)
:collection_id (serdes/fk :model/Collection)
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 3a747d21218..60f6501d2c4 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -364,7 +364,7 @@
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
:collection_id (serdes/fk :model/Collection)
- :source_database_id (serdes/fk :model/Database :name)
+ :source_database_id (serdes/fk :model/Database)
:source {:export (fn [source]
(-> source
(m/update-existing :query serdes/export-mbql)
diff --git a/src/metabase/util/compress.clj b/src/metabase/util/compress.clj
index bfafd1e1cf3..4a559a3b0ac 100644
--- a/src/metabase/util/compress.clj
+++ b/src/metabase/util/compress.clj
@@ -56,6 +56,7 @@
:when (not (str/starts-with? actual-name "."))]
(let [f (.toFile (.resolveIn e dst-path))]
(if (.isFile e)
- (io/copy tar f)
+ (do (io/make-parents f)
+ (io/copy tar f))
(.mkdirs f))
true))))))
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 13d314792d6..553ddabcb50 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -420,8 +420,8 @@
;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],
;; a trio of strings with schema maybe nil.
-(defmethod serdes/generate-path "Field" [_ field]
- (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))]
+(defmethod serdes/generate-path "Field" [_ {:keys [id]}]
+ (let [[db schema table & fields] (serdes/*export-field-fk* id)]
(->> (into (serdes/table->path [db schema table])
(map (fn [n] {:model "Field" :id n}) fields))
(filterv some?))))
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index 67c444ab9eb..bf47518bd88 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -41,9 +41,8 @@
(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)
(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]
- (let [field (t2/select-one 'Field :id field_id)]
- (conj (serdes/generate-path "Field" field)
- {:model "FieldUserSettings" :id "1"})))
+ (conj (serdes/generate-path "Field" {:id field_id})
+ {:model "FieldUserSettings" :id "1"}))
(defmethod serdes/dependencies "FieldUserSettings" [fv]
;; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index e7a52abe298..60445e60727 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -571,9 +571,8 @@
(defmethod serdes/entity-id "FieldValues" [_ _] nil)
(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]
- (let [field (t2/select-one 'Field :id field_id)]
- (conj (serdes/generate-path "Field" field)
- {:model "FieldValues" :id "0"})))
+ (conj (serdes/generate-path "Field" {:id field_id})
+ {:model "FieldValues" :id "0"}))
(defmethod serdes/dependencies "FieldValues" [fv]
;; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 3a9322a9318..f34f2516b01 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -708,7 +708,7 @@
:archived_at (serdes/date)
:deactivated_at (serdes/date)
:data_layer (serdes/optional-kw)
- :db_id (serdes/fk :model/Database :name)
+ :db_id (serdes/fk :model/Database)
:collection_id (serdes/fk :model/Collection)
:transform_id (serdes/fk :model/Transform)}
:defaults {:is_defective_duplicate false
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 3584a9eec7a..22368aeac15 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -648,7 +648,7 @@
::serdes/skip))
:import identity}
:creator_id (serdes/fk :model/User)
- :router_database_id (serdes/fk :model/Database :name)
+ :router_database_id (serdes/fk :model/Database)
:initial_sync_status {:export identity :import (constantly "complete")}}
:defaults {:auto_run_queries true
:is_attached_dwh false