The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Trivial serdes export fixes for metadata (#72434)
Alexander Polyankin28f1915be372
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/api.clj b/enterprise/backend/src/metabase_enterprise/serialization/api.cljindex fbc5aea69e8..a51cb93d7e3 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/api.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/api.clj@@ -1,13 +1,14 @@(ns metabase-enterprise.serialization.api(:require- [clojure.core.async :as a][clojure.java.io :as io][clojure.string :as str][java-time.api :as t][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase-enterprise.serialization.v2.load :as v2.load]- [metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.protocols :as v2.protocols]+ [metabase-enterprise.serialization.v2.storage :as v2.storage]+ [metabase-enterprise.serialization.v2.storage.tar :as v2.storage.tar][metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros]@@ -19,7 +20,6 @@[metabase.util :as u][metabase.util.compress :as u.compress][metabase.util.date-2 :as u.date]- [metabase.util.jvm :as u.jvm][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -68,43 +68,34 @@;;; Logic-(defn- serialize&pack ^File [{:keys [dirname full-stacktrace] :as opts}]- (let [dirname (or dirname- (format "%s-%s"- (u/slugify (appearance/site-name))- (u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))- path (io/file parent-dir dirname)- dst (io/file (str (.getPath path) ".tar.gz"))- log-file (io/file path "export.log")- err (atom nil)- report (with-open [_logger (logger/for-ns log-file ['metabase-enterprise.serialization- 'metabase.models.serialization]- {:additive *additive-logging*})]- (try ; try/catch inside logging to log errors- (let [report (serdes/with-cache- (-> (extract/extract opts)- (storage/store! path)))]- ;; not removing dumped yamls immediately to save some time before response- (u.compress/tgz path dst)- report)- (catch Exception e- (reset! err e)- (if full-stacktrace- (log/error e "Error during serialization export")- (log/error (u/strip-error e "Error during serialization export"))))))]- {:archive (when (.exists dst)- dst)- :log-file (when (.exists log-file)- log-file)- :report report- :status (:status-code (ex-data @err))- :error-message (when @err- (u/strip-error @err nil))- :callback (fn []- (when (.exists path)- (run! io/delete-file (reverse (file-seq path))))- (when (.exists dst)- (io/delete-file dst)))}))+(defn- serialize-to-stream!+ "Serialize directly to an OutputStream as streaming tar.gz. Returns result map."+ [^java.io.OutputStream output ^String dirname entities {:keys [full-stacktrace]}]+ (let [log-output (ByteArrayOutputStream.)+ writer (v2.storage.tar/tar-writer output dirname)+ error (atom nil)+ report (with-open [_logger (logger/for-ns log-output ['metabase-enterprise.serialization+ 'metabase.models.serialization]+ {:additive *additive-logging*})]+ (try+ (let [report (serdes/with-cache+ (v2.storage/store! entities writer))]+ (v2.protocols/store-log! writer (.toByteArray log-output))+ (v2.protocols/finish! writer)+ report)+ (catch Exception e+ (reset! error e)+ (if full-stacktrace+ (log/error e "Error during serialization export")+ (log/error (u/strip-error e "Error during serialization export")))+ (try+ (v2.protocols/store-log! writer (.toByteArray log-output))+ (v2.protocols/finish! writer)+ (catch Exception _)))))]+ {:report report+ :success (nil? @error)+ :error-message (when @error+ (u/strip-error @error nil))}))(defn- find-serialization-dir"Find an actual top-level dir with serialization data inside, instead of picking up various .DS_Store and similar@@ -159,31 +150,23 @@:callback #(when (.exists dst)(run! io/delete-file (reverse (file-seq dst))))}))-(defn- serialization-api-handler- [collection result-atom opts]- (let [done-chan (a/chan)]- [(u.jvm/in-virtual-thread*- (let [start (System/nanoTime)- {:keys [archive report error-message] :as pack-result} (serialize&pack opts)]- (analytics/track-event! :snowplow/serialization- {:event :serialization- :direction "export"- :source "api"- :duration_ms (int (/ (- (System/nanoTime) start) 1e6))- :count (count (:seen report))- :error_count (count (:errors report))- :collection (str/join "," (map str collection))- :all_collections (and (empty? collection)- (not (:no-collections opts)))- :data_model (not (:no-data-model opts))- :settings (not (:no-settings opts))- :field_values (:include-field-values opts)- :secrets (:include-database-secrets opts)- :success (boolean archive)- :error_message error-message})- (reset! result-atom pack-result)- (a/>!! done-chan :done)))- done-chan]))+(defn- track-export-event! [collection opts start {:keys [report success error-message]}]+ (analytics/track-event! :snowplow/serialization+ {:event :serialization+ :direction "export"+ :source "api"+ :duration_ms (int (/ (- (System/nanoTime) start) 1e6))+ :count (count (:seen report))+ :error_count (count (:errors report))+ :collection (str/join "," (map str collection))+ :all_collections (and (empty? collection)+ (not (:no-collections opts)))+ :data_model (not (:no-data-model opts))+ :settings (not (:no-settings opts))+ :field_values (:include-field-values opts)+ :secrets (:include-database-secrets opts)+ :success (boolean success)+ :error_message error-message}));;; HTTP API@@ -239,33 +222,21 @@:no-settings (not settings?):include-field-values include-field-values?:include-database-secrets include-database-secrets?- :dirname dirname:continue-on-error continue-on-error?:full-stacktrace full-stacktrace?}- result (atom nil)- [archive-thread finished-chan] (serialization-api-handler collection result opts)]- (sr/streaming-response {:content-type "application/gzip" :status 200} [os cancel-chan]- (a/<!!- (a/go- (case (a/alt! cancel-chan :cancel finished-chan :done)- :cancel (do- (future-cancel archive-thread)- (when-let [cb (:callback @result)]- (cb))- (.close os))- :done (let [{:keys [archive status log-file callback]} @result]- (try- (if archive- (do- (sr/set-header! "Content-Disposition"- (format "attachment; filename=\"%s\"" (.getName ^File archive)))- (io/copy archive os))- (do- (sr/set-status! (or status 500))- (sr/set-content-type! "text/plain")- (io/copy log-file os)))- (finally- (callback))))))))))+ export-dirname (or dirname+ (format "%s-%s"+ (u/slugify (appearance/site-name))+ (u.date/format "YYYY-MM-dd_HH-mm" (t/local-date-time))))+ ;; extract/extract runs eager setup (target resolution, escape analysis) which can throw+ ;; for invalid inputs (e.g. bad collection ID). This must happen before streaming starts.+ entities (extract/extract opts)]+ (sr/streaming-response {:content-type "application/gzip" :status 200} [output _cancel-chan]+ (sr/set-header! "Content-Disposition"+ (format "attachment; filename=\"%s.tar.gz\"" export-dirname))+ (let [start (System/nanoTime)+ result (serialize-to-stream! output export-dirname entities opts)]+ (track-export-event! collection opts start result)))));; TODO (Cam 10/28/25) -- fix this endpoint so it uses kebab-case for query parameters for consistency with the rest;; of the REST APIdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.cljindex 5bf58202df6..9f3615724ed 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj@@ -8,6 +8,7 @@[metabase-enterprise.serialization.v2.ingest :as v2.ingest][metabase-enterprise.serialization.v2.load :as v2.load][metabase-enterprise.serialization.v2.storage :as v2.storage]+ [metabase-enterprise.serialization.v2.storage.files :as v2.storage.files][metabase.analytics.core :as analytics][metabase.app-db.core :as mdb][metabase.events.core :as events]@@ -113,7 +114,7 @@report (try(serdes/with-cache(-> (v2.extract/extract opts)- (v2.storage/store! path)))+ (v2.storage/store! (v2.storage.files/file-writer path))));; we could publish :event/serdes-dump to go with :event/serdes-load above, but;; nothing would listen to it currently(catch Exception ediff --git a/enterprise/backend/src/metabase_enterprise/serialization/core.clj b/enterprise/backend/src/metabase_enterprise/serialization/core.cljindex 137e48171c7..f8d3f49c3ed 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/core.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/core.clj@@ -5,6 +5,7 @@[metabase-enterprise.serialization.v2.ingest][metabase-enterprise.serialization.v2.load][metabase-enterprise.serialization.v2.storage]+ [metabase-enterprise.serialization.v2.storage.util][potemkin :as p]))(comment@@ -12,7 +13,8 @@metabase-enterprise.serialization.v2.extract/keep-memetabase-enterprise.serialization.v2.ingest/keep-memetabase-enterprise.serialization.v2.load/keep-me- metabase-enterprise.serialization.v2.storage/keep-me)+ metabase-enterprise.serialization.v2.storage/keep-me+ metabase-enterprise.serialization.v2.storage.util/keep-me)(p/import-vars[metabase-enterprise.serialization.dump@@ -33,6 +35,7 @@[metabase-enterprise.serialization.v2.loadload-metabase!][metabase-enterprise.serialization.v2.storage+ store!]+ [metabase-enterprise.serialization.v2.storage.utilresolve-storage-path- slugify-name- store!])+ slugify-name])diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 151beae0a39..b5d594238b8 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -143,8 +143,8 @@:error ::not-found}))))(log/debug "Local" {:path (serdes/log-path-str path)})ctx)- (let [_ (log/info "Loading" (cond-> {:path (serdes/log-path-str path)}- (circular path) (assoc :stripped true)))+ (let [_ (log/trace "Loading" (cond-> {:path (serdes/log-path-str path)}+ (circular path) (assoc :stripped true)));; Use the abstract path as attached by the ingestion process, not the original one we were passed.rebuilt-path (serdes/path ingested);; If nil or absent :entity_id is taken as a signal to create a new entitydiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.cljnew file mode 100644index 00000000000..de5d04ef05d--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/protocols.clj@@ -0,0 +1,14 @@+(ns metabase-enterprise.serialization.v2.protocols+ (:require+ [potemkin.types :as p]))++(p/defprotocol+ ExportWriter+ "Protocol for serialization export writer."+ (store-entity! [this entity]+ "Store a single serialized entity. Returns its `:serdes/meta`.")+ (store-settings! [this settings]+ "Store accumulated settings as a single settings.yaml entry.")+ (store-log! [this ^bytes content]+ "Store export.log content.")+ (finish! [this]+ "Finalize the backend (e.g. close tar stream). Called after all entities are written."))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.cljindex 503305e6a72..9723303e81b 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage.clj@@ -1,90 +1,15 @@(ns metabase-enterprise.serialization.v2.storage(:require- [clojure.java.io :as io]- [clojure.string :as str]- [metabase-enterprise.serialization.dump :refer [spit-yaml!]]- [metabase.lib.core :as lib]- [metabase.models.serialization :as serdes]- [metabase.util :as u]- [metabase.util.log :as log]- [metabase.util.string :as u.str])- (:import- [java.io File]- [java.nio.file Path]))+ [metabase-enterprise.serialization.v2.protocols :as protocols]))(set! *warn-on-reflection* true)-(def ^:private max-label-length 100)-(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4--(defn slugify-name- "Slugify a name for use as a file or directory name: lowercase, replace special chars with underscores,- preserve dots and unicode, escape slashes. Truncated for filesystem safety."- [^String s]- (when (seq s)- (-> s- u/lower-case-en- (str/replace "\\" "__BACKSLASH__")- (str/replace "/" "__SLASH__")- (str/replace #"[^\p{L}\p{N}_.]" "_")- (u.str/limit-bytes max-label-bytes)- (u.str/limit-chars max-label-length))))--(defn- resolve-path- "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings- with deduplication per folder. Uses `unique-name-fns` atom `{resolved-parent-path -> unique-name-fn}` where- each fn is a `lib/non-truncating-unique-name-generator`.- Keyed on the full resolved parent path to avoid cross-contamination between unrelated directories- (e.g. `collections/transforms/` vs `databases/.../schemas/transforms/`)."- [unique-name-fns path]- (loop [remaining path- resolved []]- (if (empty? remaining)- resolved- (let [{:keys [label key]} (first remaining)- slug (slugify-name label)- gen (or (get @unique-name-fns resolved)- (let [g (lib/non-truncating-unique-name-generator)]- (swap! unique-name-fns assoc resolved g)- g))- unique-name (gen key slug)]- (recur (rest remaining)- (conj resolved unique-name))))))--(defn resolve-storage-path- "Given ctx and entity, returns a vector of resolved (slugified, deduplicated) path strings.- The last element is the filename (without extension)."- [ctx entity]- (resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))--(defn- file ^File [ctx entity]- (let [resolved (resolve-storage-path ctx entity)- dirnames (drop-last resolved)- basename (str (last resolved) ".yaml")]- (apply io/file (:root-dir ctx) (concat dirnames [basename]))))--(defn- store-entity! [opts entity]- (let [f (file opts entity)]- (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))- :file (str (.relativize (Path/of (str (:root-dir opts)) (make-array String 0))- (.toPath f)))})- (spit-yaml! f entity)- (:serdes/meta entity)))--(defn- store-settings! [{:keys [root-dir]} settings]- (when (seq settings)- (let [as-map (into (sorted-map)- (for [{:keys [key value]} settings]- [key value]))]- (spit-yaml! (io/file root-dir "settings.yaml") as-map))))-(defn store!- "Helper for storing a serialized database to a tree of YAML files."- [stream root-dir]+ "Consume the entity `stream` and store each entity via the given `writer`.+ Returns a report map with `:seen` and `:errors` vectors."+ [stream writer](let [settings (atom [])- report (atom {:seen [] :errors []})- opts (-> (serdes/storage-base-context)- (assoc :root-dir root-dir))]+ report (atom {:seen [] :errors []})](doseq [entity stream](cond(instance? Exception entity)@@ -94,8 +19,8 @@(swap! settings conj entity):else- (swap! report update :seen conj (store-entity! opts entity))))+ (swap! report update :seen conj (protocols/store-entity! writer entity))))(when (seq @settings)- (store-settings! opts @settings)+ (protocols/store-settings! writer @settings)(swap! report update :seen conj [{:model "Setting"}]))@report))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.cljnew file mode 100644index 00000000000..92bdff665de--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/files.clj@@ -0,0 +1,46 @@+(ns metabase-enterprise.serialization.v2.storage.files+ "Filesystem storage backend — writes YAML files to a directory tree."+ (:require+ [clojure.java.io :as io]+ [metabase-enterprise.serialization.dump :refer [spit-yaml!]]+ [metabase-enterprise.serialization.v2.protocols :as protocols]+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]+ [metabase.models.serialization :as serdes]+ [metabase.util.log :as log])+ (:import+ [java.io File]+ [java.nio.file Path]))++(set! *warn-on-reflection* true)++(defn- file ^File [ctx root-dir entity]+ (let [resolved (storage.util/resolve-storage-path ctx entity)+ dirnames (drop-last resolved)+ basename (str (last resolved) ".yaml")]+ (apply io/file root-dir (concat dirnames [basename]))))++(defn file-writer+ "Create a filesystem storage backend rooted at `root-dir`."+ [root-dir]+ (let [ctx (serdes/storage-base-context)]+ (reify protocols/ExportWriter+ (store-entity! [_ entity]+ (let [f (file ctx root-dir entity)]+ (log/info "Storing" {:path (serdes/log-path-str (:serdes/meta entity))+ :file (str (.relativize (Path/of (str root-dir) (make-array String 0))+ (.toPath f)))})+ (spit-yaml! f entity)+ (:serdes/meta entity)))++ (store-settings! [_ settings]+ (when (seq settings)+ (let [as-map (into (sorted-map)+ (for [{:keys [key value]} settings]+ [key value]))]+ (spit-yaml! (io/file root-dir "settings.yaml") as-map))))++ (store-log! [_ _content]+ nil)++ (finish! [_]+ nil))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.cljnew file mode 100644index 00000000000..25494d75383--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/tar.clj@@ -0,0 +1,63 @@+(ns metabase-enterprise.serialization.v2.storage.tar+ "Streaming tar.gz storage backend — writes entries directly to a TarArchiveOutputStream."+ (:require+ [clojure.string :as str]+ [metabase-enterprise.serialization.dump :refer [yaml-content]]+ [metabase-enterprise.serialization.v2.protocols :as protocols]+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]+ [metabase.models.serialization :as serdes]+ [metabase.util.log :as log])+ (:import+ [java.io OutputStream]+ [org.apache.commons.compress.archivers.tar TarArchiveEntry TarArchiveOutputStream]+ [org.apache.commons.compress.compressors.gzip GzipCompressorOutputStream GzipParameters]))++(set! *warn-on-reflection* true)++(defn- put-entry!+ "Write a single entry to a TarArchiveOutputStream."+ [^TarArchiveOutputStream tar-out ^String entry-path ^bytes content]+ (let [entry (doto (TarArchiveEntry. entry-path)+ (.setSize (alength content)))]+ (.putArchiveEntry tar-out entry)+ (.write tar-out content)+ (.closeArchiveEntry tar-out)))++(defn- entry-path+ "Build the tar entry path: dirname/resolved.../name.yaml"+ ^String [dirname resolved]+ (str dirname "/" (str/join "/" (drop-last resolved)) "/" (last resolved) ".yaml"))++(defn tar-writer+ "Create a streaming tar.gz storage backend writing to `output`."+ [^OutputStream output ^String dirname]+ (let [gzip (GzipCompressorOutputStream. output (doto (GzipParameters.)+ (.setModificationTime (System/currentTimeMillis))))+ tar (TarArchiveOutputStream. gzip 512 "UTF-8")+ ctx (serdes/storage-base-context)]+ (.setLongFileMode tar TarArchiveOutputStream/LONGFILE_POSIX)+ (.setBigNumberMode tar TarArchiveOutputStream/BIGNUMBER_POSIX)+ (reify protocols/ExportWriter+ (store-entity! [_ entity]+ (let [resolved (storage.util/resolve-storage-path ctx entity)+ path (entry-path dirname resolved)+ content (.getBytes ^String (yaml-content entity) "UTF-8")]+ (log/trace "Storing" {:path (serdes/log-path-str (:serdes/meta entity))+ :file path})+ (put-entry! tar path content)+ (:serdes/meta entity)))++ (store-settings! [_ settings]+ (when (seq settings)+ (let [as-map (into (sorted-map)+ (for [{:keys [key value]} settings]+ [key value]))+ content (.getBytes ^String (yaml-content as-map) "UTF-8")]+ (put-entry! tar (str dirname "/settings.yaml") content))))++ (store-log! [_ content]+ (put-entry! tar (str dirname "/export.log") content))++ (finish! [_]+ (.finish tar)+ (.finish gzip)))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.cljnew file mode 100644index 00000000000..0774eff3303--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/storage/util.clj@@ -0,0 +1,49 @@+(ns metabase-enterprise.serialization.v2.storage.util+ (:require+ [clojure.string :as str]+ [metabase.lib.core :as lib]+ [metabase.models.serialization :as serdes]+ [metabase.util :as u]+ [metabase.util.string :as u.str]))++(set! *warn-on-reflection* true)++(def ^:private max-label-length 100)+(def ^:private max-label-bytes 200) ;; 255 is a limit in ext4++(defn slugify-name+ "Slugify a name for use as a file or directory name: lowercase, replace special chars with underscores,+ preserve dots and unicode, escape slashes. Truncated for filesystem safety."+ [^String s]+ (when (seq s)+ (-> s+ u/lower-case-en+ (str/replace "\\" "__BACKSLASH__")+ (str/replace "/" "__SLASH__")+ (str/replace #"[^\p{L}\p{N}_.]" "_")+ (u.str/limit-bytes max-label-bytes)+ (u.str/limit-chars max-label-length))))++(defn- resolve-path+ "Given a storage path (vector of `{:label ... :key ...}` maps), resolves to a vector of strings+ with deduplication per folder."+ [unique-name-fns path]+ (loop [remaining path+ resolved []]+ (if (empty? remaining)+ resolved+ (let [{:keys [label key]} (first remaining)+ slug (slugify-name label)+ gen (or (get @unique-name-fns resolved)+ (let [g (lib/non-truncating-unique-name-generator)]+ (swap! unique-name-fns assoc resolved g)+ g))+ unique-name (gen key slug)]+ (recur (rest remaining)+ (conj resolved unique-name))))))++(defn resolve-storage-path+ "Given ctx and entity, returns a vector of resolved (slugified, deduplicated) path strings.+ The last element is the filename (without extension)."+ [ctx entity]+ (resolve-path (:unique-name-fns ctx) (serdes/storage-path entity ctx)))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/api_test.cljindex b68909f93a7..884da4a7a22 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/api_test.clj@@ -26,6 +26,27 @@(GzipCompressorInputStream.)(TarArchiveInputStream.)))+(defn- entry-names+ "Return a seq of all entry names in the given tar.gz response."+ [f]+ (with-open [tar (open-tar f)]+ (mapv (fn [^TarArchiveEntry e] (.getName e)) (u.compress/entries tar))))++(defn- first-entry-name+ "Return the name of the first entry in the given tar.gz response."+ [f]+ (first (entry-names f)))++(defn- read-export-log+ "Extract the export.log content from a tar.gz response."+ [f]+ (with-open [tar (open-tar f)]+ (loop []+ (when-let [e (.getNextEntry tar)]+ (if (str/ends-with? (.getName ^TarArchiveEntry e) "export.log")+ (slurp (io/reader tar))+ (recur))))))+(def ^:private file-types[#"([^/]+)?/$" :dir#"/settings.yaml$" :settings@@ -44,13 +65,6 @@[ftype (when (vector? m) (second m))]))(partition 2 file-types)))-(defn- log-types- "Find out entity type by log message"- [lines]- (->> lines- (keep #(second (re-find #"(?:Extracting|Loading|Storing) \{:path (\w+)" %)))- set))-(defn- tar-file-types [f & [raw?]](with-open [tar (open-tar f)](cond->> (u.compress/entries tar)@@ -117,13 +131,13 @@(testing "API respects parameters"(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}:all_collections false :data_model false :settings true)]- (is (= #{:log :dir :settings :transform :python-library}+ (is (= #{:log :settings :transform :python-library}(tar-file-types f)))))(testing "We can export just a single collection"(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}:collection (:id coll) :data_model false :settings false)]- (is (= #{:log :dir :collection-entity :transform :python-library}+ (is (= #{:log :collection-entity :transform :python-library}(tar-file-types f)))))(testing "We can export two collections"@@ -138,52 +152,44 @@(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {};; eid:... syntax is kept for backward compat:collection (str "eid:" (:entity_id coll)) :data_model false :settings false)]- (is (= #{:log :dir :collection-entity :transform :python-library}+ (is (= #{:log :collection-entity :transform :python-library}(tar-file-types f)))))(testing "We can export that collection using entity id"(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}:collection (:entity_id coll) :data_model false :settings false)]- (is (= #{:log :dir :collection-entity :transform :python-library}+ (is (= #{:log :collection-entity :transform :python-library}(tar-file-types f)))))(testing "Default export: all-collections, data-model, settings"(let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {})]- (is (= #{:transform :log :dir :collection-entity :settings :schema :database :python-library}+ (is (= #{:transform :log :collection-entity :settings :schema :database :python-library}(tar-file-types f)))))- (testing "On exception API returns log"+ (testing "On exception API returns tar.gz with error in export.log"(mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)(mt/dynamic-value serdes/extract-one))](let [res (binding [api.serialization/*additive-logging* false]- (mt/user-http-request :crowberto :post 500 "ee/serialization/export" {}+ (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}:collection (:id coll) :data_model false :settings false))- log (slurp (io/input-stream res))]- (testing "In logs we get an entry for the dashboard, then card, and then an error"- (is (= #{"Dashboard" "Card"}- (log-types (str/split-lines log))))- (is (re-find #"deliberate error message" log))- (is (= {:id "**ID**",- :entity_id "**ID**",- :model "Card",- :table :report_card- :cause "[test] deliberate error message"}- (extract-and-sanitize-exception-map log)))))))-- (testing "You can pass specific directory name"- (let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}- :dirname "check" :all_collections false :data_model false :settings false)]- (is (= "check/"- (with-open [tar (open-tar f)]- (.getName ^TarArchiveEntry (first (u.compress/entries tar))))))))-- (testing "Invalid entity ID returns an error instead of falling back to root collection"- (let [fake-eid "abcdefghijklmnopqrstu"- res (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}- :collection fake-eid :data_model false :settings false)- log (slurp (io/input-stream res))]- (is (re-find #"Could not find Collection with entity ID" log))- (is (re-find #"abcdefghijklmnopqrstu" log))))))))+ log (read-export-log res)]+ (testing "export.log inside the archive contains error details"+ (is (some? log) "export.log should be present in the archive")+ (is (re-find #"deliberate error message" log)))))))++ (testing "You can pass specific directory name"+ (let [f (mt/user-http-request :crowberto :post 200 "ee/serialization/export" {}+ :dirname "check" :all_collections false :data_model false :settings false)]+ (is (str/starts-with? (first-entry-name f) "check/")))))++ (testing "Invalid entity ID returns an error instead of falling back to root collection"+ (let [fake-eid "abcdefghijklmnopqrstu"+ res (mt/user-http-request :crowberto :post 400 "ee/serialization/export" {}+ :collection fake-eid :data_model false :settings false)]+ (is (re-find #"Could not find Collection with entity ID"+ (str (:message res))))+ (is (re-find #"abcdefghijklmnopqrstu"+ (str (:message res))))))))(testing "We've left no new files, every request is cleaned up";; if this breaks, check if you consumed every response with io/input-stream(is (= known-files@@ -214,19 +220,8 @@:collection (:id coll) :data_model false :settings false)io/input-stream)ba (#'api.serialization/ba-copy res)]- (testing "Archive contains correct number of files with proper log entries"- (is (= 13- (with-open [tar (open-tar ba)]- (count- (for [^TarArchiveEntry e (u.compress/entries tar)- :when (.isFile e)]- (do- (condp re-find (.getName e)- #"/export.log$" (testing "Log contains extract and store entries"- (is (= (+ #_extract 12 #_store 12)- (count (line-seq (io/reader tar))))))- nil)- (.getName e))))))))+ (testing "Archive contains correct number of files"+ (is (= 13 (count (filter #(not (str/ends-with? % "/")) (entry-names ba))))))(testing "Snowplow export event was sent"(is (=? {"event" "serialization"@@ -262,14 +257,10 @@(t2/delete! :model/Card (:id card))(let [re-indexed? (atom false)- res (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))]+ _res (mt/with-dynamic-fn-redefs [search/reindex! (fn [& _] (reset! re-indexed? true) (future nil))](mt/user-http-request :crowberto :post 200 "ee/serialization/import?reindex=false"{:request-options {:headers {"content-type" "multipart/form-data"}}}{:file ba}))]- (testing "Log contains imported entity types"- (is (= #{"Collection" "Dashboard" "Card" "PythonLibrary" "TransformJob" "TransformTag"}- (log-types (line-seq (io/reader (io/input-stream res)))))))-(testing "Entities are restored in the database"(is (= (:name dash) (t2/select-one-fn :name :model/Dashboard :entity_id (:entity_id dash))))(is (= (:name card) (t2/select-one-fn :name :model/Card :entity_id (:entity_id card)))))@@ -342,9 +333,7 @@{:file ba}:continue_on_error true)log (slurp (io/input-stream res))]- (testing "Log shows loaded entities and error"- (is (= #{"Dashboard" "Card" "Collection" "TransformJob" "TransformTag" "PythonLibrary"}- (log-types (str/split-lines log))))+ (testing "Log contains the missing-collection error"(is (re-find #"Collection 'DoesNotExist' was not found" log)))(testing "Snowplow event shows partial success with error count"@@ -368,15 +357,16 @@(is (re-find #"Cannot unpack archive" log))))))(deftest export-extraction-error-test- (testing "Export fails with 500 when entity extraction fails"+ (testing "Export with error still returns tar.gz with export.log containing error details"(with-serialization-test-data! [coll _dash card](mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)(mt/dynamic-value serdes/extract-one))]- (testing "Error response contains exception details"+ (testing "Error details are in export.log inside the archive"(binding [api.serialization/*additive-logging* false]- (let [res (mt/user-http-request :crowberto :post 500 "ee/serialization/export"+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export":collection (:id coll) :data_model false :settings false)- log (slurp (io/input-stream res))]+ log (read-export-log res)]+ (is (some? log) "export.log should be present in the archive")(is (= {:id "**ID**":entity_id "**ID**":model "Card"@@ -400,12 +390,12 @@"error_message" #"(?s)Error extracting Card \d+ .*"}(-> (snowplow-test/pop-event-data-and-user-id!) last :data))))- (testing "full_stacktrace parameter includes full stack trace"+ (testing "full_stacktrace parameter includes full stack trace in export.log"(binding [api.serialization/*additive-logging* false]- (let [res (mt/user-http-request :crowberto :post 500 "ee/serialization/export"+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export":collection (:id coll) :data_model false :settings false:full_stacktrace true)- log (slurp (io/input-stream res))]+ log (read-export-log res)](is (< 50 (count (str/split-lines log))));; Pop out the error event(snowplow-test/pop-event-data-and-user-id!))))))))@@ -415,17 +405,11 @@(with-serialization-test-data! [coll _dash card](mt/with-dynamic-fn-redefs [serdes/extract-one (extract-one-error (:entity_id card)(mt/dynamic-value serdes/extract-one))]- (let [res (-> (mt/user-http-request :crowberto :post 200 "ee/serialization/export"- :collection (:id coll) :data_model false :settings false- :continue_on_error true)- io/input-stream)]- (with-open [tar (open-tar res)]- (doseq [^TarArchiveEntry e (u.compress/entries tar)]- (condp re-find (.getName e)- #"/export.log$" (testing "Log shows extract entries, error, and store entries"- (is (= (+ #_extract 12 #_error 1 #_store 11)- (count (line-seq (io/reader tar))))))- nil))))+ (let [res (mt/user-http-request :crowberto :post 200 "ee/serialization/export"+ :collection (:id coll) :data_model false :settings false+ :continue_on_error true)]+ (testing "Log contains the deliberate error"+ (is (re-find #"deliberate error message" (read-export-log res)))))(testing "Snowplow event shows partial success with error count"(is (=? {"event" "serialization"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.cljindex d61019730b3..d06192da72b 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj@@ -65,8 +65,8 @@"error_message" nil}(-> (snowplow-test/pop-event-data-and-user-id!) first :data))))- (with-redefs [v2.storage/store-settings! (fn [_opts _settings]- (throw (Exception. "Cannot load settings")))]+ (with-redefs [v2.storage/store! (fn [_stream _backend]+ (throw (Exception. "Cannot load settings")))](is (thrown? Exception(cmd/export dump-dir "--collection" (str (:id coll)) "--no-data-model")))(testing "Snowplow export event about error was sent"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 122e1ed35f1..62e720a28a3 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -10,6 +10,7 @@[metabase-enterprise.serialization.v2.ingest :as ingest][metabase-enterprise.serialization.v2.load :as serdes.load][metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.storage.files :as storage.files][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.settings.core :as setting]@@ -211,7 +212,7 @@(is (= 110 (-> @entities (get "Collection") count))))(testing "storage"- (storage/store! (seq @extraction) dump-dir)+ (storage/store! (seq @extraction) (storage.files/file-writer dump-dir))(testing "for Actions"(is (= 30 (count (dir->file-set (io/file dump-dir "actions"))))))@@ -447,7 +448,7 @@:values_source_type :card}]}(set (map :parameters (by-model extraction "Card")))))- (storage/store! (seq extraction) dump-dir)))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))(testing "ingest and load"(ts/with-db dest-db@@ -555,7 +556,7 @@{:model "Table" :id "Linked table"}]}(set (serdes/dependencies extracted-dashboard))))- (storage/store! (seq extraction) dump-dir)))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))(testing "ingest and load";; ingest@@ -615,7 +616,7 @@hydrated-dashcards))))(testing "extract and store"(let [extraction (serdes/with-cache (into [] (extract/extract {})))]- (storage/store! (seq extraction) dump-dir)))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir))))(testing "ingest and load"(ts/with-db dest-db(testing "doing ingestion"@@ -672,7 +673,7 @@:card_id card-id-2:dashboard_tab_id tab-id-2}](let [extraction (serdes/with-cache (into [] (extract/extract {})))]- (storage/store! (seq extraction) dump-dir))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"(ts/with-db dest-db@@ -725,7 +726,7 @@:model/Card {card-eid :entity_id} {:name "Card on the Dashboard":dashboard_id dashboard-id}](let [extraction (serdes/with-cache (into [] (extract/extract {})))]- (storage/store! (seq extraction) dump-dir))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"(ts/with-db dest-db@@ -754,7 +755,7 @@:model/Database _ {:router_database_id router-db-id:name "Destination"}](let [extraction (serdes/with-cache (into [] (extract/extract {})))]- (storage/store! (seq extraction) dump-dir))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"(ts/with-db dest-db(testing "doing ingestion"@@ -837,7 +838,7 @@:products.title %products.title:orders.product_id %orders.product_id})(reset! card1s card)- (storage/store! (extract/extract {}) dump-dir)))))+ (storage/store! (extract/extract {}) (storage.files/file-writer dump-dir))))))(ts/with-db dest-db;; ensure there is something in db so that test-data gets different field ids for sure@@ -876,7 +877,7 @@(let [coll (ts/create! :model/Collection :name "coll")_ (ts/create! :model/Card :name "card" :collection_id (:id coll))](storage/store! (extract/extract {:no-settings true- :no-data-model true}) dump-dir)+ :no-data-model true}) (storage.files/file-writer dump-dir))(spit (io/file dump-dir "collections" ".hidden.yaml") "serdes/meta: [{do-not: read}]")@@ -900,7 +901,7 @@(let [coll (ts/create! :model/Collection :name "coll")_ (ts/create! :model/Card :name "card" :collection_id (:id coll))](storage/store! (extract/extract {:no-settings true- :no-data-model true}) dump-dir)+ :no-data-model true}) (storage.files/file-writer dump-dir))(spit (io/file dump-dir "collections" "corrupt.yaml") "\0")(testing "continue-on-error false (default) — throws on ingestion errors"@@ -927,7 +928,7 @@:type :channel/http:details {:url "http://example.com":auth-method :none}}]- (storage/store! (seq (serdes/with-cache (into [] (extract/extract {})))) dump-dir)+ (storage/store! (seq (serdes/with-cache (into [] (extract/extract {})))) (storage.files/file-writer dump-dir))(ts/with-db dest-db(testing "doing ingestion"(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))@@ -956,7 +957,7 @@{:aggregation [[:metric metric-id]]:breakout [[:field %orders.user_id nil]]})}](let [extraction (serdes/with-cache (into [] (extract/extract {})))]- (storage/store! (seq extraction) dump-dir))+ (storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"(ts/with-db dest-db(testing "doing ingestion"@@ -988,7 +989,7 @@(serdes/with-cache(-> (extract/extract {:no-settings true:no-data-model true})- (storage/store! dump-dir))))+ (storage/store! (storage.files/file-writer dump-dir)))))(testing "loads well too"(is (serdes/with-cache (serdes.load/load-metabase! (ingest/ingest-yaml dump-dir)))"ingested successfully"))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex 681426be8b9..71d5a2aa3b1 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -32,6 +32,7 @@[metabase-enterprise.serialization.v2.load :as load][metabase-enterprise.serialization.v2.models :as serdes.models][metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.storage.files :as storage.files][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.search.test-util :as search.tu]@@ -108,7 +109,7 @@;; Use a separate cache to make sure there is no cross-contamination.(serdes/with-cache(-> (extract/extract {:include-field-values true :include-metabot true})- (storage/store! output-dir))))+ (storage/store! (storage.files/file-writer output-dir)))))(defn- delete-dir-contents! [^File dir](when (and dir (.exists dir))@@ -139,7 +140,7 @@(defn add-to-baseline!"Use this within v2.extract-test where relevant to add their fixtures to the baseline."[]- (storage/store! (into [] (extract/extract {:include-field-values true :include-metabot true})) source-dir))+ (storage/store! (into [] (extract/extract {:include-field-values true :include-metabot true})) (storage.files/file-writer source-dir)));; If this test is failing, read the docstring at the top of this namespace for what to do B-)(deftest baseline-completeness-testdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.cljindex 8271fa5dfb3..7dd231c85ba 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj@@ -8,6 +8,8 @@[metabase-enterprise.serialization.test-util :as ts][metabase-enterprise.serialization.v2.extract :as extract][metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]+ [metabase-enterprise.serialization.v2.storage.util :as storage.util][metabase.models.serialization :as serdes][metabase.test :as mt][metabase.util.yaml :as yaml]@@ -28,7 +30,7 @@(ts/with-temp-dpc [:model/Collection parent {:name "Some Collection"}:model/Collection child {:name "Child Collection" :location (format "/%d/" (:id parent))}](let [export (into [] (extract/extract {:no-transforms true}))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir))(testing "the right files in the right places"(is (= #{["main" "some_collection.yaml"]["main" "some_collection" "child_collection.yaml"]}@@ -76,7 +78,7 @@:model/Card _c4 {:name "child card" :collection_id (:id child)}:model/Dashboard _d1 {:name "parent dash" :collection_id (:id parent)}](let [export (into [] (extract/extract {:no-transforms true}))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir))(testing "the right files in the right places"(is (= #{["main" "grandparent_collection.yaml"]["main" "grandparent_collection" "parent_collection.yaml"]@@ -107,7 +109,7 @@(let [export (into [] (extract/extract {:no-settings true:no-data-model true:no-transforms true}))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir))(testing "all snippet collections and snippets under collections/snippets/"(is (= #{["snippets" "grandparent_collection.yaml"]["snippets" "grandparent_collection" "parent_collection.yaml"]@@ -127,7 +129,7 @@:model/FieldValues _ {:field_id (:id website)}:model/Table _ {:name "Orders/Invoices" :db_id (:id db)}](let [export (into [] (extract/extract {:include-field-values true}))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir))(testing "the right files in the right places"(is (= #{["company__SLASH__organization_website.yaml"]["company__SLASH__organization_website___fieldvalues.yaml"]}@@ -196,7 +198,7 @@(if (str/ends-with? fname "settings.yaml")(descend coll [:settings])(descend coll)))))]- (storage/store! export dump-dir)))))))+ (storage/store! export (storage.files/file-writer dump-dir))))))))(deftest store-error-test(mt/with-empty-h2-app-db!@@ -208,7 +210,7 @@(.setWritable (io/file parent-dir) false)(is (thrown-with-msg? Exception #"Destination path is not writeable: "(storage/store! [{:serdes/meta [{:model "A" :id "B"}]}]- dump-dir))))+ (storage.files/file-writer dump-dir)))))(testing "directory exists but is not writable"(.setWritable (io/file parent-dir) true)(.mkdirs (io/file dump-dir))@@ -216,7 +218,7 @@(.setWritable (io/file dump-dir) false)(is (thrown-with-msg? Exception #"Destination path is not writeable: "(storage/store! [{:serdes/meta [{:model "A" :id "B"}]}]- dump-dir)))))))))+ (storage.files/file-writer dump-dir))))))))))(deftest nested-fields-test(ts/with-random-dump-dir [dump-dir "serdesv2-"]@@ -227,7 +229,7 @@_f2 (ts/create! :model/Field :name "child" :table_id (:id t) :parent_id (:id f1))](serdes/with-cache(-> (extract/extract {:no-settings true})- (storage/store! dump-dir)))+ (storage/store! (storage.files/file-writer dump-dir))))(testing "we get correct names for nested fields"(is (= #{["parent.yaml"]["parent.child.yaml"]}@@ -239,7 +241,7 @@(t2/delete! :model/PythonLibrary)(ts/with-temp-dpc [:model/PythonLibrary _lib {:path "common" :source "def test(): pass"}](let [export (into [] (extract/extract {:no-settings true :no-data-model true}))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir))(testing "python library stored at top-level python_libraries/"(is (= #{["common.py.yaml"]}(file-set (io/file dump-dir "python_libraries"))))))))))@@ -255,7 +257,7 @@:targets [["Card" (:id card)]]}));; 66 is 'char-count * max-bytes / byte-count'card-filename (str/join (repeat 66 "ป"))]- (storage/store! export dump-dir)+ (storage/store! export (storage.files/file-writer dump-dir));; we could also test loading here, but file names do not play significant part in how everything's loaded,;; `:serdes/meta` does and that one is not shortened or anything(testing "the right files in the right places"@@ -264,7 +266,7 @@"collections form a tree, with same-named files")))))))(deftest ^:parallel resolve-path-test- (let [resolve-path @#'storage/resolve-path]+ (let [resolve-path @#'storage.util/resolve-path](testing "basic slugification"(let [fns (atom {})](is (= ["my_collection" "some_card"]diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 3e476c235b4..4bd7a0dc436 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -426,7 +426,7 @@:skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.:legacy_query]:transform {:action_id (serdes/parent-ref)- :database_id (serdes/fk :model/Database :name)+ :database_id (serdes/fk :model/Database):dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})(defmethod serdes/generate-path "HTTPAction" [_ _] nil)diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 26413f32c7a..768fe0f1a26 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -35,7 +35,7 @@## Existing transformations- - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database :name)` - export foreign key in a portable way+ - `(serdes/fk :model/Card)` or `(serdes/fk :model/Database)` - export foreign key in a portable way- `(serdes/nested :model/DashboardCard :dashboard_id opts)` - include some entities in your entity export- `(serdes/parent-ref)` - symmetrical call for `serdes/nested` to handle parent ids (you'd use it on `:dashboard_id`in that case)@@ -450,6 +450,11 @@(defmethod make-spec :default [_ _] nil)+(defn ^:dynamic ^::cache *make-spec*+ "Cachable wrapper around [[make-spec]] that is memoized inside [[with-cache]]."+ [model-name opts]+ (make-spec model-name opts))+(defmulti extract-all"Entry point for extracting all entities of a particular model:`(extract-all \"ModelName\" {opts...})`@@ -495,7 +500,7 @@- Replace any foreign keys with portable values (eg. entity IDs, or a user ID with their email, etc.)"[model-name opts instance](try- (let [spec (make-spec model-name opts)]+ (let [spec (*make-spec* model-name opts)](assert spec (str "No serialization spec defined for model " model-name))(-> (into {}(remove (fn [[k v]] (= v (get-in spec [:defaults k]))))@@ -527,7 +532,7 @@(defn log-and-extract-one"Extracts a single entity; will replace `extract-one` as public interface once `extract-one` overrides are gone."[model opts instance]- (log/info "Extracting" {:path (log-path-str (generate-path model instance))})+ (log/tracef "Extracting %s" (log-path-str (generate-path model instance)))(try(extract-one model opts instance)(catch Exception e@@ -557,7 +562,7 @@(group-by backward-fk entities)))(defn- extract-batch-nested [model-name opts batch]- (let [spec (make-spec model-name opts)]+ (let [spec (*make-spec* model-name opts)](reduce-kv (fn [batch k transform](if-not (::nested transform)batch@@ -577,7 +582,7 @@"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personalcollection."[model {:keys [collection-set where] :as opts}]- (let [spec (make-spec (name model) opts)]+ (let [spec (*make-spec* (name model) opts)](if (or (empty? collection-set)(nil? (-> spec :transform :collection_id)));; either no collections specified or our model has no collection@@ -591,7 +596,7 @@where)]}))))(defmethod extract-query :default [model-name opts]- (let [spec (make-spec model-name opts)+ (let [spec (*make-spec* model-name opts)nested? (some ::nested (vals (:transform spec)))](cond->> (extract-query-collections (keyword "model" model-name) opts)nested? (extract-reducible-nested model-name (dissoc opts :where)))))@@ -795,7 +800,7 @@schemas))(defn- xform-one [model-name ingested]- (let [spec (make-spec model-name nil)]+ (let [spec (*make-spec* model-name nil)](assert spec (str "No serialization spec defined for model " model-name))(-> (select-keys ingested (:copy spec))(into (for [[k transform] (:transform spec)@@ -815,7 +820,7 @@(coerce-keys (:coerce spec)))))(defn- spec-nested! [model-name ingested instance]- (let [spec (make-spec model-name nil)]+ (let [spec (*make-spec* model-name nil)](doseq [[k transform] (:transform spec):when (and (::nested transform);; handling circuit-breaking@@ -1032,7 +1037,8 @@"Given a numeric database ID, return its name as a portable reference.[[*import-database-fk*]] is the inverse."[id]- (*export-fk-keyed* id :model/Database :name))+ (when id+ (t2/select-one-fn :name [:model/Database :id :name] :id id)))(defn ^:dynamic ^::cache *import-database-fk*"Given a portable database name, resolve it back to a numeric ID.@@ -1049,8 +1055,8 @@[[import-table-fk]] is the inverse."[table-id :- [:maybe ::lib.schema.id/table]](when table-id- (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)- db-name (t2/select-one-fn :name :model/Database :id db_id)]+ (let [{:keys [db_id name schema]} (t2/select-one [:model/Table :id :db_id :name :schema] :id table-id)+ db-name (*export-database-fk* db_id)][db-name schema name])))(mu/defn ^:dynamic ^::cache *import-table-fk*@@ -1128,9 +1134,12 @@[[*import-field-fk*]] is the inverse."[field-id :- [:maybe ::lib.schema.id/field]](when field-id- (let [fields (field-hierarchy field-id)- [db-name schema field-name] (*export-table-fk* (:table_id (first fields)))]- (into [db-name schema field-name] (map :name fields)))))+ (let [{field-name :name :keys [parent_id table_id]} (t2/select-one [:model/Field :name :parent_id :table_id] :id field-id)]+ (if (nil? parent_id)+ (conj (*export-table-fk* table_id) field-name)+ (let [fields (field-hierarchy field-id)+ [db-name schema table-name] (*export-table-fk* table_id)]+ (into [db-name schema table-name] (map :name fields)))))))(mu/defn ^:dynamic ^::cache *import-field-fk* :- [:maybe pos-int?]"Given a `field_id` as exported by [[*export-field-fk*]], resolve it back into a numeric `field_id`."diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 520ab07ceb3..8e1122c8408 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1326,7 +1326,7 @@:legacy_query]:transform{:created_at (serdes/date)- :database_id (serdes/fk :model/Database :name)+ :database_id (serdes/fk :model/Database):table_id (serdes/fk :model/Table):source_card_id (serdes/fk :model/Card):collection_id (serdes/fk :model/Collection)diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 3a747d21218..60f6501d2c4 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -364,7 +364,7 @@:creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User):collection_id (serdes/fk :model/Collection)- :source_database_id (serdes/fk :model/Database :name)+ :source_database_id (serdes/fk :model/Database):source {:export (fn [source](-> source(m/update-existing :query serdes/export-mbql)diff --git a/src/metabase/util/compress.clj b/src/metabase/util/compress.cljindex bfafd1e1cf3..4a559a3b0ac 100644--- a/src/metabase/util/compress.clj+++ b/src/metabase/util/compress.clj@@ -56,6 +56,7 @@:when (not (str/starts-with? actual-name "."))](let [f (.toFile (.resolveIn e dst-path))](if (.isFile e)- (io/copy tar f)+ (do (io/make-parents f)+ (io/copy tar f))(.mkdirs f))true))))))diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 13d314792d6..553ddabcb50 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -420,8 +420,8 @@;; In order to retrieve the dependencies for a field its table_id needs to be serialized as [database schema table],;; a trio of strings with schema maybe nil.-(defmethod serdes/generate-path "Field" [_ field]- (let [[db schema table & fields] (serdes/*export-field-fk* (:id field))]+(defmethod serdes/generate-path "Field" [_ {:keys [id]}]+ (let [[db schema table & fields] (serdes/*export-field-fk* id)](->> (into (serdes/table->path [db schema table])(map (fn [n] {:model "Field" :id n}) fields))(filterv some?))))diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex 67c444ab9eb..bf47518bd88 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -41,9 +41,8 @@(defmethod serdes/entity-id "FieldUserSettings" [_ _] nil)(defmethod serdes/generate-path "FieldUserSettings" [_ {:keys [field_id]}]- (let [field (t2/select-one 'Field :id field_id)]- (conj (serdes/generate-path "Field" field)- {:model "FieldUserSettings" :id "1"})))+ (conj (serdes/generate-path "Field" {:id field_id})+ {:model "FieldUserSettings" :id "1"}))(defmethod serdes/dependencies "FieldUserSettings" [fv];; Take the path, but drop the FieldUserSettings section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex e7a52abe298..60445e60727 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -571,9 +571,8 @@(defmethod serdes/entity-id "FieldValues" [_ _] nil)(defmethod serdes/generate-path "FieldValues" [_ {:keys [field_id]}]- (let [field (t2/select-one 'Field :id field_id)]- (conj (serdes/generate-path "Field" field)- {:model "FieldValues" :id "0"})))+ (conj (serdes/generate-path "Field" {:id field_id})+ {:model "FieldValues" :id "0"}))(defmethod serdes/dependencies "FieldValues" [fv];; Take the path, but drop the FieldValues section at the end, to get the parent Field's path instead.diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 3a9322a9318..f34f2516b01 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -708,7 +708,7 @@:archived_at (serdes/date):deactivated_at (serdes/date):data_layer (serdes/optional-kw)- :db_id (serdes/fk :model/Database :name)+ :db_id (serdes/fk :model/Database):collection_id (serdes/fk :model/Collection):transform_id (serdes/fk :model/Transform)}:defaults {:is_defective_duplicate falsediff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 3584a9eec7a..22368aeac15 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -648,7 +648,7 @@::serdes/skip)):import identity}:creator_id (serdes/fk :model/User)- :router_database_id (serdes/fk :model/Database :name)+ :router_database_id (serdes/fk :model/Database):initial_sync_status {:export identity :import (constantly "complete")}}:defaults {:auto_run_queries true:is_attached_dwh false