The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Make serialization output byte-stable (in some cases) (#84948)
Alexander Polyankin930efb1285fd
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 08132efa653..4420c9746cd 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -110,7 +110,7 @@(defn- clean-entity"Removes any comparison-confounding fields, like `:created_at`."[entity]- (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule))+ (dissoc entity :created_at :result_metadata));; one generated random dump feeds all storage+ingestion assertions; splitting regenerates it per test#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}@@ -297,7 +297,7 @@clean-entity)))))(testing "for Databases"(doseq [{:keys [name] :as db} (get @entities "Database")]- (is (= (assoc (clean-entity db) :initial_sync_status "complete")+ (is (= (clean-entity db)(-> (ts/extract-one "Database" [:= :name name])clean-entity)))))(testing "for Tables"@@ -501,16 +501,22 @@dashboard-name :name} {:name "Test Dashboard":collection_id coll-id}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 0:visualization_settings (link-card-viz-setting "collection" coll-id)}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 1:visualization_settings (link-card-viz-setting "database" db-id)}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 2:visualization_settings (link-card-viz-setting "table" table-id)}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 3:visualization_settings (link-card-viz-setting "dashboard" dash-id)}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 4:visualization_settings (link-card-viz-setting "card" card-id)}:model/DashboardCard _ {:dashboard_id dashboard-id+ :row 5:visualization_settings (link-card-viz-setting "dataset" model-id)}](testing "extract and store"(let [extraction (serdes/with-cache (into [] (extract/extract {})))@@ -630,16 +636,20 @@tab-eid-2 :entity_id} {:name "Tab 2" :position 1 :dashboard_id dashboard-id}:model/DashboardCard _ {:dashboard_id dashboard-id:card_id card-id-1- :dashboard_tab_id tab-id-1}+ :dashboard_tab_id tab-id-1+ :row 0}:model/DashboardCard _ {:dashboard_id dashboard-id:card_id card-id-2- :dashboard_tab_id tab-id-1}+ :dashboard_tab_id tab-id-1+ :row 1}:model/DashboardCard _ {:dashboard_id dashboard-id:card_id card-id-1- :dashboard_tab_id tab-id-2}+ :dashboard_tab_id tab-id-2+ :row 0}:model/DashboardCard _ {:dashboard_id dashboard-id:card_id card-id-2- :dashboard_tab_id tab-id-2}]+ :dashboard_tab_id tab-id-2+ :row 1}](let [extraction (serdes/with-cache (into [] (extract/extract {})))](storage/store! (seq extraction) (storage.files/file-writer dump-dir)))(testing "ingest and load"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex f9f7e8f0d77..a37afcfa56c 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -300,7 +300,8 @@:model/DashboardCard_{:action_id action-id- :dashboard_id other-dash-id}]+ :dashboard_id other-dash-id+ :row 1}](testing "table and database are extracted as [db schema table] triples"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c1-id))](is (=? {:serdes/meta [{:model "Card" :id c1-eid :label "some_question"}]@@ -472,6 +473,24 @@(serdes/extract-all "Dashboard")(ids-by-model "Dashboard")))))))))+(deftest dashcard-order-test+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc+ [:model/Collection {coll-id :id} {:name "Some Collection"}+ :model/Card {card-id :id} {:name "Some Question", :collection_id coll-id}+ :model/Dashboard {dash-id :id} {:name "Some Dashboard", :collection_id coll-id}+ :model/DashboardTab {second-tab :id} {:dashboard_id dash-id :name "Second" :position 1}+ :model/DashboardTab {first-tab :id} {:dashboard_id dash-id :name "First" :position 0}+ :model/DashboardCard {in-second :entity_id} {:card_id card-id :dashboard_id dash-id+ :dashboard_tab_id second-tab}+ :model/DashboardCard {in-first-b :entity_id} {:card_id card-id :dashboard_id dash-id+ :dashboard_tab_id first-tab :entity_id "bbbbbbbbbbbbbbbbbbbbb"}+ :model/DashboardCard {in-first-a :entity_id} {:card_id card-id :dashboard_id dash-id+ :dashboard_tab_id first-tab :entity_id "aaaaaaaaaaaaaaaaaaaaa"}]+ (testing "dashcards are ordered by their tab's position, not its local id, then by entity_id at the same place"+ (is (= [in-first-a in-first-b in-second]+ (map :entity_id (:dashcards (ts/extract-one "Dashboard" dash-id)))))))))+(deftest dashboard-card-series-test(mt/with-empty-h2-app-db!(ts/with-temp-dpc@@ -481,7 +500,7 @@:model/Card {c3-id :id, c3-eid :entity_id} {:name "Series Question B", :collection_id coll-id}:model/Dashboard {dash-id :id, dash-eid :entity_id} {:name "Shared Dashboard", :collection_id coll-id}:model/DashboardCard {dc1-id :id, dc1-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}- :model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}+ :model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id, :row 1}:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}](testing "Inlined dashcards include their series' card entity IDs"@@ -987,6 +1006,41 @@(testing "collection_id is nil"(is (nil? (:collection_id ser)))))))))+(deftest table-owner-serdes-test+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/User {user-id :id} {:email "[email protected]"}+ :model/Database {db-id :id} {:name "My Database"}+ :model/Table {table-id :id} {:name "Owned Table" :db_id db-id :owner_user_id user-id}]+ (testing "the owner is exported by email, not by local id"+ (is (= "[email protected]" (:owner_user_id (ts/extract-one "Table" table-id))))))))++(deftest sync-state-serdes-test+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"+ :engine :postgres+ :dbms_version {:version "14.0"}+ :initial_sync_status "aborted"+ :initial_sync_error "Connection refused"+ :metadata_sync_schedule "0 10 * * * ? *"+ :cache_field_values_schedule "0 0 16 * * ? *"}+ :model/Table {table-id :id} {:name "My Table" :db_id db-id :initial_sync_status "incomplete"}]+ (testing "what sync detects or schedules on the exporting instance is not exported"+ (is (=? {:name "My Database"} (ts/extract-one "Database" db-id)))+ (is (= #{} (set (keys (select-keys (ts/extract-one "Database" db-id)+ [:dbms_version :initial_sync_status :initial_sync_error+ :metadata_sync_schedule :cache_field_values_schedule])))))+ (is (not (contains? (ts/extract-one "Table" table-id) :initial_sync_status)))))))++(deftest setting-from-env-var-test+ (let [exported-site-name #(some (fn [{k :key v :value}] (when (= :site-name k) [v]))+ (serdes/extract-all "Setting" {}))]+ (mt/with-temporary-setting-values [site-name "From the app DB"]+ (testing "a Setting set in the app DB is exported"+ (is (= ["From the app DB"] (exported-site-name))))+ (mt/with-temp-env-var-value! [mb-site-name "From an env var"]+ (testing "a Setting set by an env var is not exported"+ (is (nil? (exported-site-name))))))))+(deftest implicit-action-test(mt/with-empty-h2-app-db!(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex a549cec7c88..48d09d7bd72 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -3,7 +3,9 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [clojure.walk :as walk][java-time.api :as t]+ [medley.core :as m][metabase-enterprise.serialization.test-util :as ts][metabase-enterprise.serialization.v2.extract :as serdes.extract][metabase-enterprise.serialization.v2.ingest :as serdes.ingest]@@ -318,6 +320,21 @@(load! (assoc field :dimensions []))(is (not (t2/exists? :model/Dimension :field_id (:id email)))))))))+(deftest table-owner-import-test+ (testing "a Table's owner is imported by email, and an older file's local id is kept"+ (mt/with-empty-h2-app-db!+ (let [db (ts/create! :model/Database :name "my-db")+ owner (ts/create! :model/User :first_name "Table" :last_name "Owner" :email "[email protected]")+ other (ts/create! :model/User :first_name "Other" :last_name "User" :email "[email protected]")+ table (ts/create! :model/Table :name "customers" :db_id (:id db) :owner_user_id (:id owner))+ file (ts/extract-one "Table" (:id table))+ load! (fn [entity] (serdes.load/load-metabase! (ts/ingestion-in-memory [entity])))]+ (t2/update! :model/Table (:id table) {:owner_user_id nil})+ (load! file)+ (is (= (:id owner) (t2/select-one-fn :owner_user_id :model/Table (:id table))))+ (load! (assoc file :owner_user_id (:id other)))+ (is (= (:id other) (t2/select-one-fn :owner_user_id :model/Table (:id table))))))))+(deftest legacy-table-user-settings-fields-import-test(testing "a settings file written before Field settings got files of their own replaces its Table's Field settings"(mt/with-empty-h2-app-db!@@ -1843,7 +1860,14 @@(reset! target1s target)(reset! target-eid (:entity_id target))(reset! native-eid "nativecardtagseid0001")- (reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))+ (reset! serialized (into [] (serdes.extract/extract {:no-settings true})))+ (testing "the export names the referenced card by its entity_id, not its local id"+ (let [exported-name (str "#" (:entity_id target) "-bh-population-model")]+ (is (=? {:dataset_query {:stages [{:native (format "SELECT * FROM {{%s}}" exported-name)+ :template-tags [{:name exported-name+ :display-name (u.humanization/name->human-readable-name+ :simple exported-name)}]}]}}+ (m/find-first #(= "nativecardtagseid0001" (:entity_id %)) @serialized)))))))(ts/with-db dest-db;; pre-create some cards so the referenced card can't land on its source id(let [db (ts/create! :model/Database :name "other-db")@@ -1875,8 +1899,46 @@(-> query :stages first :template-tags)))(testing "re-importing over the existing card (the update path) repairs the same way"(serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))+ (is (= query (stored-query))))+ (testing "an export that names the referenced card by its source id loads the same way"+ (let [tag-name #(str "#" % "-bh-population-model")+ display-name #(u.humanization/name->human-readable-name :simple (tag-name %))+ source-id (:id @target1s)]+ (serdes.load/load-metabase!+ (ts/ingestion-in-memory+ (walk/postwalk #(cond+ (= % (display-name @target-eid)) (display-name source-id)+ (string? %) (str/replace % (tag-name @target-eid) (tag-name source-id))+ :else %)+ @serialized))))+ (is (= query (stored-query))))+ (testing "an export whose tag name names another card's entity_id is repaired from :card-id"+ (let [tag-name #(str "#" % "-bh-population-model")+ display-name #(u.humanization/name->human-readable-name :simple (tag-name %))+ other-eid "otherCardEntityId0001"]+ (serdes.load/load-metabase!+ (ts/ingestion-in-memory+ (walk/postwalk #(cond+ (= % (display-name @target-eid)) (display-name other-eid)+ (string? %) (str/replace % (tag-name @target-eid) (tag-name other-eid))+ :else %)+ @serialized))))(is (= query (stored-query))))))))))+(deftest load-database-with-user-scheduled-syncs-test+ (testing "a Database file from an older export whose details let the user schedule syncs keeps its schedules"+ (mt/with-empty-h2-app-db!+ (let [db (ts/create! :model/Database :name "scheduled-db" :engine :postgres)+ file (assoc (ts/extract-one "Database" (:id db))+ :details {:let-user-control-scheduling true}+ :metadata_sync_schedule "0 10 * * * ? *"+ :cache_field_values_schedule "0 0 16 * * ? *")]+ (t2/delete! :model/Database (:id db))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [file]))+ (is (=? {:metadata_sync_schedule "0 10 * * * ? *"+ :cache_field_values_schedule "0 0 16 * * ? *"}+ (t2/select-one :model/Database :name "scheduled-db")))))))+(deftest load-action-test(let [serialized (atom nil)eid (u/generate-nano-id)]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.cljindex f0fadaa2b62..d490ca6c72b 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj@@ -37,7 +37,6 @@[metabase.search.core :as search][metabase.search.test-util :as search.tu][metabase.test :as mt]- [metabase.util :as u][metabase.util.log :as log][metabase.util.yaml :as yaml])(:import@@ -58,6 +57,9 @@;; instance-specific build version, no longer serialized (GHY-4013). Legacy baseline fixtures still;; carry it, which exercises that such exports still import cleanly now that it's skipped.:metabase_version+ ;; sync state, no longer serialized; legacy baseline fixtures still carry it+ :dbms_version+ :initial_sync_status;; result_metadata is non-deterministic for dashboard/document cards because the Card before-update hook;; re-computes it without :verified-result-metadata? set. Fixing this properly requires making serdes;; load set :verified-result-metadata? on Card updates, which is not straightforward.@@ -75,26 +77,14 @@(map (partial strip-base-path dir))(into (sorted-set))))-(def ^:private slug-id-prefix-re- #"#\d+([- ])")--(defn- replace-entropy- [s]- ;; Template tag slugs have the form `#1-my-card-name`, and their default display names `#1 My Card Name`, where `1`- ;; is the record ID. Record IDs depend on import order which is non-deterministic. Replace them with a deterministic- ;; placeholder.- (str/replace s slug-id-prefix-re "#<some-id>$1"))-(defn read-yaml"Reads a YAML file and returns Clojure data, with ignored fields removed."[file](walk/postwalk(fn [x]- (cond- (map? x) (reduce dissoc x ignored-fields)- (string? x) (replace-entropy x)- (keyword? x) (keyword (replace-entropy (u/qualified-name x)))- :else x))+ (if (map? x)+ (reduce dissoc x ignored-fields)+ x))(yaml/parse-string (slurp file))))(defn non-empty-diff [diff]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.cljnew file mode 100644index 00000000000..db0fc03a1df--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.clj@@ -0,0 +1,89 @@+(ns metabase-enterprise.serialization.v2.stability-test+ "Exports must be byte-stable: once content has been through one load, loading its export into an empty app DB with+ other local ids and exporting it again gives the same YAML for every entity, so no file depends on local ids."+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.serialization.v2.extract :as extract]+ [metabase-enterprise.serialization.v2.ingest :as ingest]+ [metabase-enterprise.serialization.v2.load :as load]+ [metabase-enterprise.serialization.v2.round-trip-test :as round-trip-test]+ [metabase-enterprise.serialization.v2.storage :as storage]+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]+ [metabase.models.serialization :as serdes]+ [metabase.search.test-util :as search.tu]+ [metabase.test :as mt]+ [toucan2.core :as t2])+ (:import+ (java.io File)+ (java.nio.file Files)+ (java.nio.file.attribute FileAttribute)+ (org.apache.commons.io FileUtils)))++(set! *warn-on-reflection* true)++(def ^:private extract-opts {:include-field-values true :include-metabot true})++(defn- temp-dir ^File []+ (.toFile (Files/createTempDirectory "serdes_stability" (make-array FileAttribute 0))))++(defn- load-dir! [^File dir]+ (serdes/with-cache+ (load/load-metabase! (ingest/ingest-yaml dir) :reindex? false)))++(defn- exported-yaml!+ "The YAML of every exported entity, by its `:serdes/meta`; the export is also stored into `dir` when given."+ ([] (exported-yaml! nil))+ ([dir]+ (serdes/with-cache+ (let [entities (into [] (extract/extract extract-opts))]+ (some->> dir storage.files/file-writer (storage/store! entities))+ (into {} (map (juxt :serdes/meta storage.util/entity-yaml)) entities)))))++(defn- shift-local-ids!+ "Make every identity column of the H2 app DB start at a different number past its seeded rows and `start`, so loaded+ entities get other local ids."+ [start]+ (doseq [[i {table :table_name column :column_name}]+ (map-indexed vector (t2/query ["SELECT table_name, column_name FROM information_schema.columns+ WHERE is_identity = 'YES' AND table_schema = 'PUBLIC'+ ORDER BY table_name, column_name"]))+ :let [max-query (format "SELECT MAX(\"%s\") AS \"max\" FROM \"%s\"" column table)+ seeded-max (:max (first (t2/query [max-query])))]]+ (t2/query [(format "ALTER TABLE \"%s\" ALTER COLUMN \"%s\" RESTART WITH %d"+ table column (+ (or seeded-max 0) start (* 37 i)))])))++(defn- export-of-load!+ "Load `dir` into an empty app DB with local ids past `start`, store its export into `out-dir` and return+ [[exported-yaml!]]."+ [dir out-dir start]+ (mt/with-empty-h2-app-db!+ (shift-local-ids! start)+ (load-dir! dir)+ (exported-yaml! out-dir)))++(defn- changed+ "The entities whose YAML differs between the `before` and `after` exports, with both versions."+ [before after]+ (into (sorted-map-by #(compare (pr-str %1) (pr-str %2)))+ (for [k (into (set (keys before)) (keys after))+ :when (not= (get before k) (get after k))]+ [k [(get before k) (get after k)]])))++(deftest export-load-export-test+ (search.tu/with-index-disabled+ (let [[dir-a dir-b dir-c :as dirs] (repeatedly 3 temp-dir)]+ (try+ (testing "the first export normalizes the baseline: no export wrote it, and a load re-infers model metadata"+ (export-of-load! round-trip-test/source-dir dir-a 1))+ (let [before (export-of-load! dir-a dir-b 1000)]+ (testing "the baseline covers many entities"+ (is (< 200 (count before))))+ (testing "exporting the same app DB twice gives the same YAML"+ (is (= {} (mt/with-empty-h2-app-db!+ (load-dir! dir-b)+ (changed (exported-yaml!) (exported-yaml!))))))+ (testing "loading an export into an app DB with other local ids and exporting it again gives the same YAML"+ (is (= {} (changed before (export-of-load! dir-b dir-c 9000))))))+ (finally+ (run! #(FileUtils/deleteDirectory ^File %) dirs))))))diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 4dd948181a3..f1ac4df8d8d 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -404,6 +404,13 @@;;; | SERIALIZATION |;;; +----------------------------------------------------------------------------------------------------------------++(defn- dashcard-sort-key+ "The sort key fn for the dashcards of `dashboard` in its file: the position and the `entity_id` of their tab, then+ row, column and `entity_id`, so the order does not depend on local ids."+ [dashboard]+ (let [tab-order (into {} (map (juxt :id (juxt :position :entity_id))) (:tabs dashboard))]+ (juxt (comp tab-order :dashboard_tab_id) :row :col :entity_id)))+(defmethod serdes/make-spec "Dashboard" [_model-name opts]{:copy [:archived :archived_directly :auto_apply_filters :caveats :collection_position:description :embedding_params :enable_embedding :embedding_type :entity_id :name@@ -420,7 +427,7 @@:made_public_by_id (serdes/fk :model/User):parameters {:export serdes/export-parameters :import serdes/import-parameters}:tabs (serdes/nested :model/DashboardTab :dashboard_id (merge {:sort-by (juxt :position :entity_id)} opts))- :dashcards (serdes/nested :model/DashboardCard :dashboard_id (merge {:sort-by (juxt :dashboard_tab_id :row :col)} opts))}+ :dashcards (serdes/nested :model/DashboardCard :dashboard_id (merge {:sort-by-parent dashcard-sort-key} opts))}:coerce {:parameters [:maybe [:sequential ::parameters.schema/parameter]]}:defaults {:archived false:archived_directly falsediff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljcindex fdc7b9290d0..eef1ac136e4 100644--- a/src/metabase/lib/native.cljc+++ b/src/metabase/lib/native.cljc@@ -360,13 +360,6 @@[s](str/replace s #"[.*+?^${}()|\[\]\\]" (fn [c] (str "\\" c))))-(defn- replace-tag-in-text- [text old-name new-name]- (str/replace text- (re-pattern (str "\\{\\{\\s*" (regex-escape old-name) "\\s*\\}\\}"))- ;; function replacement so a `$` in the new name can't be misread as a match reference- (constantly (str "{{" new-name "}}"))))-(defn- rename-tag"Rename a template tag, replacing its `:display-name` only if it was the humanized default for theold name (same rule as [[rename-template-tag]])."@@ -376,10 +369,16 @@(assoc :display-name (u.humanization/name->human-readable-name :simple new-name))))(mu/defn rename-template-tags-in-text :- :string- "`text` with its `{{...}}` references renamed per `renames`, a map of old tag name => new tag name."+ "`text` with its `{{...}}` references renamed per `renames`, a map of old tag name => new tag name, all in one pass,+ so a new name that is another tag's old name is not renamed again."[text :- :stringrenames :- [:map-of :string :string]]- (reduce-kv replace-tag-in-text text renames))+ (if (empty? renames)+ text+ (str/replace text+ (re-pattern (str "\\{\\{\\s*(" (str/join "|" (map regex-escape (keys renames))) ")\\s*\\}\\}"))+ (fn [[_match old-name]]+ (str "{{" (get renames old-name) "}}")))))(mu/defn rename-template-tags :- ::lib.schema.template-tag/template-tag-map"`tags`, a map of tag name => tag, with the tags in `renames` renamed the way [[replace-template-tag-names]] renamesdiff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 8694541a4da..fc59cc98c41 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -1177,13 +1177,52 @@[:measure opts id][:measure (export-mbql-map opts) id]))+(defn card-template-tag-export-rename+ "Exported name for a card template tag whose `#<id>-slug` name embeds a local card id: the id is swapped for the+ `entity_id` of the tag's card, the slug is kept verbatim. Nil when the name doesn't embed an id."+ [{tag-type :type, :keys [card-id], tag-name :name}]+ (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))+ (when-let [[_ _embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]+ (when-let [card-entity-id (fk-elide (*export-fk* card-id :model/Card))]+ (str "#" card-entity-id suffix)))))++(defn card-template-tag-rename+ "New name for a card template tag whose name embeds the `entity_id` of its (already remapped) `:card-id`, a+ different local id, or another `entity_id`: the embedded value is swapped for the id, the slug is kept verbatim. Nil+ when they already agree or the name doesn't embed an id."+ [{tag-type :type, :keys [card-id], tag-name :name}]+ (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))+ (let [entity-id-prefix (some->> (fk-elide (*export-fk* card-id :model/Card)) (str "#"))+ entity-id-suffix (when (and entity-id-prefix (str/starts-with? tag-name entity-id-prefix))+ (subs tag-name (count entity-id-prefix)))]+ (if (and entity-id-suffix (or (= "" entity-id-suffix) (str/starts-with? entity-id-suffix "-")))+ (str "#" card-id entity-id-suffix)+ (if-let [[_ embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]+ (when (not= (parse-long embedded-id) card-id)+ (str "#" card-id suffix))+ (when-let [[_ suffix] (re-matches #"#[A-Za-z0-9_-]{21}(-.*)?" tag-name)]+ (str "#" card-id suffix)))))))++(defn card-template-tag-renames+ "The `{old-name new-name}` map of the `tags` that `rename`, a fn of one tag, gives a new name."+ [rename tags]+ (into {}+ (keep (fn [{tag-name :name, :as tag}]+ (some->> (rename tag) (vector tag-name))))+ tags))++(defn- rename-card-template-tags+ "Renames the template tags of `query`, and the `{{#id-slug}}` refs in its native text, with `rename`."+ [query rename]+ (lib/replace-template-tag-names query (card-template-tag-renames rename (lib/all-template-tags query))))+(defn export-mbql"Given an MBQL expression, convert it to an EDN structure and turn the non-portable Database, Table and Field IDsinside it into portable references."[x](let [x (cond-> x(and (map? x) (= :mbql/query (:lib/type x)))- lib/prepare-for-serialization)]+ (-> (rename-card-template-tags card-template-tag-export-rename) lib/prepare-for-serialization))];; if required UUIDs are already calculated don't recalculate when we recurse.(binding [*required-lib-uuids-for-export* (or *required-lib-uuids-for-export* (collect-required-lib-uuids x))](cond@@ -1314,30 +1353,13 @@import-mbql-update-refsimport-mbql-update-maps))-(defn card-template-tag-rename- "New name for a card template tag whose `#<id>-slug` name embeds a different id than its (already- remapped) `:card-id`: the id is swapped, the slug is kept verbatim. Nil when they already agree or- the name doesn't embed an id."- [{tag-type :type, :keys [card-id], tag-name :name}]- (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))- (when-let [[_ embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]- (when (not= (parse-long embedded-id) card-id)- (str "#" card-id suffix)))))-(defn- repair-card-template-tag-names- "Card template tag names (and the `{{#id-slug}}` refs in the native text) embed the referenced- card's id, which goes stale when [[import-mbql-map]] remaps `:card-id` to the local card's id.- Swap the embedded id for the remapped one, leaving the slug as it was exported."+ "Card template tag names (and the `{{#id-slug}}` refs in the native text) embed the referenced card's `entity_id`,+ or in older exports its source id, which [[import-mbql-map]] doesn't remap. Swap it for the local id of the remapped+ `:card-id`, leaving the slug as it was exported."[x]- (if (= (:lib/type x) :mbql/query)- (lib/replace-template-tag-names- x- (into {}- (keep (fn [{tag-name :name, :as tag}]- (when-let [new-name (card-template-tag-rename tag)]- [tag-name new-name])))- (lib/all-template-tags x)))- x))+ (cond-> x+ (= (:lib/type x) :mbql/query) (rename-card-template-tags card-template-tag-rename)))(defn- legacy-mbql-query->mbql5"Converts an imported legacy MBQL query to MBQL 5, returning `x` unchanged if it isn't one or conversion fails.@@ -1947,10 +1969,14 @@:import #(*import-fk* % model)))))(defn nested- "Nested entities; `opts` may give `:sort-by`, `:key-field` and `:delete-children!`, a fn of the parent id."+ "Nested entities. `opts` may give `:key-field`, `:delete-children!` (a fn of the parent id), and either `:sort-by`+ or `:sort-by-parent`, a fn of the parent returning the `:sort-by` fn, which wins over `:sort-by` and is not passed+ on to the children's own nested entities."[model backward-fk opts](let [model-name (name model)- sorter (:sort-by opts :entity_id)+ sorter (if-let [sort-by-parent (:sort-by-parent opts)]+ sort-by-parent+ (constantly (:sort-by opts :entity_id)))key-field (:key-field opts :entity_id)delete-children! (:delete-children! opts #(models.db/delete-children! model backward-fk %))]{::nested true@@ -1963,8 +1989,8 @@;; `nil? data` check is for `extract-one` case in tests; make sure to add empty vectors in;; `extract-query` implementations for nested collections(try- (->> (sort-by sorter data)- (mapv #(extract-one model-name opts %)))+ (->> (sort-by (sorter current) data)+ (mapv #(extract-one model-name (dissoc opts :sort-by-parent) %)))(catch Exception e(throw (ex-info (format "Error extracting nested %s" model){:model modeldiff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex cc47812f964..3ffc16f90c1 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -192,23 +192,31 @@serdes/import-mbql(lib/normalize :metabase.lib.schema.template-tag/template-tag-map))]{:tags tags- :renames (into {}- (keep (fn [[tag-name tag]]- (some->> (serdes/card-template-tag-rename tag) (vector tag-name))))- tags)}))+ :renames (serdes/card-template-tag-renames serdes/card-template-tag-rename (vals tags))}))++(defn- export-renames+ "The renames of the card tags in `template-tags` whose `#<id>-slug` name embeds this instance's card id."+ [template-tags]+ (serdes/card-template-tag-renames serdes/card-template-tag-export-rename (vals template-tags)))(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]{:copy [:archived :description :entity_id :name]:skip []:transform {:collection_id (serdes/fk :model/Collection):creator_id (serdes/fk :model/User)- :content {:export identity+ :content {:export-with-context+ (fn [current _ content]+ (let [renames (export-renames (:template_tags current))]+ (cond-> content+ (and (string? content) (seq renames)) (lib/rename-template-tags-in-text renames)))):import-with-context(fn [current _ content](let [{:keys [renames]} (import-template-tags (:template_tags current))](cond-> content(and (string? content) (seq renames)) (lib/rename-template-tags-in-text renames))))}- :template_tags {:export serdes/export-mbql+ :template_tags {:export (fn [template-tags]+ (serdes/export-mbql+ (some-> template-tags (lib/rename-template-tags (export-renames template-tags))))):import (fn [template-tags](let [{:keys [tags renames]} (import-template-tags template-tags)](some-> tags (lib/rename-template-tags renames))))}}diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.cljindex 3bea2cea278..81707f3c457 100644--- a/src/metabase/settings/models/setting.clj+++ b/src/metabase/settings/models/setting.clj@@ -98,9 +98,9 @@(declare export?)(defmethod serdes/extract-all "Setting" [_model _opts]- (for [{:keys [key value]} (admin-writable-site-wide-settings- :getter (partial get-value-of-type :string))- :when (export? key)]+ (for [{:keys [key value is_env_setting]} (admin-writable-site-wide-settings+ :getter (partial get-value-of-type :string))+ :when (and (export? key) (not is_env_setting))]{:serdes/meta [{:model "Setting" :id (name key)}]:key key:value value}))diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.cljindex 322d6b62397..5ded5a410fc 100644--- a/src/metabase/warehouse_schema/models/table.clj+++ b/src/metabase/warehouse_schema/models/table.clj@@ -614,15 +614,19 @@(defmethod serdes/make-spec "Table" [_model-name _opts]{:copy [:name :description :entity_type :active :display_name :visibility_type :schema- :points_of_interest :caveats :show_in_getting_started :field_order :initial_sync_status :is_upload+ :points_of_interest :caveats :show_in_getting_started :field_order :is_upload:database_require_filter :is_defective_duplicate :unique_table_helper :is_writable :data_authority- :data_source :owner_email :owner_user_id :is_published]- :skip [:estimated_row_count :view_count :transform_target]+ :data_source :owner_email :is_published]+ :skip [:estimated_row_count :initial_sync_status :view_count :transform_target]:transform {:archived_at (serdes/date):deactivated_at (serdes/date):data_layer (serdes/optional-kw):db_id (serdes/fk :model/Database):collection_id (serdes/fk :model/Collection)+ ;; older exports copied the local user id; re-importing those files on the same instance, e.g. a+ ;; remote sync pull before the first new export, keeps the right owner, where nil would drop it+ :owner_user_id (update (serdes/fk :model/User) :import+ (fn [import-user] #(if (int? %) % (import-user %)))):transform_id (serdes/fk :model/Transform)}:defaults {:is_defective_duplicate false:is_published falsediff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 770ceb9b128..10a9ea0f23e 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -666,22 +666,31 @@(defmethod serdes/make-spec "Database"[_model-name _opts];; Connection details are never exported. Import is unconditional.- (let [details-transform {:export-with-context (fn [_current _ _details] ::serdes/skip)- :import identity}]- {:copy [:auto_run_queries :cache_field_values_schedule :caveats :dbms_version+ (let [details-transform {:export-with-context (fn [_current _ _details] ::serdes/skip)+ :import identity}+ schedule-transform {:export-with-context (fn [_current _ _schedule] ::serdes/skip)+ :import identity}]+ {:copy [:auto_run_queries :caveats:description :engine :is_audit :is_attached_dwh :is_full_sync :is_on_demand- :default_schema :metadata_sync_schedule :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled+ :default_schema :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled:uploads_schema_name :uploads_table_prefix]:skip [;; deprecated field:cache_ttl;; describes a sync on the source instance, and may name its connection details- :initial_sync_error]- :transform {:details details-transform+ :initial_sync_error+ ;; detected by sync, and set again by the importing instance's sync+ :dbms_version]+ :transform {;; randomized per instance unless the connection details, which we never export, let the user+ ;; schedule syncs; older files with such details carry the schedules those details require+ :cache_field_values_schedule schedule-transform+ :metadata_sync_schedule schedule-transform+ :details details-transform:write_data_details details-transform:admin_details details-transform:creator_id (serdes/fk :model/User):router_database_id (serdes/fk :model/Database)- :initial_sync_status {:export identity :import (constantly "complete")}+ :initial_sync_status {:export-with-context (fn [_current _ _status] ::serdes/skip)+ :import (constantly "complete")};; Never trust an archive's `is_sample`: the real Sample Database is excluded from export;; ([[warehouses.db/databases-for-serdes-reducible]]), so any incoming `is_sample true` is forged --;; and marking an imported row as the sample DB would let it ride sample-only code paths. Import false.diff --git a/test/metabase/lib/native_test.cljc b/test/metabase/lib/native_test.cljcindex 19bcbd81bf2..bba2b4533e5 100644--- a/test/metabase/lib/native_test.cljc+++ b/test/metabase/lib/native_test.cljc@@ -902,6 +902,15 @@:id "6ebf6c2e-d6e2-449e-97b7-7005047928e5"}}renames))))))+(deftest ^:parallel rename-template-tags-in-text-one-pass-test+ (testing "a new name that is another tag's old name is not renamed again"+ (is (= "{{#2}} {{#1-orders}} {{#2}}"+ (lib.native/rename-template-tags-in-text "{{ #1 }} {{#2-orders}} {{#1}}" {"#1" "#2", "#2-orders" "#1-orders"})))+ (is (= "{{#1}} {{#2}}"+ (lib.native/rename-template-tags-in-text "{{#2}} {{#1}}" {"#1" "#2", "#2" "#1"})))+ (is (= "{{#7}} {{#9}} {{#70}}"+ (lib.native/rename-template-tags-in-text "{{#ent}} {{#7}} {{#70}}" {"#ent" "#7", "#7" "#9"})))))+(deftest ^:parallel replace-template-tag-names-noop-test(let [query (card-tag-query "select * from {{#133-some-old-slug}}"{:type :carddiff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yamlindex 195d2497b8b..eeefb37e851 100644--- a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml@@ -10,7 +10,7 @@ dataset_query:stages:- native: |-SELECT *- FROM {{#1-basic_aggregations}}+ FROM {{#h5F2EjHsRd73Dqqh8sAtd-basic_aggregations}}WHERE {{snippet: Active Product Filter}}ORDER BY "count" DESCLIMIT 10@@ -18,7 +18,7 @@ dataset_query:- card-id: h5F2EjHsRd73Dqqh8sAtddisplay-name: Basic Aggregationsid: cc000001-0000-0000-0000-000000000001- name: '#1-basic_aggregations'+ name: '#h5F2EjHsRd73Dqqh8sAtd-basic_aggregations'type: card- display-name: 'Snippet: Active Product Filter'id: cc000002-0000-0000-0000-000000000002diff --git a/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml b/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yamlindex e19ec573990..c616722c25a 100644--- a/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml+++ b/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml@@ -2,7 +2,7 @@ name: Active Recent Filterdescription: Combines Active Product Filter with Date Range Filterentity_id: c5PMqT66HwWoy0m52WTI5content: '{{snippet: Active Product Filter}} AND {{snippet: Date Range Filter}} AND {{snippet: Missing Filter}}- AND category = {{category}} AND id IN (SELECT id FROM {{#1-basic-aggregations}})'+ AND category = {{category}} AND id IN (SELECT id FROM {{#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations}})'creator_id: [email protected]template_tags:'snippet: Active Product Filter':@@ -29,11 +29,11 @@ template_tags:id: dd000004-0000-0000-0000-000000000004name: categorytype: text- '#1-basic-aggregations':+ '#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations':card-id: h5F2EjHsRd73Dqqh8sAtd- display-name: '#1 Basic Aggregations'+ display-name: '#h5F2EjHsRd73Dqqh8sAtd Basic Aggregations'id: dd000005-0000-0000-0000-000000000005- name: '#1-basic-aggregations'+ name: '#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations'type: cardserdes/meta:- model: NativeQuerySnippet