Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Make serialization output byte-stable (in some cases) (#84948)

Alexander Polyankin930efb1285fd
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
index 08132efa653..4420c9746cd 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj
@@ -110,7 +110,7 @@
(defn- clean-entity
"Removes any comparison-confounding fields, like `:created_at`."
[entity]
- (dissoc entity :created_at :result_metadata :metadata_sync_schedule :cache_field_values_schedule))
+ (dissoc entity :created_at :result_metadata))
;; one generated random dump feeds all storage+ingestion assertions; splitting regenerates it per test
#_{:clj-kondo/ignore [:metabase/i-like-making-cams-eyes-bleed-with-horrifically-long-tests]}
@@ -297,7 +297,7 @@
clean-entity)))))
(testing "for Databases"
(doseq [{:keys [name] :as db} (get @entities "Database")]
- (is (= (assoc (clean-entity db) :initial_sync_status "complete")
+ (is (= (clean-entity db)
(-> (ts/extract-one "Database" [:= :name name])
clean-entity)))))
(testing "for Tables"
@@ -501,16 +501,22 @@
dashboard-name :name} {:name "Test Dashboard"
:collection_id coll-id}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 0
:visualization_settings (link-card-viz-setting "collection" coll-id)}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 1
:visualization_settings (link-card-viz-setting "database" db-id)}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 2
:visualization_settings (link-card-viz-setting "table" table-id)}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 3
:visualization_settings (link-card-viz-setting "dashboard" dash-id)}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 4
:visualization_settings (link-card-viz-setting "card" card-id)}
:model/DashboardCard _ {:dashboard_id dashboard-id
+ :row 5
:visualization_settings (link-card-viz-setting "dataset" model-id)}]
(testing "extract and store"
(let [extraction (serdes/with-cache (into [] (extract/extract {})))
@@ -630,16 +636,20 @@
tab-eid-2 :entity_id} {:name "Tab 2" :position 1 :dashboard_id dashboard-id}
:model/DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-1
- :dashboard_tab_id tab-id-1}
+ :dashboard_tab_id tab-id-1
+ :row 0}
:model/DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-2
- :dashboard_tab_id tab-id-1}
+ :dashboard_tab_id tab-id-1
+ :row 1}
:model/DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-1
- :dashboard_tab_id tab-id-2}
+ :dashboard_tab_id tab-id-2
+ :row 0}
:model/DashboardCard _ {:dashboard_id dashboard-id
:card_id card-id-2
- :dashboard_tab_id tab-id-2}]
+ :dashboard_tab_id tab-id-2
+ :row 1}]
(let [extraction (serdes/with-cache (into [] (extract/extract {})))]
(storage/store! (seq extraction) (storage.files/file-writer dump-dir)))
(testing "ingest and load"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index f9f7e8f0d77..a37afcfa56c 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -300,7 +300,8 @@
:model/DashboardCard
_
{:action_id action-id
- :dashboard_id other-dash-id}]
+ :dashboard_id other-dash-id
+ :row 1}]
(testing "table and database are extracted as [db schema table] triples"
(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c1-id))]
(is (=? {:serdes/meta [{:model "Card" :id c1-eid :label "some_question"}]
@@ -472,6 +473,24 @@
(serdes/extract-all "Dashboard")
(ids-by-model "Dashboard")))))))))
+(deftest dashcard-order-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc
+ [:model/Collection {coll-id :id} {:name "Some Collection"}
+ :model/Card {card-id :id} {:name "Some Question", :collection_id coll-id}
+ :model/Dashboard {dash-id :id} {:name "Some Dashboard", :collection_id coll-id}
+ :model/DashboardTab {second-tab :id} {:dashboard_id dash-id :name "Second" :position 1}
+ :model/DashboardTab {first-tab :id} {:dashboard_id dash-id :name "First" :position 0}
+ :model/DashboardCard {in-second :entity_id} {:card_id card-id :dashboard_id dash-id
+ :dashboard_tab_id second-tab}
+ :model/DashboardCard {in-first-b :entity_id} {:card_id card-id :dashboard_id dash-id
+ :dashboard_tab_id first-tab :entity_id "bbbbbbbbbbbbbbbbbbbbb"}
+ :model/DashboardCard {in-first-a :entity_id} {:card_id card-id :dashboard_id dash-id
+ :dashboard_tab_id first-tab :entity_id "aaaaaaaaaaaaaaaaaaaaa"}]
+ (testing "dashcards are ordered by their tab's position, not its local id, then by entity_id at the same place"
+ (is (= [in-first-a in-first-b in-second]
+ (map :entity_id (:dashcards (ts/extract-one "Dashboard" dash-id)))))))))
+
(deftest dashboard-card-series-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc
@@ -481,7 +500,7 @@
:model/Card {c3-id :id, c3-eid :entity_id} {:name "Series Question B", :collection_id coll-id}
:model/Dashboard {dash-id :id, dash-eid :entity_id} {:name "Shared Dashboard", :collection_id coll-id}
:model/DashboardCard {dc1-id :id, dc1-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
- :model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id}
+ :model/DashboardCard {dc2-eid :entity_id} {:card_id c1-id, :dashboard_id dash-id, :row 1}
:model/DashboardCardSeries _ {:card_id c3-id, :dashboardcard_id dc1-id, :position 1}
:model/DashboardCardSeries _ {:card_id c2-id, :dashboardcard_id dc1-id, :position 0}]
(testing "Inlined dashcards include their series' card entity IDs"
@@ -987,6 +1006,41 @@
(testing "collection_id is nil"
(is (nil? (:collection_id ser)))))))))
+(deftest table-owner-serdes-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/User {user-id :id} {:email "[email protected]"}
+ :model/Database {db-id :id} {:name "My Database"}
+ :model/Table {table-id :id} {:name "Owned Table" :db_id db-id :owner_user_id user-id}]
+ (testing "the owner is exported by email, not by local id"
+ (is (= "[email protected]" (:owner_user_id (ts/extract-one "Table" table-id))))))))
+
+(deftest sync-state-serdes-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Database {db-id :id} {:name "My Database"
+ :engine :postgres
+ :dbms_version {:version "14.0"}
+ :initial_sync_status "aborted"
+ :initial_sync_error "Connection refused"
+ :metadata_sync_schedule "0 10 * * * ? *"
+ :cache_field_values_schedule "0 0 16 * * ? *"}
+ :model/Table {table-id :id} {:name "My Table" :db_id db-id :initial_sync_status "incomplete"}]
+ (testing "what sync detects or schedules on the exporting instance is not exported"
+ (is (=? {:name "My Database"} (ts/extract-one "Database" db-id)))
+ (is (= #{} (set (keys (select-keys (ts/extract-one "Database" db-id)
+ [:dbms_version :initial_sync_status :initial_sync_error
+ :metadata_sync_schedule :cache_field_values_schedule])))))
+ (is (not (contains? (ts/extract-one "Table" table-id) :initial_sync_status)))))))
+
+(deftest setting-from-env-var-test
+ (let [exported-site-name #(some (fn [{k :key v :value}] (when (= :site-name k) [v]))
+ (serdes/extract-all "Setting" {}))]
+ (mt/with-temporary-setting-values [site-name "From the app DB"]
+ (testing "a Setting set in the app DB is exported"
+ (is (= ["From the app DB"] (exported-site-name))))
+ (mt/with-temp-env-var-value! [mb-site-name "From an env var"]
+ (testing "a Setting set by an env var is not exported"
+ (is (nil? (exported-site-name))))))))
+
(deftest implicit-action-test
(mt/with-empty-h2-app-db!
(ts/with-temp-dpc [:model/User {ann-id :id} {:first_name "Ann"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index a549cec7c88..48d09d7bd72 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -3,7 +3,9 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [clojure.walk :as walk]
[java-time.api :as t]
+ [medley.core :as m]
[metabase-enterprise.serialization.test-util :as ts]
[metabase-enterprise.serialization.v2.extract :as serdes.extract]
[metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
@@ -318,6 +320,21 @@
(load! (assoc field :dimensions []))
(is (not (t2/exists? :model/Dimension :field_id (:id email)))))))))
+(deftest table-owner-import-test
+ (testing "a Table's owner is imported by email, and an older file's local id is kept"
+ (mt/with-empty-h2-app-db!
+ (let [db (ts/create! :model/Database :name "my-db")
+ owner (ts/create! :model/User :first_name "Table" :last_name "Owner" :email "[email protected]")
+ other (ts/create! :model/User :first_name "Other" :last_name "User" :email "[email protected]")
+ table (ts/create! :model/Table :name "customers" :db_id (:id db) :owner_user_id (:id owner))
+ file (ts/extract-one "Table" (:id table))
+ load! (fn [entity] (serdes.load/load-metabase! (ts/ingestion-in-memory [entity])))]
+ (t2/update! :model/Table (:id table) {:owner_user_id nil})
+ (load! file)
+ (is (= (:id owner) (t2/select-one-fn :owner_user_id :model/Table (:id table))))
+ (load! (assoc file :owner_user_id (:id other)))
+ (is (= (:id other) (t2/select-one-fn :owner_user_id :model/Table (:id table))))))))
+
(deftest legacy-table-user-settings-fields-import-test
(testing "a settings file written before Field settings got files of their own replaces its Table's Field settings"
(mt/with-empty-h2-app-db!
@@ -1843,7 +1860,14 @@
(reset! target1s target)
(reset! target-eid (:entity_id target))
(reset! native-eid "nativecardtagseid0001")
- (reset! serialized (into [] (serdes.extract/extract {:no-settings true})))))
+ (reset! serialized (into [] (serdes.extract/extract {:no-settings true})))
+ (testing "the export names the referenced card by its entity_id, not its local id"
+ (let [exported-name (str "#" (:entity_id target) "-bh-population-model")]
+ (is (=? {:dataset_query {:stages [{:native (format "SELECT * FROM {{%s}}" exported-name)
+ :template-tags [{:name exported-name
+ :display-name (u.humanization/name->human-readable-name
+ :simple exported-name)}]}]}}
+ (m/find-first #(= "nativecardtagseid0001" (:entity_id %)) @serialized)))))))
(ts/with-db dest-db
;; pre-create some cards so the referenced card can't land on its source id
(let [db (ts/create! :model/Database :name "other-db")
@@ -1875,8 +1899,46 @@
(-> query :stages first :template-tags)))
(testing "re-importing over the existing card (the update path) repairs the same way"
(serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
+ (is (= query (stored-query))))
+ (testing "an export that names the referenced card by its source id loads the same way"
+ (let [tag-name #(str "#" % "-bh-population-model")
+ display-name #(u.humanization/name->human-readable-name :simple (tag-name %))
+ source-id (:id @target1s)]
+ (serdes.load/load-metabase!
+ (ts/ingestion-in-memory
+ (walk/postwalk #(cond
+ (= % (display-name @target-eid)) (display-name source-id)
+ (string? %) (str/replace % (tag-name @target-eid) (tag-name source-id))
+ :else %)
+ @serialized))))
+ (is (= query (stored-query))))
+ (testing "an export whose tag name names another card's entity_id is repaired from :card-id"
+ (let [tag-name #(str "#" % "-bh-population-model")
+ display-name #(u.humanization/name->human-readable-name :simple (tag-name %))
+ other-eid "otherCardEntityId0001"]
+ (serdes.load/load-metabase!
+ (ts/ingestion-in-memory
+ (walk/postwalk #(cond
+ (= % (display-name @target-eid)) (display-name other-eid)
+ (string? %) (str/replace % (tag-name @target-eid) (tag-name other-eid))
+ :else %)
+ @serialized))))
(is (= query (stored-query))))))))))
+(deftest load-database-with-user-scheduled-syncs-test
+ (testing "a Database file from an older export whose details let the user schedule syncs keeps its schedules"
+ (mt/with-empty-h2-app-db!
+ (let [db (ts/create! :model/Database :name "scheduled-db" :engine :postgres)
+ file (assoc (ts/extract-one "Database" (:id db))
+ :details {:let-user-control-scheduling true}
+ :metadata_sync_schedule "0 10 * * * ? *"
+ :cache_field_values_schedule "0 0 16 * * ? *")]
+ (t2/delete! :model/Database (:id db))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [file]))
+ (is (=? {:metadata_sync_schedule "0 10 * * * ? *"
+ :cache_field_values_schedule "0 0 16 * * ? *"}
+ (t2/select-one :model/Database :name "scheduled-db")))))))
+
(deftest load-action-test
(let [serialized (atom nil)
eid (u/generate-nano-id)]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index f0fadaa2b62..d490ca6c72b 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -37,7 +37,6 @@
[metabase.search.core :as search]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
- [metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.yaml :as yaml])
(:import
@@ -58,6 +57,9 @@
;; instance-specific build version, no longer serialized (GHY-4013). Legacy baseline fixtures still
;; carry it, which exercises that such exports still import cleanly now that it's skipped.
:metabase_version
+ ;; sync state, no longer serialized; legacy baseline fixtures still carry it
+ :dbms_version
+ :initial_sync_status
;; result_metadata is non-deterministic for dashboard/document cards because the Card before-update hook
;; re-computes it without :verified-result-metadata? set. Fixing this properly requires making serdes
;; load set :verified-result-metadata? on Card updates, which is not straightforward.
@@ -75,26 +77,14 @@
(map (partial strip-base-path dir))
(into (sorted-set))))
-(def ^:private slug-id-prefix-re
- #"#\d+([- ])")
-
-(defn- replace-entropy
- [s]
- ;; Template tag slugs have the form `#1-my-card-name`, and their default display names `#1 My Card Name`, where `1`
- ;; is the record ID. Record IDs depend on import order which is non-deterministic. Replace them with a deterministic
- ;; placeholder.
- (str/replace s slug-id-prefix-re "#<some-id>$1"))
-
(defn read-yaml
"Reads a YAML file and returns Clojure data, with ignored fields removed."
[file]
(walk/postwalk
(fn [x]
- (cond
- (map? x) (reduce dissoc x ignored-fields)
- (string? x) (replace-entropy x)
- (keyword? x) (keyword (replace-entropy (u/qualified-name x)))
- :else x))
+ (if (map? x)
+ (reduce dissoc x ignored-fields)
+ x))
(yaml/parse-string (slurp file))))
(defn non-empty-diff [diff]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.clj
new file mode 100644
index 00000000000..db0fc03a1df
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/stability_test.clj
@@ -0,0 +1,89 @@
+(ns metabase-enterprise.serialization.v2.stability-test
+ "Exports must be byte-stable: once content has been through one load, loading its export into an empty app DB with
+ other local ids and exporting it again gives the same YAML for every entity, so no file depends on local ids."
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.serialization.v2.extract :as extract]
+ [metabase-enterprise.serialization.v2.ingest :as ingest]
+ [metabase-enterprise.serialization.v2.load :as load]
+ [metabase-enterprise.serialization.v2.round-trip-test :as round-trip-test]
+ [metabase-enterprise.serialization.v2.storage :as storage]
+ [metabase-enterprise.serialization.v2.storage.files :as storage.files]
+ [metabase-enterprise.serialization.v2.storage.util :as storage.util]
+ [metabase.models.serialization :as serdes]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (java.io File)
+ (java.nio.file Files)
+ (java.nio.file.attribute FileAttribute)
+ (org.apache.commons.io FileUtils)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private extract-opts {:include-field-values true :include-metabot true})
+
+(defn- temp-dir ^File []
+ (.toFile (Files/createTempDirectory "serdes_stability" (make-array FileAttribute 0))))
+
+(defn- load-dir! [^File dir]
+ (serdes/with-cache
+ (load/load-metabase! (ingest/ingest-yaml dir) :reindex? false)))
+
+(defn- exported-yaml!
+ "The YAML of every exported entity, by its `:serdes/meta`; the export is also stored into `dir` when given."
+ ([] (exported-yaml! nil))
+ ([dir]
+ (serdes/with-cache
+ (let [entities (into [] (extract/extract extract-opts))]
+ (some->> dir storage.files/file-writer (storage/store! entities))
+ (into {} (map (juxt :serdes/meta storage.util/entity-yaml)) entities)))))
+
+(defn- shift-local-ids!
+ "Make every identity column of the H2 app DB start at a different number past its seeded rows and `start`, so loaded
+ entities get other local ids."
+ [start]
+ (doseq [[i {table :table_name column :column_name}]
+ (map-indexed vector (t2/query ["SELECT table_name, column_name FROM information_schema.columns
+ WHERE is_identity = 'YES' AND table_schema = 'PUBLIC'
+ ORDER BY table_name, column_name"]))
+ :let [max-query (format "SELECT MAX(\"%s\") AS \"max\" FROM \"%s\"" column table)
+ seeded-max (:max (first (t2/query [max-query])))]]
+ (t2/query [(format "ALTER TABLE \"%s\" ALTER COLUMN \"%s\" RESTART WITH %d"
+ table column (+ (or seeded-max 0) start (* 37 i)))])))
+
+(defn- export-of-load!
+ "Load `dir` into an empty app DB with local ids past `start`, store its export into `out-dir` and return
+ [[exported-yaml!]]."
+ [dir out-dir start]
+ (mt/with-empty-h2-app-db!
+ (shift-local-ids! start)
+ (load-dir! dir)
+ (exported-yaml! out-dir)))
+
+(defn- changed
+ "The entities whose YAML differs between the `before` and `after` exports, with both versions."
+ [before after]
+ (into (sorted-map-by #(compare (pr-str %1) (pr-str %2)))
+ (for [k (into (set (keys before)) (keys after))
+ :when (not= (get before k) (get after k))]
+ [k [(get before k) (get after k)]])))
+
+(deftest export-load-export-test
+ (search.tu/with-index-disabled
+ (let [[dir-a dir-b dir-c :as dirs] (repeatedly 3 temp-dir)]
+ (try
+ (testing "the first export normalizes the baseline: no export wrote it, and a load re-infers model metadata"
+ (export-of-load! round-trip-test/source-dir dir-a 1))
+ (let [before (export-of-load! dir-a dir-b 1000)]
+ (testing "the baseline covers many entities"
+ (is (< 200 (count before))))
+ (testing "exporting the same app DB twice gives the same YAML"
+ (is (= {} (mt/with-empty-h2-app-db!
+ (load-dir! dir-b)
+ (changed (exported-yaml!) (exported-yaml!))))))
+ (testing "loading an export into an app DB with other local ids and exporting it again gives the same YAML"
+ (is (= {} (changed before (export-of-load! dir-b dir-c 9000))))))
+ (finally
+ (run! #(FileUtils/deleteDirectory ^File %) dirs))))))
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index 4dd948181a3..f1ac4df8d8d 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -404,6 +404,13 @@
;;; | SERIALIZATION |
;;; +----------------------------------------------------------------------------------------------------------------+
+(defn- dashcard-sort-key
+ "The sort key fn for the dashcards of `dashboard` in its file: the position and the `entity_id` of their tab, then
+ row, column and `entity_id`, so the order does not depend on local ids."
+ [dashboard]
+ (let [tab-order (into {} (map (juxt :id (juxt :position :entity_id))) (:tabs dashboard))]
+ (juxt (comp tab-order :dashboard_tab_id) :row :col :entity_id)))
+
(defmethod serdes/make-spec "Dashboard" [_model-name opts]
{:copy [:archived :archived_directly :auto_apply_filters :caveats :collection_position
:description :embedding_params :enable_embedding :embedding_type :entity_id :name
@@ -420,7 +427,7 @@
:made_public_by_id (serdes/fk :model/User)
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
:tabs (serdes/nested :model/DashboardTab :dashboard_id (merge {:sort-by (juxt :position :entity_id)} opts))
- :dashcards (serdes/nested :model/DashboardCard :dashboard_id (merge {:sort-by (juxt :dashboard_tab_id :row :col)} opts))}
+ :dashcards (serdes/nested :model/DashboardCard :dashboard_id (merge {:sort-by-parent dashcard-sort-key} opts))}
:coerce {:parameters [:maybe [:sequential ::parameters.schema/parameter]]}
:defaults {:archived false
:archived_directly false
diff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljc
index fdc7b9290d0..eef1ac136e4 100644
--- a/src/metabase/lib/native.cljc
+++ b/src/metabase/lib/native.cljc
@@ -360,13 +360,6 @@
[s]
(str/replace s #"[.*+?^${}()|\[\]\\]" (fn [c] (str "\\" c))))
-(defn- replace-tag-in-text
- [text old-name new-name]
- (str/replace text
- (re-pattern (str "\\{\\{\\s*" (regex-escape old-name) "\\s*\\}\\}"))
- ;; function replacement so a `$` in the new name can't be misread as a match reference
- (constantly (str "{{" new-name "}}"))))
-
(defn- rename-tag
"Rename a template tag, replacing its `:display-name` only if it was the humanized default for the
old name (same rule as [[rename-template-tag]])."
@@ -376,10 +369,16 @@
(assoc :display-name (u.humanization/name->human-readable-name :simple new-name))))
(mu/defn rename-template-tags-in-text :- :string
- "`text` with its `{{...}}` references renamed per `renames`, a map of old tag name => new tag name."
+ "`text` with its `{{...}}` references renamed per `renames`, a map of old tag name => new tag name, all in one pass,
+ so a new name that is another tag's old name is not renamed again."
[text :- :string
renames :- [:map-of :string :string]]
- (reduce-kv replace-tag-in-text text renames))
+ (if (empty? renames)
+ text
+ (str/replace text
+ (re-pattern (str "\\{\\{\\s*(" (str/join "|" (map regex-escape (keys renames))) ")\\s*\\}\\}"))
+ (fn [[_match old-name]]
+ (str "{{" (get renames old-name) "}}")))))
(mu/defn rename-template-tags :- ::lib.schema.template-tag/template-tag-map
"`tags`, a map of tag name => tag, with the tags in `renames` renamed the way [[replace-template-tag-names]] renames
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 8694541a4da..fc59cc98c41 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -1177,13 +1177,52 @@
[:measure opts id]
[:measure (export-mbql-map opts) id]))
+(defn card-template-tag-export-rename
+ "Exported name for a card template tag whose `#<id>-slug` name embeds a local card id: the id is swapped for the
+ `entity_id` of the tag's card, the slug is kept verbatim. Nil when the name doesn't embed an id."
+ [{tag-type :type, :keys [card-id], tag-name :name}]
+ (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))
+ (when-let [[_ _embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]
+ (when-let [card-entity-id (fk-elide (*export-fk* card-id :model/Card))]
+ (str "#" card-entity-id suffix)))))
+
+(defn card-template-tag-rename
+ "New name for a card template tag whose name embeds the `entity_id` of its (already remapped) `:card-id`, a
+ different local id, or another `entity_id`: the embedded value is swapped for the id, the slug is kept verbatim. Nil
+ when they already agree or the name doesn't embed an id."
+ [{tag-type :type, :keys [card-id], tag-name :name}]
+ (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))
+ (let [entity-id-prefix (some->> (fk-elide (*export-fk* card-id :model/Card)) (str "#"))
+ entity-id-suffix (when (and entity-id-prefix (str/starts-with? tag-name entity-id-prefix))
+ (subs tag-name (count entity-id-prefix)))]
+ (if (and entity-id-suffix (or (= "" entity-id-suffix) (str/starts-with? entity-id-suffix "-")))
+ (str "#" card-id entity-id-suffix)
+ (if-let [[_ embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]
+ (when (not= (parse-long embedded-id) card-id)
+ (str "#" card-id suffix))
+ (when-let [[_ suffix] (re-matches #"#[A-Za-z0-9_-]{21}(-.*)?" tag-name)]
+ (str "#" card-id suffix)))))))
+
+(defn card-template-tag-renames
+ "The `{old-name new-name}` map of the `tags` that `rename`, a fn of one tag, gives a new name."
+ [rename tags]
+ (into {}
+ (keep (fn [{tag-name :name, :as tag}]
+ (some->> (rename tag) (vector tag-name))))
+ tags))
+
+(defn- rename-card-template-tags
+ "Renames the template tags of `query`, and the `{{#id-slug}}` refs in its native text, with `rename`."
+ [query rename]
+ (lib/replace-template-tag-names query (card-template-tag-renames rename (lib/all-template-tags query))))
+
(defn export-mbql
"Given an MBQL expression, convert it to an EDN structure and turn the non-portable Database, Table and Field IDs
inside it into portable references."
[x]
(let [x (cond-> x
(and (map? x) (= :mbql/query (:lib/type x)))
- lib/prepare-for-serialization)]
+ (-> (rename-card-template-tags card-template-tag-export-rename) lib/prepare-for-serialization))]
;; if required UUIDs are already calculated don't recalculate when we recurse.
(binding [*required-lib-uuids-for-export* (or *required-lib-uuids-for-export* (collect-required-lib-uuids x))]
(cond
@@ -1314,30 +1353,13 @@
import-mbql-update-refs
import-mbql-update-maps))
-(defn card-template-tag-rename
- "New name for a card template tag whose `#<id>-slug` name embeds a different id than its (already
- remapped) `:card-id`: the id is swapped, the slug is kept verbatim. Nil when they already agree or
- the name doesn't embed an id."
- [{tag-type :type, :keys [card-id], tag-name :name}]
- (when (and (= tag-type :card) (pos-int? card-id) (string? tag-name))
- (when-let [[_ embedded-id suffix] (re-matches #"#(\d+)(-.*)?" tag-name)]
- (when (not= (parse-long embedded-id) card-id)
- (str "#" card-id suffix)))))
-
(defn- repair-card-template-tag-names
- "Card template tag names (and the `{{#id-slug}}` refs in the native text) embed the referenced
- card's id, which goes stale when [[import-mbql-map]] remaps `:card-id` to the local card's id.
- Swap the embedded id for the remapped one, leaving the slug as it was exported."
+ "Card template tag names (and the `{{#id-slug}}` refs in the native text) embed the referenced card's `entity_id`,
+ or in older exports its source id, which [[import-mbql-map]] doesn't remap. Swap it for the local id of the remapped
+ `:card-id`, leaving the slug as it was exported."
[x]
- (if (= (:lib/type x) :mbql/query)
- (lib/replace-template-tag-names
- x
- (into {}
- (keep (fn [{tag-name :name, :as tag}]
- (when-let [new-name (card-template-tag-rename tag)]
- [tag-name new-name])))
- (lib/all-template-tags x)))
- x))
+ (cond-> x
+ (= (:lib/type x) :mbql/query) (rename-card-template-tags card-template-tag-rename)))
(defn- legacy-mbql-query->mbql5
"Converts an imported legacy MBQL query to MBQL 5, returning `x` unchanged if it isn't one or conversion fails.
@@ -1947,10 +1969,14 @@
:import #(*import-fk* % model)))))
(defn nested
- "Nested entities; `opts` may give `:sort-by`, `:key-field` and `:delete-children!`, a fn of the parent id."
+ "Nested entities. `opts` may give `:key-field`, `:delete-children!` (a fn of the parent id), and either `:sort-by`
+ or `:sort-by-parent`, a fn of the parent returning the `:sort-by` fn, which wins over `:sort-by` and is not passed
+ on to the children's own nested entities."
[model backward-fk opts]
(let [model-name (name model)
- sorter (:sort-by opts :entity_id)
+ sorter (if-let [sort-by-parent (:sort-by-parent opts)]
+ sort-by-parent
+ (constantly (:sort-by opts :entity_id)))
key-field (:key-field opts :entity_id)
delete-children! (:delete-children! opts #(models.db/delete-children! model backward-fk %))]
{::nested true
@@ -1963,8 +1989,8 @@
;; `nil? data` check is for `extract-one` case in tests; make sure to add empty vectors in
;; `extract-query` implementations for nested collections
(try
- (->> (sort-by sorter data)
- (mapv #(extract-one model-name opts %)))
+ (->> (sort-by (sorter current) data)
+ (mapv #(extract-one model-name (dissoc opts :sort-by-parent) %)))
(catch Exception e
(throw (ex-info (format "Error extracting nested %s" model)
{:model model
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index cc47812f964..3ffc16f90c1 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -192,23 +192,31 @@
serdes/import-mbql
(lib/normalize :metabase.lib.schema.template-tag/template-tag-map))]
{:tags tags
- :renames (into {}
- (keep (fn [[tag-name tag]]
- (some->> (serdes/card-template-tag-rename tag) (vector tag-name))))
- tags)}))
+ :renames (serdes/card-template-tag-renames serdes/card-template-tag-rename (vals tags))}))
+
+(defn- export-renames
+ "The renames of the card tags in `template-tags` whose `#<id>-slug` name embeds this instance's card id."
+ [template-tags]
+ (serdes/card-template-tag-renames serdes/card-template-tag-export-rename (vals template-tags)))
(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]
{:copy [:archived :description :entity_id :name]
:skip []
:transform {:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)
- :content {:export identity
+ :content {:export-with-context
+ (fn [current _ content]
+ (let [renames (export-renames (:template_tags current))]
+ (cond-> content
+ (and (string? content) (seq renames)) (lib/rename-template-tags-in-text renames))))
:import-with-context
(fn [current _ content]
(let [{:keys [renames]} (import-template-tags (:template_tags current))]
(cond-> content
(and (string? content) (seq renames)) (lib/rename-template-tags-in-text renames))))}
- :template_tags {:export serdes/export-mbql
+ :template_tags {:export (fn [template-tags]
+ (serdes/export-mbql
+ (some-> template-tags (lib/rename-template-tags (export-renames template-tags)))))
:import (fn [template-tags]
(let [{:keys [tags renames]} (import-template-tags template-tags)]
(some-> tags (lib/rename-template-tags renames))))}}
diff --git a/src/metabase/settings/models/setting.clj b/src/metabase/settings/models/setting.clj
index 3bea2cea278..81707f3c457 100644
--- a/src/metabase/settings/models/setting.clj
+++ b/src/metabase/settings/models/setting.clj
@@ -98,9 +98,9 @@
(declare export?)
(defmethod serdes/extract-all "Setting" [_model _opts]
- (for [{:keys [key value]} (admin-writable-site-wide-settings
- :getter (partial get-value-of-type :string))
- :when (export? key)]
+ (for [{:keys [key value is_env_setting]} (admin-writable-site-wide-settings
+ :getter (partial get-value-of-type :string))
+ :when (and (export? key) (not is_env_setting))]
{:serdes/meta [{:model "Setting" :id (name key)}]
:key key
:value value}))
diff --git a/src/metabase/warehouse_schema/models/table.clj b/src/metabase/warehouse_schema/models/table.clj
index 322d6b62397..5ded5a410fc 100644
--- a/src/metabase/warehouse_schema/models/table.clj
+++ b/src/metabase/warehouse_schema/models/table.clj
@@ -614,15 +614,19 @@
(defmethod serdes/make-spec "Table" [_model-name _opts]
{:copy [:name :description :entity_type :active :display_name :visibility_type :schema
- :points_of_interest :caveats :show_in_getting_started :field_order :initial_sync_status :is_upload
+ :points_of_interest :caveats :show_in_getting_started :field_order :is_upload
:database_require_filter :is_defective_duplicate :unique_table_helper :is_writable :data_authority
- :data_source :owner_email :owner_user_id :is_published]
- :skip [:estimated_row_count :view_count :transform_target]
+ :data_source :owner_email :is_published]
+ :skip [:estimated_row_count :initial_sync_status :view_count :transform_target]
:transform {:archived_at (serdes/date)
:deactivated_at (serdes/date)
:data_layer (serdes/optional-kw)
:db_id (serdes/fk :model/Database)
:collection_id (serdes/fk :model/Collection)
+ ;; older exports copied the local user id; re-importing those files on the same instance, e.g. a
+ ;; remote sync pull before the first new export, keeps the right owner, where nil would drop it
+ :owner_user_id (update (serdes/fk :model/User) :import
+ (fn [import-user] #(if (int? %) % (import-user %))))
:transform_id (serdes/fk :model/Transform)}
:defaults {:is_defective_duplicate false
:is_published false
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 770ceb9b128..10a9ea0f23e 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -666,22 +666,31 @@
(defmethod serdes/make-spec "Database"
[_model-name _opts]
;; Connection details are never exported. Import is unconditional.
- (let [details-transform {:export-with-context (fn [_current _ _details] ::serdes/skip)
- :import identity}]
- {:copy [:auto_run_queries :cache_field_values_schedule :caveats :dbms_version
+ (let [details-transform {:export-with-context (fn [_current _ _details] ::serdes/skip)
+ :import identity}
+ schedule-transform {:export-with-context (fn [_current _ _schedule] ::serdes/skip)
+ :import identity}]
+ {:copy [:auto_run_queries :caveats
:description :engine :is_audit :is_attached_dwh :is_full_sync :is_on_demand
- :default_schema :metadata_sync_schedule :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled
+ :default_schema :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled
:uploads_schema_name :uploads_table_prefix]
:skip [;; deprecated field
:cache_ttl
;; describes a sync on the source instance, and may name its connection details
- :initial_sync_error]
- :transform {:details details-transform
+ :initial_sync_error
+ ;; detected by sync, and set again by the importing instance's sync
+ :dbms_version]
+ :transform {;; randomized per instance unless the connection details, which we never export, let the user
+ ;; schedule syncs; older files with such details carry the schedules those details require
+ :cache_field_values_schedule schedule-transform
+ :metadata_sync_schedule schedule-transform
+ :details details-transform
:write_data_details details-transform
:admin_details details-transform
:creator_id (serdes/fk :model/User)
:router_database_id (serdes/fk :model/Database)
- :initial_sync_status {:export identity :import (constantly "complete")}
+ :initial_sync_status {:export-with-context (fn [_current _ _status] ::serdes/skip)
+ :import (constantly "complete")}
;; Never trust an archive's `is_sample`: the real Sample Database is excluded from export
;; ([[warehouses.db/databases-for-serdes-reducible]]), so any incoming `is_sample true` is forged --
;; and marking an imported row as the sample DB would let it ride sample-only code paths. Import false.
diff --git a/test/metabase/lib/native_test.cljc b/test/metabase/lib/native_test.cljc
index 19bcbd81bf2..bba2b4533e5 100644
--- a/test/metabase/lib/native_test.cljc
+++ b/test/metabase/lib/native_test.cljc
@@ -902,6 +902,15 @@
:id "6ebf6c2e-d6e2-449e-97b7-7005047928e5"}}
renames))))))
+(deftest ^:parallel rename-template-tags-in-text-one-pass-test
+ (testing "a new name that is another tag's old name is not renamed again"
+ (is (= "{{#2}} {{#1-orders}} {{#2}}"
+ (lib.native/rename-template-tags-in-text "{{ #1 }} {{#2-orders}} {{#1}}" {"#1" "#2", "#2-orders" "#1-orders"})))
+ (is (= "{{#1}} {{#2}}"
+ (lib.native/rename-template-tags-in-text "{{#2}} {{#1}}" {"#1" "#2", "#2" "#1"})))
+ (is (= "{{#7}} {{#9}} {{#70}}"
+ (lib.native/rename-template-tags-in-text "{{#ent}} {{#7}} {{#70}}" {"#ent" "#7", "#7" "#9"})))))
+
(deftest ^:parallel replace-template-tag-names-noop-test
(let [query (card-tag-query "select * from {{#133-some-old-slug}}"
{:type :card
diff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
index 195d2497b8b..eeefb37e851 100644
--- a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
@@ -10,7 +10,7 @@ dataset_query:
stages:
- native: |-
SELECT *
- FROM {{#1-basic_aggregations}}
+ FROM {{#h5F2EjHsRd73Dqqh8sAtd-basic_aggregations}}
WHERE {{snippet: Active Product Filter}}
ORDER BY "count" DESC
LIMIT 10
@@ -18,7 +18,7 @@ dataset_query:
- card-id: h5F2EjHsRd73Dqqh8sAtd
display-name: Basic Aggregations
id: cc000001-0000-0000-0000-000000000001
- name: '#1-basic_aggregations'
+ name: '#h5F2EjHsRd73Dqqh8sAtd-basic_aggregations'
type: card
- display-name: 'Snippet: Active Product Filter'
id: cc000002-0000-0000-0000-000000000002
diff --git a/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml b/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml
index e19ec573990..c616722c25a 100644
--- a/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml
+++ b/test_resources/serialization_baseline/collections/snippets/active_recent_filter.yaml
@@ -2,7 +2,7 @@ name: Active Recent Filter
description: Combines Active Product Filter with Date Range Filter
entity_id: c5PMqT66HwWoy0m52WTI5
content: '{{snippet: Active Product Filter}} AND {{snippet: Date Range Filter}} AND {{snippet: Missing Filter}}
- AND category = {{category}} AND id IN (SELECT id FROM {{#1-basic-aggregations}})'
+ AND category = {{category}} AND id IN (SELECT id FROM {{#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations}})'
creator_id: [email protected]
template_tags:
'snippet: Active Product Filter':
@@ -29,11 +29,11 @@ template_tags:
id: dd000004-0000-0000-0000-000000000004
name: category
type: text
- '#1-basic-aggregations':
+ '#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations':
card-id: h5F2EjHsRd73Dqqh8sAtd
- display-name: '#1 Basic Aggregations'
+ display-name: '#h5F2EjHsRd73Dqqh8sAtd Basic Aggregations'
id: dd000005-0000-0000-0000-000000000005
- name: '#1-basic-aggregations'
+ name: '#h5F2EjHsRd73Dqqh8sAtd-basic-aggregations'
type: card
serdes/meta:
- model: NativeQuerySnippet