The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3754 Order serdes export by created_at for stable filename de-dup (#74992)
Eric Normand16593d046c96
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.cljindex 85a73b8144d..3203646cb87 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj@@ -320,6 +320,57 @@(file-set (io/file dump-dir "collections")))"collections form a tree, with same-named files")))))))+(defn- export-and-read-eids!+ "Export everything to `dump-dir`, then return the entity_id read from each of `filenames` under+ collections/main/my_collection/."+ [dump-dir filenames]+ (storage/store! (into [] (extract/extract {:no-settings true+ :no-data-model true+ :no-transforms true}))+ (storage.files/file-writer dump-dir))+ (mapv (fn [filename]+ (:entity_id (yaml/from-file (io/file dump-dir "collections" "main" "my_collection" filename))))+ filenames))++(deftest same-name-stable-filename-test+ (testing "two same-named entities in a folder get deterministic filename de-dup suffixes (`foo.yaml` vs+ `foo_2.yaml`) across exports — otherwise re-exports swap which card lands in which file, producing huge+ phantom git-sync diffs (GHY-3754)"+ (testing "the older entity (earlier created_at) keeps the unsuffixed file, so adding a new same-named entity+ appends `_2` rather than displacing the existing file. created_at decides even when entity_id order and+ insertion order both disagree"+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (mt/with-empty-h2-app-db!+ ;; The OLD card has the *larger* entity_id and is inserted *second*, so created_at is the only thing that+ ;; can put it in the unsuffixed file — proving created_at takes precedence over entity_id and row order.+ (ts/with-temp-dpc [:model/Collection coll {:name "My Collection"}+ :model/Card _new {:name "Dup Name"+ :collection_id (:id coll)+ :created_at #t "2024-01-01T00:00:00Z"+ :entity_id "aaaaaaaaaaaaaaaaaaaaa"}+ :model/Card _old {:name "Dup Name"+ :collection_id (:id coll)+ :created_at #t "2020-01-01T00:00:00Z"+ :entity_id "zzzzzzzzzzzzzzzzzzzzz"}]+ (is (= ["zzzzzzzzzzzzzzzzzzzzz" "aaaaaaaaaaaaaaaaaaaaa"]+ (export-and-read-eids! dump-dir ["dup_name.yaml" "dup_name_2.yaml"]))+ "older card (earlier created_at) wins the unsuffixed file; newer card gets _2")))))+ (testing "entity_id breaks ties deterministically when created_at is identical"+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc [:model/Collection coll {:name "My Collection"}+ :model/Card _zzz {:name "Dup Name"+ :collection_id (:id coll)+ :created_at #t "2024-01-01T00:00:00Z"+ :entity_id "zzzzzzzzzzzzzzzzzzzzz"}+ :model/Card _aaa {:name "Dup Name"+ :collection_id (:id coll)+ :created_at #t "2024-01-01T00:00:00Z"+ :entity_id "aaaaaaaaaaaaaaaaaaaaa"}]+ (is (= ["aaaaaaaaaaaaaaaaaaaaa" "zzzzzzzzzzzzzzzzzzzzz"]+ (export-and-read-eids! dump-dir ["dup_name.yaml" "dup_name_2.yaml"]))+ "with equal created_at, smaller entity_id wins the unsuffixed file")))))))+(deftest ^:parallel resolve-path-test(let [resolve-path @#'storage.util/resolve-path](testing "basic slugification"diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.cljindex 67817039df3..4cecec27a3f 100644--- a/src/metabase/collections/models/collection.clj+++ b/src/metabase/collections/models/collection.clj@@ -2078,14 +2078,18 @@[:in :id collection-set](when (some nil? collection-set) [:= :id nil])]not-trash-clause- (or where true)]})+ (or where true)]+ ;; stable filename de-dup suffixes across exports, see GHY-3754+ :order-by serdes/stable-storage-order})(t2/reducible-select :model/Collection{:where[:and(when skip-archived [:not :archived])[:= :personal_owner_id nil]not-trash-clause- (or where true)]}))))+ (or where true)]+ ;; stable filename de-dup suffixes across exports, see GHY-3754+ :order-by serdes/stable-storage-order}))))(defmethod serdes/dependencies "Collection"[{:keys [parent_id]}]diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex b98457bcbc7..f2ace67a3ee 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -558,7 +558,7 @@(defn- transform->nested [transform opts batch](let [backward-fk (:backward-fk transform)entities (-> (extract-query (name (:model transform))- (assoc opts :where [:in backward-fk (map :id batch)]))+ (assoc opts :where [:in backward-fk (map :id batch)] ::nested-fetch true))t2.realize/realize)](group-by backward-fk entities)))@@ -579,22 +579,47 @@cat)reducible))+(def stable-storage-order+ "A deterministic `:order-by` for serdes export queries, so that entities sharing a name within a folder get+ reproducible filename de-dup suffixes (`foo.yaml` vs `foo_2.yaml`) across exports — otherwise re-exports swap which+ entity lands in which file and produce large phantom git-sync diffs (GHY-3754).++ Orders by `created_at` first so the *oldest* sibling keeps the unsuffixed file: adding a new same-named entity+ appends a `_2` rather than displacing the existing file. `entity_id` (random NanoID) would not have this property —+ a newcomer with a smaller id would steal the base file. `entity_id` is then a deterministic tiebreaker for entities+ created in the same instant. Assumes the model serializes both columns; use [[stable-storage-order-by]] to gate on a+ spec."+ [[:created_at :asc] [:entity_id :asc]])++(defn stable-storage-order-by+ "[[stable-storage-order]] restricted to the columns `spec` actually serializes (via `:copy` or `:transform`).+ Returns nil when the model serializes neither column."+ [spec]+ (let [serialized? (into (set (:copy spec)) (keys (:transform spec)))]+ (not-empty (filterv (comp serialized? first) stable-storage-order))))+(defn extract-query-collections"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personalcollection."[model {:keys [collection-set where] :as opts}]- (let [spec (*make-spec* (name model) opts)]+ (let [spec (*make-spec* (name model) opts)+ ;; Nested fetches (e.g. a Dashboard's DashboardCards) are embedded as lists inside the parent's file+ ;; rather than written to their own files, so they keep their natural order and are left untouched.+ order-by (when-not (::nested-fetch opts)+ (stable-storage-order-by spec))](if (or (empty? collection-set)(nil? (-> spec :transform :collection_id)));; either no collections specified or our model has no collection- (t2/reducible-select model {:where (or where true)})- (t2/reducible-select model {:where [:and- [:or- [:in :collection_id collection-set]- (when (some nil? collection-set)- [:= :collection_id nil])]- (when where- where)]}))))+ (t2/reducible-select model (cond-> {:where (or where true)}+ order-by (assoc :order-by order-by)))+ (t2/reducible-select model (cond-> {:where [:and+ [:or+ [:in :collection_id collection-set]+ (when (some nil? collection-set)+ [:= :collection_id nil])]+ (when where+ where)]}+ order-by (assoc :order-by order-by))))))(defmethod extract-query :default [model-name opts](let [spec (*make-spec* model-name opts)diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex 22cf75d276c..6b7a6c6a4c8 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -168,7 +168,9 @@(when-let [collection-ids (not-empty (remove nil? collection-set))][:in :collection_id collection-ids])(when (some nil? collection-set)- [:= :collection_id nil])]]}+ [:= :collection_id nil])]]+ ;; stable filename de-dup suffixes across exports, see GHY-3754+ :order-by serdes/stable-storage-order}where (sql.helpers/where :or where))))(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yamlindex 5f91fbd0e11..2a12e6be1f0 100644--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml@@ -18,9 +18,9 @@ definition:- PRODUCTSlib/type: mbql.stage/mbqllib/type: mbql/query-entity_id: xK7mPqR2sT4uVwXyZ9a1b+entity_id: mSrAvgProdPrice00008xname: Average Product Priceserdes/meta:-- id: xK7mPqR2sT4uVwXyZ9a1b+- id: mSrAvgProdPrice00008xlabel: average_product_pricemodel: Measurediff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yamlindex 2a12e6be1f0..5f91fbd0e11 100644--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml@@ -18,9 +18,9 @@ definition:- PRODUCTSlib/type: mbql.stage/mbqllib/type: mbql/query-entity_id: mSrAvgProdPrice00008x+entity_id: xK7mPqR2sT4uVwXyZ9a1bname: Average Product Priceserdes/meta:-- id: mSrAvgProdPrice00008x+- id: xK7mPqR2sT4uVwXyZ9a1blabel: average_product_pricemodel: Measure