Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3754 Order serdes export by created_at for stable filename de-dup (#74992)

Eric Normand16593d046c96
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 85a73b8144d..3203646cb87 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -320,6 +320,57 @@
(file-set (io/file dump-dir "collections")))
"collections form a tree, with same-named files")))))))
+(defn- export-and-read-eids!
+ "Export everything to `dump-dir`, then return the entity_id read from each of `filenames` under
+ collections/main/my_collection/."
+ [dump-dir filenames]
+ (storage/store! (into [] (extract/extract {:no-settings true
+ :no-data-model true
+ :no-transforms true}))
+ (storage.files/file-writer dump-dir))
+ (mapv (fn [filename]
+ (:entity_id (yaml/from-file (io/file dump-dir "collections" "main" "my_collection" filename))))
+ filenames))
+
+(deftest same-name-stable-filename-test
+ (testing "two same-named entities in a folder get deterministic filename de-dup suffixes (`foo.yaml` vs
+ `foo_2.yaml`) across exports — otherwise re-exports swap which card lands in which file, producing huge
+ phantom git-sync diffs (GHY-3754)"
+ (testing "the older entity (earlier created_at) keeps the unsuffixed file, so adding a new same-named entity
+ appends `_2` rather than displacing the existing file. created_at decides even when entity_id order and
+ insertion order both disagree"
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (mt/with-empty-h2-app-db!
+ ;; The OLD card has the *larger* entity_id and is inserted *second*, so created_at is the only thing that
+ ;; can put it in the unsuffixed file — proving created_at takes precedence over entity_id and row order.
+ (ts/with-temp-dpc [:model/Collection coll {:name "My Collection"}
+ :model/Card _new {:name "Dup Name"
+ :collection_id (:id coll)
+ :created_at #t "2024-01-01T00:00:00Z"
+ :entity_id "aaaaaaaaaaaaaaaaaaaaa"}
+ :model/Card _old {:name "Dup Name"
+ :collection_id (:id coll)
+ :created_at #t "2020-01-01T00:00:00Z"
+ :entity_id "zzzzzzzzzzzzzzzzzzzzz"}]
+ (is (= ["zzzzzzzzzzzzzzzzzzzzz" "aaaaaaaaaaaaaaaaaaaaa"]
+ (export-and-read-eids! dump-dir ["dup_name.yaml" "dup_name_2.yaml"]))
+ "older card (earlier created_at) wins the unsuffixed file; newer card gets _2")))))
+ (testing "entity_id breaks ties deterministically when created_at is identical"
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc [:model/Collection coll {:name "My Collection"}
+ :model/Card _zzz {:name "Dup Name"
+ :collection_id (:id coll)
+ :created_at #t "2024-01-01T00:00:00Z"
+ :entity_id "zzzzzzzzzzzzzzzzzzzzz"}
+ :model/Card _aaa {:name "Dup Name"
+ :collection_id (:id coll)
+ :created_at #t "2024-01-01T00:00:00Z"
+ :entity_id "aaaaaaaaaaaaaaaaaaaaa"}]
+ (is (= ["aaaaaaaaaaaaaaaaaaaaa" "zzzzzzzzzzzzzzzzzzzzz"]
+ (export-and-read-eids! dump-dir ["dup_name.yaml" "dup_name_2.yaml"]))
+ "with equal created_at, smaller entity_id wins the unsuffixed file")))))))
+
(deftest ^:parallel resolve-path-test
(let [resolve-path @#'storage.util/resolve-path]
(testing "basic slugification"
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index 67817039df3..4cecec27a3f 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -2078,14 +2078,18 @@
[:in :id collection-set]
(when (some nil? collection-set) [:= :id nil])]
not-trash-clause
- (or where true)]})
+ (or where true)]
+ ;; stable filename de-dup suffixes across exports, see GHY-3754
+ :order-by serdes/stable-storage-order})
(t2/reducible-select :model/Collection
{:where
[:and
(when skip-archived [:not :archived])
[:= :personal_owner_id nil]
not-trash-clause
- (or where true)]}))))
+ (or where true)]
+ ;; stable filename de-dup suffixes across exports, see GHY-3754
+ :order-by serdes/stable-storage-order}))))
(defmethod serdes/dependencies "Collection"
[{:keys [parent_id]}]
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index b98457bcbc7..f2ace67a3ee 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -558,7 +558,7 @@
(defn- transform->nested [transform opts batch]
(let [backward-fk (:backward-fk transform)
entities (-> (extract-query (name (:model transform))
- (assoc opts :where [:in backward-fk (map :id batch)]))
+ (assoc opts :where [:in backward-fk (map :id batch)] ::nested-fetch true))
t2.realize/realize)]
(group-by backward-fk entities)))
@@ -579,22 +579,47 @@
cat)
reducible))
+(def stable-storage-order
+ "A deterministic `:order-by` for serdes export queries, so that entities sharing a name within a folder get
+ reproducible filename de-dup suffixes (`foo.yaml` vs `foo_2.yaml`) across exports — otherwise re-exports swap which
+ entity lands in which file and produce large phantom git-sync diffs (GHY-3754).
+
+ Orders by `created_at` first so the *oldest* sibling keeps the unsuffixed file: adding a new same-named entity
+ appends a `_2` rather than displacing the existing file. `entity_id` (random NanoID) would not have this property —
+ a newcomer with a smaller id would steal the base file. `entity_id` is then a deterministic tiebreaker for entities
+ created in the same instant. Assumes the model serializes both columns; use [[stable-storage-order-by]] to gate on a
+ spec."
+ [[:created_at :asc] [:entity_id :asc]])
+
+(defn stable-storage-order-by
+ "[[stable-storage-order]] restricted to the columns `spec` actually serializes (via `:copy` or `:transform`).
+ Returns nil when the model serializes neither column."
+ [spec]
+ (let [serialized? (into (set (:copy spec)) (keys (:transform spec)))]
+ (not-empty (filterv (comp serialized? first) stable-storage-order))))
+
(defn extract-query-collections
"Helper for the common (but not default) [[extract-query]] case of fetching everything that isn't in a personal
collection."
[model {:keys [collection-set where] :as opts}]
- (let [spec (*make-spec* (name model) opts)]
+ (let [spec (*make-spec* (name model) opts)
+ ;; Nested fetches (e.g. a Dashboard's DashboardCards) are embedded as lists inside the parent's file
+ ;; rather than written to their own files, so they keep their natural order and are left untouched.
+ order-by (when-not (::nested-fetch opts)
+ (stable-storage-order-by spec))]
(if (or (empty? collection-set)
(nil? (-> spec :transform :collection_id)))
;; either no collections specified or our model has no collection
- (t2/reducible-select model {:where (or where true)})
- (t2/reducible-select model {:where [:and
- [:or
- [:in :collection_id collection-set]
- (when (some nil? collection-set)
- [:= :collection_id nil])]
- (when where
- where)]}))))
+ (t2/reducible-select model (cond-> {:where (or where true)}
+ order-by (assoc :order-by order-by)))
+ (t2/reducible-select model (cond-> {:where [:and
+ [:or
+ [:in :collection_id collection-set]
+ (when (some nil? collection-set)
+ [:= :collection_id nil])]
+ (when where
+ where)]}
+ order-by (assoc :order-by order-by))))))
(defmethod extract-query :default [model-name opts]
(let [spec (*make-spec* model-name opts)
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index 22cf75d276c..6b7a6c6a4c8 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -168,7 +168,9 @@
(when-let [collection-ids (not-empty (remove nil? collection-set))]
[:in :collection_id collection-ids])
(when (some nil? collection-set)
- [:= :collection_id nil])]]}
+ [:= :collection_id nil])]]
+ ;; stable filename de-dup suffixes across exports, see GHY-3754
+ :order-by serdes/stable-storage-order}
where (sql.helpers/where :or where))))
(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml
index 5f91fbd0e11..2a12e6be1f0 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price.yaml
@@ -18,9 +18,9 @@ definition:
- PRODUCTS
lib/type: mbql.stage/mbql
lib/type: mbql/query
-entity_id: xK7mPqR2sT4uVwXyZ9a1b
+entity_id: mSrAvgProdPrice00008x
name: Average Product Price
serdes/meta:
-- id: xK7mPqR2sT4uVwXyZ9a1b
+- id: mSrAvgProdPrice00008x
label: average_product_price
model: Measure
diff --git a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml
index 2a12e6be1f0..5f91fbd0e11 100644
--- a/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml
+++ b/test_resources/serialization_baseline/databases/sample_database/schemas/public/tables/products/measures/average_product_price_2.yaml
@@ -18,9 +18,9 @@ definition:
- PRODUCTS
lib/type: mbql.stage/mbql
lib/type: mbql/query
-entity_id: mSrAvgProdPrice00008x
+entity_id: xK7mPqR2sT4uVwXyZ9a1b
name: Average Product Price
serdes/meta:
-- id: mSrAvgProdPrice00008x
+- id: xK7mPqR2sT4uVwXyZ9a1b
label: average_product_price
model: Measure