Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Ghy 3370/remote sync phantom diffs (#72495)

Ben Grabowee847893ffe5
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
index 71d5a2aa3b1..ccd8d582da7 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/round_trip_test.clj
@@ -176,7 +176,7 @@
out-file (io/file output-dir file)]
:when (.exists out-file)
:let [delta (compare-files ref-file out-file)]]
- (is (nil? delta)
+ (is (= nil delta)
(str "Content mismatch for file: " (strip-base-path source-dir file)))
;; Leave behind files for developers to inspect
(when (and (.exists dev-inspect-dir) delta)
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
index 7dd231c85ba..a23f71119cf 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/storage_test.clj
@@ -246,6 +246,64 @@
(is (= #{["common.py.yaml"]}
(file-set (io/file dump-dir "python_libraries"))))))))))
+(deftest dashcard-ordering-test
+ (testing "Test that dashcard ordering in YAML export is stable when the DB returns dashcards
+ in a different order (simulating non-deterministic query results from Aurora/postgres)."
+ (mt/with-empty-h2-app-db!
+ (ts/with-random-dump-dir [dump-dir "serdesv2-"]
+ (let [now (t/offset-date-time)
+ col (ts/create! :model/Collection :name "coll")
+ db (ts/create! :model/Database :name "mydb")
+ tbl (ts/create! :model/Table :name "mytable" :db_id (:id db))
+ fld (ts/create! :model/Field :name "myfield" :table_id (:id tbl))
+ c1 (ts/create! :model/Card :name "Card A" :collection_id (:id col)
+ :database_id (:id db) :table_id (:id tbl))
+ c2 (ts/create! :model/Card :name "Card B" :collection_id (:id col)
+ :database_id (:id db) :table_id (:id tbl))
+ c3 (ts/create! :model/Card :name "Card C" :collection_id (:id col)
+ :database_id (:id db) :table_id (:id tbl))
+ dash (ts/create! :model/Dashboard :name "My Dashboard" :collection_id (:id col))
+ ;; Create all dashcards with the same created_at (simulating a batch save)
+ _dc1 (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id c1)
+ :row 0 :col 0 :size_x 4 :size_y 4 :created_at now
+ :parameter_mappings [{:parameter_id "param1"
+ :card_id (:id c1)
+ :target [:dimension [:field (:id fld) nil]]}])
+ _dc2 (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id c2)
+ :row 0 :col 4 :size_x 4 :size_y 4 :created_at now)
+ _dc3 (ts/create! :model/DashboardCard :dashboard_id (:id dash) :card_id (:id c3)
+ :row 4 :col 0 :size_x 4 :size_y 4 :created_at now)
+
+ find-dash-file (fn [dir]
+ (->> (file-seq (io/file dir))
+ (filter #(str/ends-with? (.getName ^java.io.File %) ".yaml"))
+ (filter #(str/includes? (.getName ^java.io.File %) "my_dashboard"))
+ first))
+ clean-and-export! (fn []
+ (doseq [^java.io.File f (reverse (file-seq (io/file dump-dir)))]
+ (.delete f))
+ (.mkdirs (io/file dump-dir))
+ (storage/store! (serdes/with-cache (into [] (extract/extract {:no-settings true
+ :no-transforms true})))
+ (storage.files/file-writer dump-dir))
+ (slurp (find-dash-file dump-dir)))
+
+ ;; Export with normal DB order
+ yaml-before (clean-and-export!)
+
+ ;; Export again but with nested entity query results reversed,
+ ;; simulating non-deterministic DB row ordering (as seen on Aurora Postgres)
+ original-fn @#'serdes/transform->nested
+ yaml-reversed
+ (with-redefs [serdes/transform->nested
+ (fn [transform opts batch]
+ (update-vals (original-fn transform opts batch) reverse))]
+ (clean-and-export!))]
+
+ (testing "Dashcard ordering should be stable regardless of DB return order"
+ (is (= yaml-before yaml-reversed)
+ "Dashboard YAML should be identical even when DB returns dashcards in different order")))))))
+
(deftest name-too-long-test
(ts/with-random-dump-dir [dump-dir "serdesv2-"]
(mt/with-empty-h2-app-db!
diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.clj
index 4bd7a0dc436..0786e5bfc89 100644
--- a/src/metabase/actions/models.clj
+++ b/src/metabase/actions/models.clj
@@ -447,15 +447,15 @@
:creator_id (serdes/fk :model/User)
:made_public_by_id (serdes/fk :model/User)
:model_id (serdes/fk :model/Card)
- :query (serdes/nested :model/QueryAction :action_id opts)
- :http (serdes/nested :model/HTTPAction :action_id opts)
- :implicit (serdes/nested :model/ImplicitAction :action_id opts)
+ :query (serdes/nested :model/QueryAction :action_id (merge {:sort-by (juxt :name :created_at)} opts))
+ :http (serdes/nested :model/HTTPAction :action_id (merge {:sort-by (juxt :name :created_at)} opts))
+ :implicit (serdes/nested :model/ImplicitAction :action_id (merge {:sort-by (juxt :name :created_at)} opts))
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
:parameter_mappings {:export serdes/export-parameter-mappings
:import serdes/import-parameter-mappings}
:visualization_settings {:export serdes/export-visualization-settings
:import serdes/import-visualization-settings}}
- :defaults {:archived false}})
+ :defaults {:archived false}})
(defmethod serdes/dependencies "Action" [action]
(set
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index 3d385d706ec..0d5d7cf664f 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -409,14 +409,14 @@
:creator_id (serdes/fk :model/User)
:made_public_by_id (serdes/fk :model/User)
:parameters {:export serdes/export-parameters :import serdes/import-parameters}
- :tabs (serdes/nested :model/DashboardTab :dashboard_id opts)
- :dashcards (serdes/nested :model/DashboardCard :dashboard_id opts)}
- :coerce {:parameters [:maybe [:sequential ::parameters.schema/parameter]]}
- :defaults {:archived false
- :archived_directly false
- :auto_apply_filters true
- :enable_embedding false
- :show_in_getting_started false}})
+ :tabs (serdes/nested :model/DashboardTab :dashboard_id (merge {:sort-by (juxt :position :created_at)} opts))
+ :dashcards (serdes/nested :model/DashboardCard :dashboard_id (merge {:sort-by (juxt :dashboard_tab_id :row :col)} opts))}
+ :coerce {:parameters [:maybe [:sequential ::parameters.schema/parameter]]}
+ :defaults {:archived false
+ :archived_directly false
+ :auto_apply_filters true
+ :enable_embedding false
+ :show_in_getting_started false}})
(defn- serdes-deps-dashcard
[{:keys [action_id card_id parameter_mappings visualization_settings series]}]
diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.clj
index f6c8bfbf6a8..97165dbc945 100644
--- a/src/metabase/metabot/models/metabot.clj
+++ b/src/metabase/metabot/models/metabot.clj
@@ -49,5 +49,5 @@
:transform {:created_at (serdes/date)
:updated_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
- :prompts (serdes/nested :model/MetabotPrompt :metabot_id opts)}
- :defaults {:use_verified_content false}})
+ :prompts (serdes/nested :model/MetabotPrompt :metabot_id (merge {:sort-by (juxt :prompt :created_at)} opts))}
+ :defaults {:use_verified_content false}})
diff --git a/src/metabase/timeline/models/timeline.clj b/src/metabase/timeline/models/timeline.clj
index e63957cb5a7..e3a79a99cba 100644
--- a/src/metabase/timeline/models/timeline.clj
+++ b/src/metabase/timeline/models/timeline.clj
@@ -61,5 +61,5 @@
:transform {:created_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)
- :events (serdes/nested :model/TimelineEvent :timeline_id opts)}
- :defaults {:archived false :default false}})
+ :events (serdes/nested :model/TimelineEvent :timeline_id (merge {:sort-by (juxt :name :created_at)} opts))}
+ :defaults {:archived false :default false}})
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 60f6501d2c4..acf42ebf317 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -389,7 +389,7 @@
(m/update-existing :database_id import-maybe-int-database-fk)))))))))}
:target {:export #(serdes/export-mbql (dissoc % :table_id))
:import serdes/import-mbql}
- :tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})
+ :tags (serdes/nested :model/TransformTransformTag :transform_id (merge {:sort-by (juxt :position :created_at)} opts))}})
(defmethod serdes/dependencies "Transform"
[{:keys [collection_id source tags source_database_id]}]
diff --git a/src/metabase/transforms/models/transform_job.clj b/src/metabase/transforms/models/transform_job.clj
index 94d253be944..6dcbdd8d510 100644
--- a/src/metabase/transforms/models/transform_job.clj
+++ b/src/metabase/transforms/models/transform_job.clj
@@ -189,7 +189,7 @@
:transform {:name {:export str :import identity}
:description {:export str :import identity}
:created_at (serdes/date)
- :job_tags (serdes/nested :model/TransformJobTransformTag :job_id opts)}})
+ :job_tags (serdes/nested :model/TransformJobTransformTag :job_id (merge {:sort-by (juxt :position :created_at)} opts))}})
(defmethod serdes/dependencies "TransformJob"
[{:keys [job_tags]}]
diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.clj
index 553ddabcb50..99e89050417 100644
--- a/src/metabase/warehouse_schema/models/field.clj
+++ b/src/metabase/warehouse_schema/models/field.clj
@@ -465,13 +465,13 @@
:table_id (serdes/fk :model/Table)
:fk_target_field_id (serdes/fk :model/Field)
:parent_id (serdes/fk :model/Field)
- :dimensions (serdes/nested :model/Dimension :field_id opts)}
- :defaults {:active true
- :database_is_auto_increment false
- :database_required false
- :is_defective_duplicate false
- :json_unfolding false
- :preview_display true}})
+ :dimensions (serdes/nested :model/Dimension :field_id (merge {:sort-by (juxt :name :created_at)} opts))}
+ :defaults {:active true
+ :database_is_auto_increment false
+ :database_required false
+ :is_defective_duplicate false
+ :json_unfolding false
+ :preview_display true}})
(defmethod serdes/storage-path "Field" [field _]
(let [[path fields] (split-with #(not= "Field" (:model %)) (serdes/path field))