The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GHY-4600] Improve memory allocation of remote sync (#82625)
Tyler Plude56232fc3f8d8
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 0da994890c1..c8ab89eed2e 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -522,7 +522,11 @@- Local changes stay dirty- Sets version to remote tip"[snapshot base-snapshot task-id sync-timestamp]- (let [{:keys [conflicts merged summary]} (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id)]+ (let [{:keys [conflicts merged summary]} (serdes/with-cache+ (let [targets (spec/exportable-entities)]+ (source/compute-merge (spec/extract-entities-for-export targets)+ snapshot base-snapshot task-id+ :total (spec/exportable-entity-count targets))))](if (seq conflicts)(let [labels (mapv remote-sync.merge/conflict-label conflicts)](log/infof "Pull merge conflict on %d entit(ies)" (count labels))@@ -738,9 +742,9 @@changes.Returns a `:success` result with a `:merge-summary`."- [source snapshot base-snapshot task-id message sync-timestamp models]+ [source snapshot base-snapshot task-id message sync-timestamp models & {:keys [total]}](let [pushed-count (count (remote-sync.object/dirty-rows))- {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id)]+ {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id :total total)](if (seq conflicts)(let [labels (mapv remote-sync.merge/conflict-label conflicts)](log/infof "Export merge conflict on %d entit(ies)" (count labels))@@ -1208,8 +1212,10 @@:message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}:else- (export-merged! src snapshot base-snapshot task-id message sync-timestamp- (spec/extract-entities-for-export)))+ (let [targets (spec/exportable-entities)]+ (export-merged! src snapshot base-snapshot task-id message sync-timestamp+ (spec/extract-entities-for-export targets)+ :total (spec/exportable-entity-count targets))))diverged? ;; and not merge? option{:status :conflict@@ -1567,17 +1573,21 @@no-changes(if-let [base-snapshot (source.p/snapshot-at source base-version)](serdes/with-cache- (if-let [models (seq (spec/extract-entities-for-export))]- (assoc (source/preview-merge models snapshot base-snapshot nil) :diverged? true)- (assoc no-changes :diverged? true)))+ (let [targets (spec/exportable-entities)]+ (if (seq targets)+ (assoc (source/preview-merge (spec/extract-entities-for-export targets) snapshot base-snapshot nil)+ :diverged? true)+ (assoc no-changes :diverged? true))));; No merge base — the remote history was rewritten. A merge is impossible, but a force push is;; still offered, so surface what it would discard (every remote entity not identical to ours).{:diverged? true :clean? false :reason :history-rewritten:conflicts [] :summary {:added 0 :updated 0 :removed 0}:force-push-casualties (serdes/with-cache- (if-let [models (seq (spec/extract-entities-for-export))]- (source/force-push-casualties-no-base models snapshot)- {:deleted [] :overwritten []}))}))))+ (let [targets (spec/exportable-entities)]+ (if (seq targets)+ (source/force-push-casualties-no-base+ (spec/extract-entities-for-export targets) snapshot)+ {:deleted [] :overwritten []})))}))))(defn create-branch!"Creates a new remote branch from `base-branch` and switches `remote-sync-branch`diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.cljindex c3293c72427..b2c877b0132 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj@@ -98,22 +98,10 @@(cond-> (or (:name entity) descriptor)(and path (not= descriptor path)) (str " (" path ")"))))-(defn three-way-merge- "Three-way merge of serialized content keyed on serdes identity.-- `base`, `ours`, `theirs` are each sequences of `{:path :content}` file specs (base/theirs typically read- from the corresponding git trees, ours freshly serialized from the app DB).-- Returns a map:- - `:merged` - sequence of winning `{:path :content}` specs to write- - `:conflicts` - sequence of `{:key :ours :theirs :base}` for entities changed differently on both sides- - `:summary` - `{:added :updated :removed}` counts of remote-originated changes folded into the result- (i.e. changes coming from `theirs` that `ours` did not already have)"- [base ours theirs]- (let [b (index-by-key base)- o (index-by-key ours)- t (index-by-key theirs)- all-keys (into #{} (concat (keys b) (keys o) (keys t)))]+(defn- merge-indexed+ "[[three-way-merge]] over sides already indexed by [[index-by-key]]."+ [b o t]+ (let [all-keys (into #{} (concat (keys b) (keys o) (keys t)))](reduce(fn [acc k](let [bv (get b k)@@ -148,6 +136,35 @@{:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}}all-keys)))+(defn three-way-merge+ "Three-way merge of serialized content keyed on serdes identity.++ `base`, `ours`, `theirs` are each sequences of `{:path :content}` file specs (base/theirs typically read+ from the corresponding git trees, ours freshly serialized from the app DB).++ Returns a map:+ - `:merged` - sequence of winning `{:path :content}` specs to write+ - `:conflicts` - sequence of `{:key :ours :theirs :base}` for entities changed differently on both sides+ - `:summary` - `{:added :updated :removed}` counts of remote-originated changes folded into the result+ (i.e. changes coming from `theirs` that `ours` did not already have)"+ [base ours theirs]+ (merge-indexed (index-by-key base) (index-by-key ours) (index-by-key theirs)))++(defn- casualties-indexed+ "[[force-push-casualties]] over sides already indexed by [[index-by-key]]."+ [b o t]+ (reduce-kv+ (fn [acc k tv]+ (let [ov (get o k)+ bv (get b k)]+ (cond+ ;; remote unchanged since base, or already matches ours -> nothing lost+ (or (same? tv bv) (same? tv ov)) acc+ (nil? ov) (update acc :deleted conj (conflict-label {:key k :theirs tv}))+ :else (update acc :overwritten conj (conflict-label {:key k :ours ov :theirs tv})))))+ {:deleted [] :overwritten []}+ t))+(defn force-push-casualties"Remote content that a force push would discard. A force export rewrites every managed file from `ours`,so any change the remote made since the merge `base` is lost. Returns `{:deleted :overwritten}`, each a@@ -160,16 +177,15 @@casualties — that's a routine push, not a loss. `base`, `ours`, `theirs` are sequences of`{:path :content}` specs."[base ours theirs]+ (casualties-indexed (index-by-key base) (index-by-key ours) (index-by-key theirs)))++(defn merge-with-casualties+ "[[three-way-merge]] with `:force-push-casualties` (see [[force-push-casualties]]) assoc'd, from one+ indexing pass per side. Indexing parses every document's YAML, so callers that need both use this rather+ than the two functions separately."+ [base ours theirs](let [b (index-by-key base)- o (index-by-key ours)]- (reduce-kv- (fn [acc k tv]- (let [ov (get o k)- bv (get b k)]- (cond- ;; remote unchanged since base, or already matches ours -> nothing lost- (or (same? tv bv) (same? tv ov)) acc- (nil? ov) (update acc :deleted conj (conflict-label {:key k :theirs tv}))- :else (update acc :overwritten conj (conflict-label {:key k :ours ov :theirs tv})))))- {:deleted [] :overwritten []}- (index-by-key theirs))))+ o (index-by-key ours)+ t (index-by-key theirs)]+ (assoc (merge-indexed b o t)+ :force-push-casualties (casualties-indexed b o t))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex fd3705a5a23..6fbd471c03d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -118,18 +118,25 @@via `task-id` as specs are produced; pass nil for `task-id` to serialize without progress reporting(e.g. for a dry-run merge preview).+ `stream` is traversed exactly once. Progress needs a denominator: pass `:total` (see+ [[metabase-enterprise.remote-sync.spec/exportable-entity-count]]) to keep an uncounted stream such as the+ extraction eduction streaming; without it an uncounted stream is realized first.+Throws Exception if any entity in the stream is an Exception instance."- [stream task-id]- (let [opts (storage-context)- stream-count (bounded-count 10000 stream)]+ [stream task-id & {:keys [total]}]+ (let [opts (storage-context)+ stream (if (or (nil? task-id) total (counted? stream)) stream (vec stream))+ total (or total (when task-id (count stream)))+ report (if (and task-id (pos? total))+ (fn [n]+ (remote-sync.task/update-progress! task-id (-> n (/ total) (min 1) (* 0.65) (+ 0.3))))+ (constantly nil))](into [](map-indexed (fn [idx entity](when (instance? Exception entity)(throw entity))(let [spec (entity->file-spec opts entity)]- (when task-id- (remote-sync.task/update-progress!- task-id (-> (inc idx) (/ stream-count) (* 0.65) (+ 0.3))))+ (report (inc idx))spec)))stream)))@@ -150,19 +157,19 @@"Runs the entity-identity 3-way merge of local state against the remote tip, without writing. Returnsthe raw merge result `{:merged :conflicts :summary}` from [[remote-sync.merge/three-way-merge]], plus`:force-push-casualties` (remote content a force push would discard; see- [[remote-sync.merge/force-push-casualties]]):+ [[remote-sync.merge/force-push-casualties]]), via [[remote-sync.merge/merge-with-casualties]]:- `base-snapshot` - the last successfully synced state (the merge base)- `stream` - the local state to serialize (ours)- `snapshot` - the current remote tip (theirs)`:merged` is the full reconciled set of `{:path :content}` specs. The export path writes it to the- remote; the local-only pull merge loads it into the app DB via [[specs->snapshot]]."- [stream snapshot base-snapshot task-id]- (let [ours (serialize-specs stream task-id)+ remote; the local-only pull merge loads it into the app DB via [[specs->snapshot]]. `:total` is passed+ through to [[serialize-specs]]."+ [stream snapshot base-snapshot task-id & {:keys [total]}]+ (let [ours (serialize-specs stream task-id :total total)base (snapshot->specs base-snapshot)theirs (snapshot->specs snapshot)]- (assoc (remote-sync.merge/three-way-merge base ours theirs)- :force-push-casualties (remote-sync.merge/force-push-casualties base ours theirs))))+ (remote-sync.merge/merge-with-casualties base ours theirs)))(defn specs->snapshot"Builds an in-memory read-only SourceSnapshot backed by `specs` (a seq of `{:path :content}`), so mergeddiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 379a76d630c..82be60f11ca 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -1229,22 +1229,34 @@(into (keys (u/traverse root-targets #(serdes/required (first %) (second %))))))](apply dissoc (u/group-by first second targets) models-traversed-but-not-stored)))+(defn exportable-entity-count+ "Number of entities `targets` (a map as returned by [[exportable-entities]]) names. An upper bound on what+ [[extract-entities-for-export]] yields, suitable for progress reporting."+ [targets]+ (transduce (map count) + 0 (vals targets)))+(defn extract-entities-for-export"Extracts all entities for remote-sync export based on enabled specs.- Returns a lazy sequence of serialized entities ready for storage.+ Returns an eduction of serialized entities ready for storage. Every traversal re-runs the extraction, so+ callers must reduce it once.++ `targets` defaults to [[exportable-entities]]; pass it explicitly when the same map is needed for+ [[exportable-entity-count]] so the dependency walk runs once.Only extracts models that:1. Have a spec in remote-sync-specs2. Are currently enabled (based on :enabled? field)3. Are in one of the provided collections (or descendants)"- []- (eduction (map (fn [[model ids]]- (serdes/extract-all model (merge git-sync-extract-opts- {:filter-column (serdes/primary-key model)- :filter-ids (vec ids)}))))- cat- (exportable-entities)))+ ([]+ (extract-entities-for-export (exportable-entities)))+ ([targets]+ (eduction (map (fn [[model ids]]+ (serdes/extract-all model (merge git-sync-extract-opts+ {:filter-column (serdes/primary-key model)+ :filter-ids (vec ids)}))))+ cat+ targets)))(defn extract-entities-for-rows"Serializes the entities named by `rows`, grouped by model type. Each row is a map with adiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 5d9f76930cd..e0bc364d41e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -14,6 +14,7 @@[metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.app-db.core :as app-db][metabase.collections.models.collection :as collection]+ [metabase.models.serialization.resolve :as resolve][metabase.search.core :as search][metabase.settings.core :as setting][metabase.test :as mt]@@ -1870,10 +1871,10 @@ serdes/meta:(let [reconciled (atom nil)](mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/compute-merge (fn [_ _ _ _]- {:merged [{:path "collections/x.yaml" :content "x"}]- :conflicts []- :summary {:added 2 :updated 1 :removed 0}})+ source/compute-merge (fn [_ _ _ _ & _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []+ :summary {:added 2 :updated 1 :removed 0}})impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}](reset! reconciled (source.p/version snap))(when finalize! (finalize!)))]@@ -1893,10 +1894,10 @@ serdes/meta:(let [reconciled (atom nil)](mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/compute-merge (fn [_ _ _ _]- {:merged [{:path "collections/x.yaml" :content "x"}]- :conflicts []- :summary {:added 1 :updated 0 :removed 0}})+ source/compute-merge (fn [_ _ _ _ & _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []+ :summary {:added 1 :updated 0 :removed 0}})impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}](reset! reconciled (source.p/version snap))(when finalize! (finalize!)))]@@ -1920,12 +1921,12 @@ serdes/meta:(let [reconciled? (atom false)](mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/compute-merge (fn [_ _ _ _]- {:merged []- :conflicts [{:key [["Card" "A"]]- :ours {:path "collections/a.yaml" :content "x"}- :theirs {:path "collections/a.yaml" :content "y"}}]- :summary {:added 0 :updated 0 :removed 0}})+ source/compute-merge (fn [_ _ _ _ & _]+ {:merged []+ :conflicts [{:key [["Card" "A"]]+ :ours {:path "collections/a.yaml" :content "x"}+ :theirs {:path "collections/a.yaml" :content "y"}}]+ :summary {:added 0 :updated 0 :removed 0}})impl/load-snapshot! (fn [_ _ _] (reset! reconciled? true))](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":merge? true@@ -1949,10 +1950,10 @@ serdes/meta:(snapshot-at [_ _] nil))](mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/compute-merge (fn [_ _ _ _]- {:merged [{:path "collections/x.yaml" :content "x"}]- :conflicts []- :summary {:added 1 :updated 0 :removed 0}})+ source/compute-merge (fn [_ _ _ _ & _]+ {:merged [{:path "collections/x.yaml" :content "x"}]+ :conflicts []+ :summary {:added 1 :updated 0 :removed 0}})impl/load-snapshot! (fn [_ _ _ & _] (reset! reconciled? true))](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":merge? true@@ -2036,6 +2037,7 @@ serdes/meta:(testing "preview reports a clean merge with a summary when changes don't conflict"(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")source/source-from-settings (constantly (export-test-source))+ spec/exportable-entities (constantly {"Card" [1]})spec/extract-entities-for-export (constantly [{:dummy true}])source/preview-merge (fn [_ _ _ _]{:clean? true :conflicts []@@ -2047,6 +2049,7 @@ serdes/meta:(testing "preview reports conflicts when the same entity changed on both sides"(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")source/source-from-settings (constantly (export-test-source))+ spec/exportable-entities (constantly {"Card" [1]})spec/extract-entities-for-export (constantly [{:dummy true}])source/preview-merge (fn [_ _ _ _]{:clean? false :conflicts ["Card A (collections/a.yaml)"]@@ -2056,6 +2059,33 @@ serdes/meta::summary {:added 0 :updated 0 :removed 0}}(impl/preview-export-merge "main"))))))+(deftest preview-export-merge-streams-extraction-test+ (testing "preview hands the extraction stream to the merge unrealized and walks the targets once"+ (let [walks (atom 0)+ stream (eduction (map identity) [{:dummy true}])+ received (atom nil)]+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")+ source/source-from-settings (constantly (export-test-source))+ spec/exportable-entities (fn [] (swap! walks inc) {"Card" [1]})+ spec/extract-entities-for-export (fn [_targets] stream)+ source/preview-merge (fn [s _ _ _]+ (reset! received s)+ {:clean? true :conflicts []+ :summary {:added 0 :updated 0 :removed 0}})]+ (impl/preview-export-merge "main")+ (is (identical? stream @received))+ (is (= 1 @walks))))))++(deftest preview-export-merge-nothing-exportable-test+ (testing "preview reports no changes but still diverged when nothing is exportable, without extracting"+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")+ source/source-from-settings (constantly (export-test-source))+ spec/exportable-entities (constantly {})+ spec/extract-entities-for-export (fn [& _] (throw (ex-info "must not extract" {})))]+ (is (= {:diverged? true :clean? true :conflicts [] :summary {:added 0 :updated 0 :removed 0}+ :force-push-casualties {:deleted [] :overwritten []}}+ (impl/preview-export-merge "main"))))))+(deftest preview-export-merge-history-rewritten-test(testing "preview reports :history-rewritten when the merge base is gone"(let [no-base-source (reify source.p/Source@@ -2066,6 +2096,7 @@ serdes/meta:(snapshot-at [_ _] nil))](mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "gone-base")source/source-from-settings (constantly no-base-source)+ spec/exportable-entities (constantly {"Card" [1]})spec/extract-entities-for-export (constantly [{:dummy true}])source/force-push-casualties-no-base (fn [_ _] {:deleted ["Audit Logs"] :overwritten []})](let [result (impl/preview-export-merge "main")]@@ -2086,7 +2117,7 @@ serdes/meta::model_name "Local Card" :status_changed_at :%now}:model/RemoteSyncObject _ {:model_type "Card" :model_id 9992 :status "synced":model_name "Remote Card" :status_changed_at :%now}]- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]{:merged [{:path "collections/x.yaml" :content "y"}]:conflicts []:summary {:added 1 :updated 0 :removed 0}})@@ -2114,7 +2145,7 @@ serdes/meta:(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}:model/RemoteSyncObject _ {:model_type "Card" :model_id 8881 :status "delete":model_name "Deleted Card" :status_changed_at :%now}]- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]{:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}});; simulate the load wiping and not re-inserting the deleted entity's row, then the;; in-transaction finalize (restore-dirty + set-version)@@ -2136,7 +2167,7 @@ serdes/meta::model_name "Locally Deleted" :status_changed_at :%now}:model/RemoteSyncObject _ {:model_type "Card" :model_id 9992 :status "synced":model_name "Remote Card" :status_changed_at :%now}]- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]{:merged [{:path "collections/remote.yaml" :content "y"}]:conflicts []:summary {:added 0 :updated 1 :removed 0}})@@ -2161,7 +2192,7 @@ serdes/meta:(deftest import!-merge-conflict-test(testing "a local-only merge with a genuine conflict returns :conflict and does not load"(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]{:merged []:conflicts [{:key [["Card" "A"]]:ours {:path "collections/a.yaml" :content "x"}@@ -2184,6 +2215,68 @@ serdes/meta:(is (= :conflict (:status result)))(is (str/includes? (:message result) "rewritten"))))))+;;; ------------------------------- merging pull and push extract the library once -------------------------------++(defn- merge-extraction-probe+ "Stubs for a merging sync that record how the extraction reaches `compute-merge`. Returns+ `{:targets :stream :walks :received :stubs}`: `walks` counts target walks; `received` captures the targets the+ extraction was built from, the stream `compute-merge` got, its `:total`, and whether the serdes cache was+ bound; `stream` is the marker eduction the extraction stub hands back."+ []+ (let [walks (atom 0)+ received (atom {})+ targets {"Card" [1 2 3] "Collection" [4]}+ stream (eduction (map identity) [{:dummy true}])]+ {:targets targets+ :stream stream+ :walks walks+ :received received+ :stubs {:exportable-entities (fn [] (swap! walks inc) targets)+ :extract (fn [t] (swap! received assoc :targets t) stream)+ :compute-merge (fn [s _ _ _ & {:keys [total]}]+ (swap! received assoc+ :stream s+ :total total+ :cached? (some? resolve/*export-resolver*))+ {:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}})+ :load-snapshot! (fn [_ _ _ & {:keys [finalize!]}] (when finalize! (finalize!)))}}))++(defn- assert-extracted-once+ [{:keys [targets stream walks received]}]+ (is (= 1 @walks) "the dependency walk runs once")+ (is (identical? targets (:targets @received)) "the extraction is built from the walked targets")+ (is (identical? stream (:stream @received)) "the merge receives the extraction stream unrealized")+ (is (= (spec/exportable-entity-count targets) (:total @received)) ":total is the count of the same targets")+ (is (true? (:cached? @received)) "the merge extracts under serdes/with-cache"))++(deftest import!-merge-extracts-once-test+ (testing "a merging pull walks the targets once and hands the merge the unrealized extraction with its total"+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]+ (let [{{:keys [exportable-entities extract compute-merge load-snapshot!]} :stubs :as probe} (merge-extraction-probe)]+ (mt/with-dynamic-fn-redefs [spec/exportable-entities exportable-entities+ spec/extract-entities-for-export extract+ source/compute-merge compute-merge+ impl/load-snapshot! load-snapshot!]+ (is (= :success (:status (impl/import! (export-test-snapshot "remote-R") task-id+ :merge? true+ :base-snapshot (export-test-snapshot "base-B")))))+ (assert-extracted-once probe))))))++(deftest export!-merge-extracts-once-test+ (testing "a merging push walks the targets once and hands the merge the unrealized extraction with its total"+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]+ (let [{{:keys [exportable-entities extract compute-merge load-snapshot!]} :stubs :as probe} (merge-extraction-probe)]+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")+ spec/exportable-entities exportable-entities+ spec/extract-entities-for-export extract+ source/compute-merge compute-merge+ impl/load-snapshot! load-snapshot!]+ (is (= :success (:status (impl/export! (export-test-snapshot "remote-R") task-id "msg"+ :merge? true+ :source (export-test-source)+ :base-snapshot (export-test-snapshot "base-B")))))+ (assert-extracted-once probe))))))+;;; --------------------------------- Table/Field user-settings inline round trip ---------------------------------(deftest table-and-field-user-settings-round-trip-testdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.cljindex a25602a45eb..d0d67c52acf 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj@@ -1,7 +1,9 @@(ns metabase-enterprise.remote-sync.merge-test(:require[clojure.test :refer :all]- [metabase-enterprise.remote-sync.merge :as remote-sync.merge]))+ [metabase-enterprise.remote-sync.merge :as remote-sync.merge]+ [metabase.test :as mt]+ [metabase.util.yaml :as yaml]))(defn- card"Builds a `{:path :content}` spec for a Card with the given entity `id`, `name` (which drives the on-disk@@ -214,3 +216,22 @@[][(card "A" "a")][(card "A" "a")]))))))++(deftest ^:parallel merge-with-casualties-indexes-each-side-once-test+ (let [base [(card "A" "a") (card "B" "b")]+ ours [(card "A" "a" "x: ours\n") (card "B" "b")]+ theirs [(card "A" "a" "x: theirs\n") (card "C" "c")]+ expected (assoc (remote-sync.merge/three-way-merge base ours theirs)+ :force-push-casualties (remote-sync.merge/force-push-casualties base ours theirs))+ parses (atom 0)+ orig (mt/original-fn #'yaml/parse-string)+ counting (fn [f] (reset! parses 0) (f) @parses)]+ (mt/with-dynamic-fn-redefs [yaml/parse-string (fn [& args] (swap! parses inc) (apply orig args))]+ (testing "the combined result equals the two functions run separately"+ (is (= expected (remote-sync.merge/merge-with-casualties base ours theirs))))+ (testing "combining saves exactly one identity parse per document across the three sides"+ (let [separate (counting #(do (remote-sync.merge/three-way-merge base ours theirs)+ (remote-sync.merge/force-push-casualties base ours theirs)))+ combined (counting #(remote-sync.merge/merge-with-casualties base ours theirs))]+ (is (= (+ (count base) (count ours) (count theirs))+ (- separate combined))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex bef49b46bb9..b052e6cdb72 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -7,7 +7,8 @@[metabase-enterprise.remote-sync.test-helpers :as th][metabase.models.serialization :as serdes][metabase.test :as mt]- [metabase.test.fixtures :as fixtures])+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2])(:import(org.eclipse.jgit.lib ObjectChecker)))@@ -118,3 +119,23 @@(is (= 1 (count (:conflicts result))))(is (every? string? (:conflicts result)))(is (nil? @written))))))++(deftest serialize-specs-traverses-stream-once-test+ (let [entities [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")]+ counting (fn [traversals] (eduction (map (fn [e] (swap! traversals inc) e)) entities))]+ (testing "an uncounted stream with :total is serialized in one pass and progress is reported against :total"+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]+ (let [traversals (atom 0)+ specs (source/serialize-specs (counting traversals) task-id :total 4)]+ (is (= 2 (count specs)))+ (is (= 2 @traversals))+ (is (= 0.625 (double (t2/select-one-fn :progress :model/RemoteSyncTask :id task-id)))))))+ (testing "an uncounted stream without :total is still traversed once"+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]+ (let [traversals (atom 0)]+ (is (= 2 (count (source/serialize-specs (counting traversals) task-id))))+ (is (= 2 @traversals)))))+ (testing "without a task id the stream is serialized in one pass with no progress writes"+ (let [traversals (atom 0)]+ (is (= 2 (count (source/serialize-specs (counting traversals) nil))))+ (is (= 2 @traversals))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex 62db8591515..06a54387207 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -678,3 +678,8 @@"the Table has no settings row of its own")(is (contains? (set (get exportable "TableUserSettings")) table-id)"the Table is still exportable, synthesized from its Field's edit"))))))++(deftest ^:parallel exportable-entity-count-test+ (testing "exportable-entity-count sums the ids across every model in the targets map"+ (is (= 0 (spec/exportable-entity-count {})))+ (is (= 5 (spec/exportable-entity-count {"Card" [1 2 3] "Collection" [4 5]})))))diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 860cb734479..613a6a7a683 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -61,8 +61,6 @@[malli.core :as mc][malli.transform :as mtx][medley.core :as m]- ;; legacy usages -- do not use in new code- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common]@@ -1020,11 +1018,7 @@(let [tag (mbql-ref? mbql)schema (case tag:field-id ::mbql-3-field-id-ref- :field [:multi- {:dispatch #(and (vector? %)- (map? (second %)))}- [true :mbql.clause/field]- [false ::mbql.s/field]] ; legacy MBQL clause+ :field ::resolve/field-ref:dimension ::lib.schema.parameter/dimension:metric :mbql.clause/metric:segment :mbql.clause/segmentdiff --git a/src/metabase/models/serialization/resolve.clj b/src/metabase/models/serialization/resolve.cljindex 77d592cee92..3afee116f3b 100644--- a/src/metabase/models/serialization/resolve.clj+++ b/src/metabase/models/serialization/resolve.clj@@ -222,14 +222,19 @@(#{:field :dimension :metric :segment :measure} (keyword (first form))))(keyword (first form))))+(mr/def ::field-ref+ "MBQL 5 or legacy `:field` clause. Registered under a keyword so [[lib/normalize]] reuses one cached coercer; an+ inline literal with a fresh dispatch fn misses the registry cache on every call."+ [:multi+ {:dispatch #(and (vector? %)+ (map? (second %)))}+ [true :mbql.clause/field]+ [false ::mbql.s/field]])+(defn- normalize [mbql](let [tag (mbql-clause-tag mbql)schema (case tag- :field [:multi- {:dispatch #(and (vector? %)- (map? (second %)))}- [true :mbql.clause/field]- [false ::mbql.s/field]] ; legacy MBQL clause+ :field ::field-ref:dimension ::lib.schema.parameter/dimension:metric :mbql.clause/metric:segment :mbql.clause/segmentdiff --git a/test/metabase/models/serialization/resolve_test.clj b/test/metabase/models/serialization/resolve_test.cljnew file mode 100644index 00000000000..da2dc545a37--- /dev/null+++ b/test/metabase/models/serialization/resolve_test.clj@@ -0,0 +1,17 @@+(ns metabase.models.serialization.resolve-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.normalize :as lib.normalize]+ [metabase.models.serialization.resolve :as resolve]+ [metabase.util.malli.registry :as mr]))++(deftest ^:parallel normalize-field-ref-reuses-cached-coercer-test+ (testing "normalizing :field refs hits the registry coercer cache after the first call"+ (let [misses (atom 0)]+ (binding [mr/*cache-miss-hook* (fn [k _schema _value]+ (when (= k ::lib.normalize/coercer)+ (swap! misses inc)))]+ (dotimes [_ 3]+ (#'resolve/normalize [:field 1 nil])+ (#'resolve/normalize [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))+ (is (<= @misses 1)))))diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.cljindex a28ea8521b7..ba509a35d26 100644--- a/test/metabase/models/serialization_test.clj+++ b/test/metabase/models/serialization_test.clj@@ -2,8 +2,10 @@(:require[clojure.test :refer :all][metabase.lib.core :as lib]+ [metabase.lib.normalize :as lib.normalize][metabase.lib.test-metadata :as meta]- [metabase.models.serialization :as serdes]))+ [metabase.models.serialization :as serdes]+ [metabase.util.malli.registry :as mr]))(defn- fake-uuid"Deterministic placeholder `:lib/uuid` for tests, e.g. `(fake-uuid 1)` => \"00000000-0000-0000-0000-000000000001\"."@@ -183,6 +185,17 @@["fk->" ["field-id" 1] ["field-id" 2]]["fk->" [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]))))+(deftest ^:parallel normalize-field-ref-reuses-cached-coercer-test+ (testing "normalizing :field refs hits the registry coercer cache after the first call"+ (let [misses (atom 0)]+ (binding [mr/*cache-miss-hook* (fn [k _schema _value]+ (when (= k ::lib.normalize/coercer)+ (swap! misses inc)))]+ (dotimes [_ 3]+ (#'serdes/normalize-mbql-ref [:field 1 nil])+ (#'serdes/normalize-mbql-ref [:field {:lib/uuid (fake-uuid 1)} 1])))+ (is (<= @misses 1)))))+(deftest ^:parallel export-visualization-settings-test(binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])serdes/*export-fk* (fn [id model]