Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GHY-4600] Improve memory allocation of remote sync (#82625)

Tyler Plude56232fc3f8d8
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 0da994890c1..c8ab89eed2e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -522,7 +522,11 @@
- Local changes stay dirty
- Sets version to remote tip"
[snapshot base-snapshot task-id sync-timestamp]
- (let [{:keys [conflicts merged summary]} (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id)]
+ (let [{:keys [conflicts merged summary]} (serdes/with-cache
+ (let [targets (spec/exportable-entities)]
+ (source/compute-merge (spec/extract-entities-for-export targets)
+ snapshot base-snapshot task-id
+ :total (spec/exportable-entity-count targets))))]
(if (seq conflicts)
(let [labels (mapv remote-sync.merge/conflict-label conflicts)]
(log/infof "Pull merge conflict on %d entit(ies)" (count labels))
@@ -738,9 +742,9 @@
changes.
Returns a `:success` result with a `:merge-summary`."
- [source snapshot base-snapshot task-id message sync-timestamp models]
+ [source snapshot base-snapshot task-id message sync-timestamp models & {:keys [total]}]
(let [pushed-count (count (remote-sync.object/dirty-rows))
- {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id)]
+ {:keys [merged conflicts summary]} (source/compute-merge models snapshot base-snapshot task-id :total total)]
(if (seq conflicts)
(let [labels (mapv remote-sync.merge/conflict-label conflicts)]
(log/infof "Export merge conflict on %d entit(ies)" (count labels))
@@ -1208,8 +1212,10 @@
:message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}
:else
- (export-merged! src snapshot base-snapshot task-id message sync-timestamp
- (spec/extract-entities-for-export)))
+ (let [targets (spec/exportable-entities)]
+ (export-merged! src snapshot base-snapshot task-id message sync-timestamp
+ (spec/extract-entities-for-export targets)
+ :total (spec/exportable-entity-count targets))))
diverged? ;; and not merge? option
{:status :conflict
@@ -1567,17 +1573,21 @@
no-changes
(if-let [base-snapshot (source.p/snapshot-at source base-version)]
(serdes/with-cache
- (if-let [models (seq (spec/extract-entities-for-export))]
- (assoc (source/preview-merge models snapshot base-snapshot nil) :diverged? true)
- (assoc no-changes :diverged? true)))
+ (let [targets (spec/exportable-entities)]
+ (if (seq targets)
+ (assoc (source/preview-merge (spec/extract-entities-for-export targets) snapshot base-snapshot nil)
+ :diverged? true)
+ (assoc no-changes :diverged? true))))
;; No merge base — the remote history was rewritten. A merge is impossible, but a force push is
;; still offered, so surface what it would discard (every remote entity not identical to ours).
{:diverged? true :clean? false :reason :history-rewritten
:conflicts [] :summary {:added 0 :updated 0 :removed 0}
:force-push-casualties (serdes/with-cache
- (if-let [models (seq (spec/extract-entities-for-export))]
- (source/force-push-casualties-no-base models snapshot)
- {:deleted [] :overwritten []}))}))))
+ (let [targets (spec/exportable-entities)]
+ (if (seq targets)
+ (source/force-push-casualties-no-base
+ (spec/extract-entities-for-export targets) snapshot)
+ {:deleted [] :overwritten []})))}))))
(defn create-branch!
"Creates a new remote branch from `base-branch` and switches `remote-sync-branch`
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
index c3293c72427..b2c877b0132 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/merge.clj
@@ -98,22 +98,10 @@
(cond-> (or (:name entity) descriptor)
(and path (not= descriptor path)) (str " (" path ")"))))
-(defn three-way-merge
- "Three-way merge of serialized content keyed on serdes identity.
-
- `base`, `ours`, `theirs` are each sequences of `{:path :content}` file specs (base/theirs typically read
- from the corresponding git trees, ours freshly serialized from the app DB).
-
- Returns a map:
- - `:merged` - sequence of winning `{:path :content}` specs to write
- - `:conflicts` - sequence of `{:key :ours :theirs :base}` for entities changed differently on both sides
- - `:summary` - `{:added :updated :removed}` counts of remote-originated changes folded into the result
- (i.e. changes coming from `theirs` that `ours` did not already have)"
- [base ours theirs]
- (let [b (index-by-key base)
- o (index-by-key ours)
- t (index-by-key theirs)
- all-keys (into #{} (concat (keys b) (keys o) (keys t)))]
+(defn- merge-indexed
+ "[[three-way-merge]] over sides already indexed by [[index-by-key]]."
+ [b o t]
+ (let [all-keys (into #{} (concat (keys b) (keys o) (keys t)))]
(reduce
(fn [acc k]
(let [bv (get b k)
@@ -148,6 +136,35 @@
{:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}}
all-keys)))
+(defn three-way-merge
+ "Three-way merge of serialized content keyed on serdes identity.
+
+ `base`, `ours`, `theirs` are each sequences of `{:path :content}` file specs (base/theirs typically read
+ from the corresponding git trees, ours freshly serialized from the app DB).
+
+ Returns a map:
+ - `:merged` - sequence of winning `{:path :content}` specs to write
+ - `:conflicts` - sequence of `{:key :ours :theirs :base}` for entities changed differently on both sides
+ - `:summary` - `{:added :updated :removed}` counts of remote-originated changes folded into the result
+ (i.e. changes coming from `theirs` that `ours` did not already have)"
+ [base ours theirs]
+ (merge-indexed (index-by-key base) (index-by-key ours) (index-by-key theirs)))
+
+(defn- casualties-indexed
+ "[[force-push-casualties]] over sides already indexed by [[index-by-key]]."
+ [b o t]
+ (reduce-kv
+ (fn [acc k tv]
+ (let [ov (get o k)
+ bv (get b k)]
+ (cond
+ ;; remote unchanged since base, or already matches ours -> nothing lost
+ (or (same? tv bv) (same? tv ov)) acc
+ (nil? ov) (update acc :deleted conj (conflict-label {:key k :theirs tv}))
+ :else (update acc :overwritten conj (conflict-label {:key k :ours ov :theirs tv})))))
+ {:deleted [] :overwritten []}
+ t))
+
(defn force-push-casualties
"Remote content that a force push would discard. A force export rewrites every managed file from `ours`,
so any change the remote made since the merge `base` is lost. Returns `{:deleted :overwritten}`, each a
@@ -160,16 +177,15 @@
casualties — that's a routine push, not a loss. `base`, `ours`, `theirs` are sequences of
`{:path :content}` specs."
[base ours theirs]
+ (casualties-indexed (index-by-key base) (index-by-key ours) (index-by-key theirs)))
+
+(defn merge-with-casualties
+ "[[three-way-merge]] with `:force-push-casualties` (see [[force-push-casualties]]) assoc'd, from one
+ indexing pass per side. Indexing parses every document's YAML, so callers that need both use this rather
+ than the two functions separately."
+ [base ours theirs]
(let [b (index-by-key base)
- o (index-by-key ours)]
- (reduce-kv
- (fn [acc k tv]
- (let [ov (get o k)
- bv (get b k)]
- (cond
- ;; remote unchanged since base, or already matches ours -> nothing lost
- (or (same? tv bv) (same? tv ov)) acc
- (nil? ov) (update acc :deleted conj (conflict-label {:key k :theirs tv}))
- :else (update acc :overwritten conj (conflict-label {:key k :ours ov :theirs tv})))))
- {:deleted [] :overwritten []}
- (index-by-key theirs))))
+ o (index-by-key ours)
+ t (index-by-key theirs)]
+ (assoc (merge-indexed b o t)
+ :force-push-casualties (casualties-indexed b o t))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index fd3705a5a23..6fbd471c03d 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -118,18 +118,25 @@
via `task-id` as specs are produced; pass nil for `task-id` to serialize without progress reporting
(e.g. for a dry-run merge preview).
+ `stream` is traversed exactly once. Progress needs a denominator: pass `:total` (see
+ [[metabase-enterprise.remote-sync.spec/exportable-entity-count]]) to keep an uncounted stream such as the
+ extraction eduction streaming; without it an uncounted stream is realized first.
+
Throws Exception if any entity in the stream is an Exception instance."
- [stream task-id]
- (let [opts (storage-context)
- stream-count (bounded-count 10000 stream)]
+ [stream task-id & {:keys [total]}]
+ (let [opts (storage-context)
+ stream (if (or (nil? task-id) total (counted? stream)) stream (vec stream))
+ total (or total (when task-id (count stream)))
+ report (if (and task-id (pos? total))
+ (fn [n]
+ (remote-sync.task/update-progress! task-id (-> n (/ total) (min 1) (* 0.65) (+ 0.3))))
+ (constantly nil))]
(into []
(map-indexed (fn [idx entity]
(when (instance? Exception entity)
(throw entity))
(let [spec (entity->file-spec opts entity)]
- (when task-id
- (remote-sync.task/update-progress!
- task-id (-> (inc idx) (/ stream-count) (* 0.65) (+ 0.3))))
+ (report (inc idx))
spec)))
stream)))
@@ -150,19 +157,19 @@
"Runs the entity-identity 3-way merge of local state against the remote tip, without writing. Returns
the raw merge result `{:merged :conflicts :summary}` from [[remote-sync.merge/three-way-merge]], plus
`:force-push-casualties` (remote content a force push would discard; see
- [[remote-sync.merge/force-push-casualties]]):
+ [[remote-sync.merge/force-push-casualties]]), via [[remote-sync.merge/merge-with-casualties]]:
- `base-snapshot` - the last successfully synced state (the merge base)
- `stream` - the local state to serialize (ours)
- `snapshot` - the current remote tip (theirs)
`:merged` is the full reconciled set of `{:path :content}` specs. The export path writes it to the
- remote; the local-only pull merge loads it into the app DB via [[specs->snapshot]]."
- [stream snapshot base-snapshot task-id]
- (let [ours (serialize-specs stream task-id)
+ remote; the local-only pull merge loads it into the app DB via [[specs->snapshot]]. `:total` is passed
+ through to [[serialize-specs]]."
+ [stream snapshot base-snapshot task-id & {:keys [total]}]
+ (let [ours (serialize-specs stream task-id :total total)
base (snapshot->specs base-snapshot)
theirs (snapshot->specs snapshot)]
- (assoc (remote-sync.merge/three-way-merge base ours theirs)
- :force-push-casualties (remote-sync.merge/force-push-casualties base ours theirs))))
+ (remote-sync.merge/merge-with-casualties base ours theirs)))
(defn specs->snapshot
"Builds an in-memory read-only SourceSnapshot backed by `specs` (a seq of `{:path :content}`), so merged
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 379a76d630c..82be60f11ca 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -1229,22 +1229,34 @@
(into (keys (u/traverse root-targets #(serdes/required (first %) (second %))))))]
(apply dissoc (u/group-by first second targets) models-traversed-but-not-stored)))
+(defn exportable-entity-count
+ "Number of entities `targets` (a map as returned by [[exportable-entities]]) names. An upper bound on what
+ [[extract-entities-for-export]] yields, suitable for progress reporting."
+ [targets]
+ (transduce (map count) + 0 (vals targets)))
+
(defn extract-entities-for-export
"Extracts all entities for remote-sync export based on enabled specs.
- Returns a lazy sequence of serialized entities ready for storage.
+ Returns an eduction of serialized entities ready for storage. Every traversal re-runs the extraction, so
+ callers must reduce it once.
+
+ `targets` defaults to [[exportable-entities]]; pass it explicitly when the same map is needed for
+ [[exportable-entity-count]] so the dependency walk runs once.
Only extracts models that:
1. Have a spec in remote-sync-specs
2. Are currently enabled (based on :enabled? field)
3. Are in one of the provided collections (or descendants)"
- []
- (eduction (map (fn [[model ids]]
- (serdes/extract-all model (merge git-sync-extract-opts
- {:filter-column (serdes/primary-key model)
- :filter-ids (vec ids)}))))
- cat
- (exportable-entities)))
+ ([]
+ (extract-entities-for-export (exportable-entities)))
+ ([targets]
+ (eduction (map (fn [[model ids]]
+ (serdes/extract-all model (merge git-sync-extract-opts
+ {:filter-column (serdes/primary-key model)
+ :filter-ids (vec ids)}))))
+ cat
+ targets)))
(defn extract-entities-for-rows
"Serializes the entities named by `rows`, grouped by model type. Each row is a map with a
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 5d9f76930cd..e0bc364d41e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -14,6 +14,7 @@
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.app-db.core :as app-db]
[metabase.collections.models.collection :as collection]
+ [metabase.models.serialization.resolve :as resolve]
[metabase.search.core :as search]
[metabase.settings.core :as setting]
[metabase.test :as mt]
@@ -1870,10 +1871,10 @@ serdes/meta:
(let [reconciled (atom nil)]
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/compute-merge (fn [_ _ _ _]
- {:merged [{:path "collections/x.yaml" :content "x"}]
- :conflicts []
- :summary {:added 2 :updated 1 :removed 0}})
+ source/compute-merge (fn [_ _ _ _ & _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
+ :summary {:added 2 :updated 1 :removed 0}})
impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}]
(reset! reconciled (source.p/version snap))
(when finalize! (finalize!)))]
@@ -1893,10 +1894,10 @@ serdes/meta:
(let [reconciled (atom nil)]
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/compute-merge (fn [_ _ _ _]
- {:merged [{:path "collections/x.yaml" :content "x"}]
- :conflicts []
- :summary {:added 1 :updated 0 :removed 0}})
+ source/compute-merge (fn [_ _ _ _ & _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
+ :summary {:added 1 :updated 0 :removed 0}})
impl/load-snapshot! (fn [snap _ _ & {:keys [finalize!]}]
(reset! reconciled (source.p/version snap))
(when finalize! (finalize!)))]
@@ -1920,12 +1921,12 @@ serdes/meta:
(let [reconciled? (atom false)]
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/compute-merge (fn [_ _ _ _]
- {:merged []
- :conflicts [{:key [["Card" "A"]]
- :ours {:path "collections/a.yaml" :content "x"}
- :theirs {:path "collections/a.yaml" :content "y"}}]
- :summary {:added 0 :updated 0 :removed 0}})
+ source/compute-merge (fn [_ _ _ _ & _]
+ {:merged []
+ :conflicts [{:key [["Card" "A"]]
+ :ours {:path "collections/a.yaml" :content "x"}
+ :theirs {:path "collections/a.yaml" :content "y"}}]
+ :summary {:added 0 :updated 0 :removed 0}})
impl/load-snapshot! (fn [_ _ _] (reset! reconciled? true))]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:merge? true
@@ -1949,10 +1950,10 @@ serdes/meta:
(snapshot-at [_ _] nil))]
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/compute-merge (fn [_ _ _ _]
- {:merged [{:path "collections/x.yaml" :content "x"}]
- :conflicts []
- :summary {:added 1 :updated 0 :removed 0}})
+ source/compute-merge (fn [_ _ _ _ & _]
+ {:merged [{:path "collections/x.yaml" :content "x"}]
+ :conflicts []
+ :summary {:added 1 :updated 0 :removed 0}})
impl/load-snapshot! (fn [_ _ _ & _] (reset! reconciled? true))]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:merge? true
@@ -2036,6 +2037,7 @@ serdes/meta:
(testing "preview reports a clean merge with a summary when changes don't conflict"
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
source/source-from-settings (constantly (export-test-source))
+ spec/exportable-entities (constantly {"Card" [1]})
spec/extract-entities-for-export (constantly [{:dummy true}])
source/preview-merge (fn [_ _ _ _]
{:clean? true :conflicts []
@@ -2047,6 +2049,7 @@ serdes/meta:
(testing "preview reports conflicts when the same entity changed on both sides"
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
source/source-from-settings (constantly (export-test-source))
+ spec/exportable-entities (constantly {"Card" [1]})
spec/extract-entities-for-export (constantly [{:dummy true}])
source/preview-merge (fn [_ _ _ _]
{:clean? false :conflicts ["Card A (collections/a.yaml)"]
@@ -2056,6 +2059,33 @@ serdes/meta:
:summary {:added 0 :updated 0 :removed 0}}
(impl/preview-export-merge "main"))))))
+(deftest preview-export-merge-streams-extraction-test
+ (testing "preview hands the extraction stream to the merge unrealized and walks the targets once"
+ (let [walks (atom 0)
+ stream (eduction (map identity) [{:dummy true}])
+ received (atom nil)]
+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
+ source/source-from-settings (constantly (export-test-source))
+ spec/exportable-entities (fn [] (swap! walks inc) {"Card" [1]})
+ spec/extract-entities-for-export (fn [_targets] stream)
+ source/preview-merge (fn [s _ _ _]
+ (reset! received s)
+ {:clean? true :conflicts []
+ :summary {:added 0 :updated 0 :removed 0}})]
+ (impl/preview-export-merge "main")
+ (is (identical? stream @received))
+ (is (= 1 @walks))))))
+
+(deftest preview-export-merge-nothing-exportable-test
+ (testing "preview reports no changes but still diverged when nothing is exportable, without extracting"
+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
+ source/source-from-settings (constantly (export-test-source))
+ spec/exportable-entities (constantly {})
+ spec/extract-entities-for-export (fn [& _] (throw (ex-info "must not extract" {})))]
+ (is (= {:diverged? true :clean? true :conflicts [] :summary {:added 0 :updated 0 :removed 0}
+ :force-push-casualties {:deleted [] :overwritten []}}
+ (impl/preview-export-merge "main"))))))
+
(deftest preview-export-merge-history-rewritten-test
(testing "preview reports :history-rewritten when the merge base is gone"
(let [no-base-source (reify source.p/Source
@@ -2066,6 +2096,7 @@ serdes/meta:
(snapshot-at [_ _] nil))]
(mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "gone-base")
source/source-from-settings (constantly no-base-source)
+ spec/exportable-entities (constantly {"Card" [1]})
spec/extract-entities-for-export (constantly [{:dummy true}])
source/force-push-casualties-no-base (fn [_ _] {:deleted ["Audit Logs"] :overwritten []})]
(let [result (impl/preview-export-merge "main")]
@@ -2086,7 +2117,7 @@ serdes/meta:
:model_name "Local Card" :status_changed_at :%now}
:model/RemoteSyncObject _ {:model_type "Card" :model_id 9992 :status "synced"
:model_name "Remote Card" :status_changed_at :%now}]
- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]
+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]
{:merged [{:path "collections/x.yaml" :content "y"}]
:conflicts []
:summary {:added 1 :updated 0 :removed 0}})
@@ -2114,7 +2145,7 @@ serdes/meta:
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}
:model/RemoteSyncObject _ {:model_type "Card" :model_id 8881 :status "delete"
:model_name "Deleted Card" :status_changed_at :%now}]
- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]
+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]
{:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}})
;; simulate the load wiping and not re-inserting the deleted entity's row, then the
;; in-transaction finalize (restore-dirty + set-version)
@@ -2136,7 +2167,7 @@ serdes/meta:
:model_name "Locally Deleted" :status_changed_at :%now}
:model/RemoteSyncObject _ {:model_type "Card" :model_id 9992 :status "synced"
:model_name "Remote Card" :status_changed_at :%now}]
- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]
+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]
{:merged [{:path "collections/remote.yaml" :content "y"}]
:conflicts []
:summary {:added 0 :updated 1 :removed 0}})
@@ -2161,7 +2192,7 @@ serdes/meta:
(deftest import!-merge-conflict-test
(testing "a local-only merge with a genuine conflict returns :conflict and does not load"
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]
- (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _]
+ (mt/with-dynamic-fn-redefs [source/compute-merge (fn [_ _ _ _ & _]
{:merged []
:conflicts [{:key [["Card" "A"]]
:ours {:path "collections/a.yaml" :content "x"}
@@ -2184,6 +2215,68 @@ serdes/meta:
(is (= :conflict (:status result)))
(is (str/includes? (:message result) "rewritten"))))))
+;;; ------------------------------- merging pull and push extract the library once -------------------------------
+
+(defn- merge-extraction-probe
+ "Stubs for a merging sync that record how the extraction reaches `compute-merge`. Returns
+ `{:targets :stream :walks :received :stubs}`: `walks` counts target walks; `received` captures the targets the
+ extraction was built from, the stream `compute-merge` got, its `:total`, and whether the serdes cache was
+ bound; `stream` is the marker eduction the extraction stub hands back."
+ []
+ (let [walks (atom 0)
+ received (atom {})
+ targets {"Card" [1 2 3] "Collection" [4]}
+ stream (eduction (map identity) [{:dummy true}])]
+ {:targets targets
+ :stream stream
+ :walks walks
+ :received received
+ :stubs {:exportable-entities (fn [] (swap! walks inc) targets)
+ :extract (fn [t] (swap! received assoc :targets t) stream)
+ :compute-merge (fn [s _ _ _ & {:keys [total]}]
+ (swap! received assoc
+ :stream s
+ :total total
+ :cached? (some? resolve/*export-resolver*))
+ {:merged [] :conflicts [] :summary {:added 0 :updated 0 :removed 0}})
+ :load-snapshot! (fn [_ _ _ & {:keys [finalize!]}] (when finalize! (finalize!)))}}))
+
+(defn- assert-extracted-once
+ [{:keys [targets stream walks received]}]
+ (is (= 1 @walks) "the dependency walk runs once")
+ (is (identical? targets (:targets @received)) "the extraction is built from the walked targets")
+ (is (identical? stream (:stream @received)) "the merge receives the extraction stream unrealized")
+ (is (= (spec/exportable-entity-count targets) (:total @received)) ":total is the count of the same targets")
+ (is (true? (:cached? @received)) "the merge extracts under serdes/with-cache"))
+
+(deftest import!-merge-extracts-once-test
+ (testing "a merging pull walks the targets once and hands the merge the unrealized extraction with its total"
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]
+ (let [{{:keys [exportable-entities extract compute-merge load-snapshot!]} :stubs :as probe} (merge-extraction-probe)]
+ (mt/with-dynamic-fn-redefs [spec/exportable-entities exportable-entities
+ spec/extract-entities-for-export extract
+ source/compute-merge compute-merge
+ impl/load-snapshot! load-snapshot!]
+ (is (= :success (:status (impl/import! (export-test-snapshot "remote-R") task-id
+ :merge? true
+ :base-snapshot (export-test-snapshot "base-B")))))
+ (assert-extracted-once probe))))))
+
+(deftest export!-merge-extracts-once-test
+ (testing "a merging push walks the targets once and hands the merge the unrealized extraction with its total"
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
+ (let [{{:keys [exportable-entities extract compute-merge load-snapshot!]} :stubs :as probe} (merge-extraction-probe)]
+ (mt/with-dynamic-fn-redefs [remote-sync.task/last-version (constantly "base-B")
+ spec/exportable-entities exportable-entities
+ spec/extract-entities-for-export extract
+ source/compute-merge compute-merge
+ impl/load-snapshot! load-snapshot!]
+ (is (= :success (:status (impl/export! (export-test-snapshot "remote-R") task-id "msg"
+ :merge? true
+ :source (export-test-source)
+ :base-snapshot (export-test-snapshot "base-B")))))
+ (assert-extracted-once probe))))))
+
;;; --------------------------------- Table/Field user-settings inline round trip ---------------------------------
(deftest table-and-field-user-settings-round-trip-test
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj
index a25602a45eb..d0d67c52acf 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/merge_test.clj
@@ -1,7 +1,9 @@
(ns metabase-enterprise.remote-sync.merge-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.remote-sync.merge :as remote-sync.merge]))
+ [metabase-enterprise.remote-sync.merge :as remote-sync.merge]
+ [metabase.test :as mt]
+ [metabase.util.yaml :as yaml]))
(defn- card
"Builds a `{:path :content}` spec for a Card with the given entity `id`, `name` (which drives the on-disk
@@ -214,3 +216,22 @@
[]
[(card "A" "a")]
[(card "A" "a")]))))))
+
+(deftest ^:parallel merge-with-casualties-indexes-each-side-once-test
+ (let [base [(card "A" "a") (card "B" "b")]
+ ours [(card "A" "a" "x: ours\n") (card "B" "b")]
+ theirs [(card "A" "a" "x: theirs\n") (card "C" "c")]
+ expected (assoc (remote-sync.merge/three-way-merge base ours theirs)
+ :force-push-casualties (remote-sync.merge/force-push-casualties base ours theirs))
+ parses (atom 0)
+ orig (mt/original-fn #'yaml/parse-string)
+ counting (fn [f] (reset! parses 0) (f) @parses)]
+ (mt/with-dynamic-fn-redefs [yaml/parse-string (fn [& args] (swap! parses inc) (apply orig args))]
+ (testing "the combined result equals the two functions run separately"
+ (is (= expected (remote-sync.merge/merge-with-casualties base ours theirs))))
+ (testing "combining saves exactly one identity parse per document across the three sides"
+ (let [separate (counting #(do (remote-sync.merge/three-way-merge base ours theirs)
+ (remote-sync.merge/force-push-casualties base ours theirs)))
+ combined (counting #(remote-sync.merge/merge-with-casualties base ours theirs))]
+ (is (= (+ (count base) (count ours) (count theirs))
+ (- separate combined))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index bef49b46bb9..b052e6cdb72 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -7,7 +7,8 @@
[metabase-enterprise.remote-sync.test-helpers :as th]
[metabase.models.serialization :as serdes]
[metabase.test :as mt]
- [metabase.test.fixtures :as fixtures])
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2])
(:import
(org.eclipse.jgit.lib ObjectChecker)))
@@ -118,3 +119,23 @@
(is (= 1 (count (:conflicts result))))
(is (every? string? (:conflicts result)))
(is (nil? @written))))))
+
+(deftest serialize-specs-traverses-stream-once-test
+ (let [entities [(create-test-entity "A" "a" "Card") (create-test-entity "B" "b" "Card")]
+ counting (fn [traversals] (eduction (map (fn [e] (swap! traversals inc) e)) entities))]
+ (testing "an uncounted stream with :total is serialized in one pass and progress is reported against :total"
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]
+ (let [traversals (atom 0)
+ specs (source/serialize-specs (counting traversals) task-id :total 4)]
+ (is (= 2 (count specs)))
+ (is (= 2 @traversals))
+ (is (= 0.625 (double (t2/select-one-fn :progress :model/RemoteSyncTask :id task-id)))))))
+ (testing "an uncounted stream without :total is still traversed once"
+ (mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "import"}]
+ (let [traversals (atom 0)]
+ (is (= 2 (count (source/serialize-specs (counting traversals) task-id))))
+ (is (= 2 @traversals)))))
+ (testing "without a task id the stream is serialized in one pass with no progress writes"
+ (let [traversals (atom 0)]
+ (is (= 2 (count (source/serialize-specs (counting traversals) nil))))
+ (is (= 2 @traversals))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index 62db8591515..06a54387207 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -678,3 +678,8 @@
"the Table has no settings row of its own")
(is (contains? (set (get exportable "TableUserSettings")) table-id)
"the Table is still exportable, synthesized from its Field's edit"))))))
+
+(deftest ^:parallel exportable-entity-count-test
+ (testing "exportable-entity-count sums the ids across every model in the targets map"
+ (is (= 0 (spec/exportable-entity-count {})))
+ (is (= 5 (spec/exportable-entity-count {"Card" [1 2 3] "Collection" [4 5]})))))
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 860cb734479..613a6a7a683 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -61,8 +61,6 @@
[malli.core :as mc]
[malli.transform :as mtx]
[medley.core :as m]
- ;; legacy usages -- do not use in new code
- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.schema :as mbql.s]
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
@@ -1020,11 +1018,7 @@
(let [tag (mbql-ref? mbql)
schema (case tag
:field-id ::mbql-3-field-id-ref
- :field [:multi
- {:dispatch #(and (vector? %)
- (map? (second %)))}
- [true :mbql.clause/field]
- [false ::mbql.s/field]] ; legacy MBQL clause
+ :field ::resolve/field-ref
:dimension ::lib.schema.parameter/dimension
:metric :mbql.clause/metric
:segment :mbql.clause/segment
diff --git a/src/metabase/models/serialization/resolve.clj b/src/metabase/models/serialization/resolve.clj
index 77d592cee92..3afee116f3b 100644
--- a/src/metabase/models/serialization/resolve.clj
+++ b/src/metabase/models/serialization/resolve.clj
@@ -222,14 +222,19 @@
(#{:field :dimension :metric :segment :measure} (keyword (first form))))
(keyword (first form))))
+(mr/def ::field-ref
+ "MBQL 5 or legacy `:field` clause. Registered under a keyword so [[lib/normalize]] reuses one cached coercer; an
+ inline literal with a fresh dispatch fn misses the registry cache on every call."
+ [:multi
+ {:dispatch #(and (vector? %)
+ (map? (second %)))}
+ [true :mbql.clause/field]
+ [false ::mbql.s/field]])
+
(defn- normalize [mbql]
(let [tag (mbql-clause-tag mbql)
schema (case tag
- :field [:multi
- {:dispatch #(and (vector? %)
- (map? (second %)))}
- [true :mbql.clause/field]
- [false ::mbql.s/field]] ; legacy MBQL clause
+ :field ::field-ref
:dimension ::lib.schema.parameter/dimension
:metric :mbql.clause/metric
:segment :mbql.clause/segment
diff --git a/test/metabase/models/serialization/resolve_test.clj b/test/metabase/models/serialization/resolve_test.clj
new file mode 100644
index 00000000000..da2dc545a37
--- /dev/null
+++ b/test/metabase/models/serialization/resolve_test.clj
@@ -0,0 +1,17 @@
+(ns metabase.models.serialization.resolve-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.lib.normalize :as lib.normalize]
+ [metabase.models.serialization.resolve :as resolve]
+ [metabase.util.malli.registry :as mr]))
+
+(deftest ^:parallel normalize-field-ref-reuses-cached-coercer-test
+ (testing "normalizing :field refs hits the registry coercer cache after the first call"
+ (let [misses (atom 0)]
+ (binding [mr/*cache-miss-hook* (fn [k _schema _value]
+ (when (= k ::lib.normalize/coercer)
+ (swap! misses inc)))]
+ (dotimes [_ 3]
+ (#'resolve/normalize [:field 1 nil])
+ (#'resolve/normalize [:field {:lib/uuid "00000000-0000-0000-0000-000000000001"} 1])))
+ (is (<= @misses 1)))))
diff --git a/test/metabase/models/serialization_test.clj b/test/metabase/models/serialization_test.clj
index a28ea8521b7..ba509a35d26 100644
--- a/test/metabase/models/serialization_test.clj
+++ b/test/metabase/models/serialization_test.clj
@@ -2,8 +2,10 @@
(:require
[clojure.test :refer :all]
[metabase.lib.core :as lib]
+ [metabase.lib.normalize :as lib.normalize]
[metabase.lib.test-metadata :as meta]
- [metabase.models.serialization :as serdes]))
+ [metabase.models.serialization :as serdes]
+ [metabase.util.malli.registry :as mr]))
(defn- fake-uuid
"Deterministic placeholder `:lib/uuid` for tests, e.g. `(fake-uuid 1)` => \"00000000-0000-0000-0000-000000000001\"."
@@ -183,6 +185,17 @@
["fk->" ["field-id" 1] ["field-id" 2]]
["fk->" [:field-id ["A" "B" "C" "D"]] [:field-id ["A" "B" "C" "D"]]]))))
+(deftest ^:parallel normalize-field-ref-reuses-cached-coercer-test
+ (testing "normalizing :field refs hits the registry coercer cache after the first call"
+ (let [misses (atom 0)]
+ (binding [mr/*cache-miss-hook* (fn [k _schema _value]
+ (when (= k ::lib.normalize/coercer)
+ (swap! misses inc)))]
+ (dotimes [_ 3]
+ (#'serdes/normalize-mbql-ref [:field 1 nil])
+ (#'serdes/normalize-mbql-ref [:field {:lib/uuid (fake-uuid 1)} 1])))
+ (is (<= @misses 1)))))
+
(deftest ^:parallel export-visualization-settings-test
(binding [serdes/*export-field-fk* (constantly ["A" "B" "C" "D"])
serdes/*export-fk* (fn [id model]