Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Incremental remote-sync import: load only changed entities on pull (GHY-3779) (#75897)

Eric Normandc70f62de5c0c
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index c7e6a2fb662..273ceace815 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3256,6 +3256,7 @@
events
models
premium-features
+ search
settings
startup
task
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index ffdfcd65226..104c645ff56 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -19,6 +19,7 @@
[metabase.collections.models.collection :as collection]
[metabase.events.core :as events]
[metabase.models.serialization :as serdes]
+ [metabase.search.core :as search]
[metabase.settings.core :as setting]
[metabase.util :as u]
[metabase.util.jvm :as u.jvm]
@@ -27,6 +28,8 @@
[toucan2.core :as t2])
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
+(set! *warn-on-reflection* true)
+
(defn- snapshot-has-transforms?
"Checks if the snapshot contains any Transform, PythonLibrary entities,
non-built-in TransformTags, or transforms-namespace collections.
@@ -166,24 +169,6 @@
:else
(format "Failed to reload from git repository: %s" (ex-message e))))
-(defn- handle-import-exception
- "Handles exceptions that occur during import by logging and returning an error status map.
-
- If the exception indicates cancellation (has :cancelled? in ex-data), logs an info message and returns nil.
- Otherwise logs the error, increments the failed imports analytics metric, and returns a map with :status :error,
- a user-friendly :message, the source :version, and error :details."
- [e snapshot]
- (if (:cancelled? (ex-data e))
- (log/info "Import from git repository was cancelled")
- (do
- (log/errorf e "Failed to reload from git repository: %s" (ex-message e))
- (analytics/inc! :metabase-remote-sync/imports-failed)
- {:status :error
- :message (source-error-message e)
- :version (source.p/version snapshot)
-
- :details {:error-type (type e)}})))
-
(defn- get-conflicts
"Detects conflicts that would prevent or complicate import. Returns a map with two classes:
- :first-import-conflicts - feature/namespace/library conflicts that only block on the very first import
@@ -300,6 +285,121 @@
(remote-sync.task/update-progress! task-id 0.95)
imported-data))
+;;; ------------------------------------------- Incremental Import Fast-Path -------------------------------------------
+
+(def ^:private incremental-not-possible
+ "Sentinel returned by [[incremental-load-snapshot!]] when a change can't be applied incrementally and the
+ caller must run the full [[load-snapshot!]] instead."
+ :remote-sync/incremental-import-not-possible)
+
+(def ^:private full-import-models
+ "Models whose change forces a full import on the incremental fast-path: Collection (a rename moves every
+ descendant's file, a delete cascades to its contents) and the feature models (their presence drives the
+ remote-sync-transforms / library settings, which need whole-snapshot knowledge to toggle correctly)."
+ #{"Collection" "Transform" "TransformTag" "PythonLibrary" "NativeQuerySnippet"})
+
+(defn- legal-yaml-path?
+ "True for a managed-directory `.yaml` entity file — the only changed paths the importer acts on."
+ [^String path]
+ (and (str/ends-with? path ".yaml")
+ (not (str/starts-with? path "."))
+ (when-let [i (str/index-of path "/")]
+ (contains? serialization/legal-top-level-paths (subs path 0 i)))))
+
+(defn- rso-where-for
+ "HoneySQL :where matching the RemoteSyncObject rows for the given `[{:model_type :model_id}]` maps."
+ [rows]
+ (into [:or] (map (fn [{:keys [model_type model_id]}]
+ [:and [:= :model_type model_type] [:= :model_id model_id]]))
+ rows))
+
+(defn- incremental-import-plan
+ "What an incremental load would touch, or [[incremental-not-possible]] if the change must fall back to a full
+ import. On success returns `{:ingestable <ingestable-or-nil> :deleted-rsos <RemoteSyncObject rows>}`. Falls back
+ when there is no diff, a deleted file can't be mapped back to a tracked entity, or a
+ structural/feature/non-entity-id model changed."
+ [snapshot last-version]
+ (let [changed (when last-version (source.p/changed-files snapshot last-version))
+ add-mod (into #{} (filter legal-yaml-path?) (into (:added changed) (:modified changed)))
+ deleted (into #{} (filter legal-yaml-path?) (:deleted changed))
+ ;; N+1: one query per deleted path (bounded by deletions in a single pull; left un-batched because
+ ;; file_path values are long, making an IN clause bulky for marginal gain).
+ deleted-rsos (mapv (fn [p] (t2/select-one :model/RemoteSyncObject :file_path p)) deleted)
+ ingestable (when (seq add-mod)
+ (source.p/->ingestable snapshot {:path-filters (mapv #(re-pattern (java.util.regex.Pattern/quote %)) add-mod)}))
+ add-models (if ingestable
+ (into #{} (map #(:model (last %))) (serialization/ingest-list ingestable))
+ #{})
+ all-models (into add-models (map :model_type) deleted-rsos)]
+ (cond
+ (nil? changed) ;; first import, or diffing not possible (force-push/rebase, non-diffable source)
+ incremental-not-possible
+
+ (some nil? deleted-rsos) ;; some deleted file not present in appdb
+ incremental-not-possible
+
+ (some full-import-models all-models) ;; anything not incrementally importable by model type?
+ incremental-not-possible
+
+ (some #(not= :entity-id (:identity (spec/spec-for-model-type %))) all-models) ;; anything not entity-id model?
+ incremental-not-possible
+
+ :else
+ {:ingestable ingestable :deleted-rsos deleted-rsos})))
+
+(defn- incremental-load-snapshot!
+ "Applies an incremental `plan` from [[incremental-import-plan]]: loads only its added/modified entities,
+ deletes only those genuinely removed, and reconciles just those rows of the RemoteSyncObject table —
+ leaving everything else untouched. Runs `finalize!` inside the reconcile transaction, then logs success and
+ returns [[import!]]'s `:success` result map carrying `snapshot-version`. The caller decides whether an
+ incremental load is safe (see [[incremental-import-plan]] and [[import!]]); this assumes the plan is valid
+ and local state matches the diff base.
+
+ Renames are handled by entity identity, not path: a rename re-loads the same entity_id at the new path
+ (an add), so the old path's delete is recognized as a rename and the entity is not removed."
+ [{:keys [ingestable deleted-rsos] :as _plan} snapshot-version task-id sync-timestamp & {:keys [finalize!]}]
+ (let [load-result (when ingestable
+ (serdes/with-cache
+ (serialization/load-metabase!
+ (source.ingestable/wrap-progress-ingestable task-id 0.7 ingestable)
+ :backfill? false :reindex? false)))
+ imported-data (spec/extract-imported-entities (:seen load-result))
+ loaded-eid? (fn [model-type eid]
+ ;; by-entity-id holds sets of raw entity_id strings, keyed by model type
+ (contains? (get-in imported-data [:by-entity-id model-type]) eid))
+ ;; A tracked entity whose entity_id was NOT re-loaded is a genuine delete; if it WAS
+ ;; re-loaded (at a new path) it's a rename, so we keep it.
+ deletes (for [{:keys [model_type model_id]} deleted-rsos
+ ;; N+1: one entity_id query per delete candidate (bounded by deletions in a single
+ ;; pull; batching would need grouping by model-key into per-table IN queries).
+ :let [model-key (:model-key (spec/spec-for-model-type model_type))
+ eid (when model-key (t2/select-one-fn :entity_id model-key :id model_id))]
+ :when (not (loaded-eid? model_type eid))]
+ {:model_type model_type :model_id model_id :model-key model-key})
+ sync-rows (spec/sync-all-entities! sync-timestamp imported-data)]
+ (remote-sync.task/update-progress! task-id 0.8)
+ (t2/with-transaction [_conn]
+ (doseq [[model-key ds] (group-by :model-key deletes)]
+ (t2/delete! model-key :id [:in (mapv :model_id ds)]))
+ (when (seq deletes)
+ (t2/delete! :model/RemoteSyncObject {:where (rso-where-for deletes)}))
+ (when (seq sync-rows)
+ (t2/delete! :model/RemoteSyncObject {:where (rso-where-for sync-rows)})
+ (t2/insert! :model/RemoteSyncObject sync-rows))
+ (when ingestable
+ (record-exported-paths! (source.ingestable/cached-file-paths ingestable)))
+ (when finalize! (finalize!)))
+ ;; We skip the whole-appdb reindex the full load runs. Added/modified entities are already
+ ;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index
+ ;; after-insert/after-update hooks. Deletes have no such hook, so remove them explicitly.
+ (doseq [[model-key ds] (group-by :model-key deletes)]
+ (search/delete! model-key (mapv :model_id ds)))
+ (remote-sync.task/update-progress! task-id 0.95)
+ (log/info "Successfully reloaded entities from git repository")
+ {:status :success
+ :version snapshot-version
+ :message "Successfully reloaded from git repository"}))
+
(defn- capture-dirty-objects
"Returns the current non-synced RemoteSyncObject rows — the local changes that have not been pushed.
Captured before a local-only merge so they can be restored afterwards (see [[import-merged!]])."
@@ -330,52 +430,54 @@
orphaned by a force-push/rebase) — that surfaces as a conflict. When the remote has not advanced past
the base, there is nothing to fold in, so it is a no-op success that keeps local changes dirty."
[snapshot base-snapshot task-id sync-timestamp]
- (cond
- (nil? base-snapshot)
- {:status :conflict
- :version (source.p/version snapshot)
- :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
- :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}
-
- ;; The remote hasn't advanced past the merge base — nothing to fold in. Keep local changes as-is (still
- ;; dirty, to be pushed later); a no-op success, not a spurious "history was rewritten" conflict.
- (= (source.p/version base-snapshot) (source.p/version snapshot))
- (u/prog1 {:status :success
- :version (source.p/version snapshot)
- :merge-summary {:added 0 :updated 0 :removed 0}}
- (log/info "Pull merge: remote has not advanced; keeping local changes unchanged"))
-
- :else
- (let [models (spec/extract-entities-for-export)
- {:keys [merged conflicts summary]}
- (source/compute-merge models snapshot base-snapshot task-id)]
- (if (seq conflicts)
- (u/prog1 {:status :conflict
- :version (source.p/version snapshot)
- :conflicts (mapv remote-sync.merge/conflict-label conflicts)
- :message "Import blocked: the same content was changed both locally and on the remote branch."}
- (log/infof "Pull merge conflict on %d entit(ies): %s"
- (count conflicts) (str/join ", " (:conflicts <>))))
- ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are
- ;; disjoint from the remote changes, so restoring them reproduces exactly the local diff vs remote.
- ;; Restore + set-version run inside the load's transaction so that a crash can't leave the dirty
- ;; markers overwritten by the load (which would lose them — a retry captures nothing).
- (let [dirty-objects (capture-dirty-objects)]
- (load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp
- :finalize! (fn []
- (restore-dirty-objects! dirty-objects sync-timestamp)
- (remote-sync.task/set-version! task-id (source.p/version snapshot))))
- (log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"
- (apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)
- (count dirty-objects))
- {:status :success
- :version (source.p/version snapshot)
- :merge-summary summary})))))
+ (let [merge-result (delay (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id))]
+ (cond
+ (nil? base-snapshot)
+ {:status :conflict
+ :version (source.p/version snapshot)
+ :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
+ :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}
+
+ ;; The remote hasn't advanced past the merge base — nothing to fold in. Keep local changes as-is (still
+ ;; dirty, to be pushed later); a no-op success, not a spurious "history was rewritten" conflict.
+ (= (source.p/version base-snapshot) (source.p/version snapshot))
+ (do
+ (log/info "Pull merge: remote has not advanced; keeping local changes unchanged")
+ {:status :success
+ :version (source.p/version snapshot)
+ :merge-summary {:added 0 :updated 0 :removed 0}})
+
+ (seq (:conflicts @merge-result))
+ (let [conflicts (mapv remote-sync.merge/conflict-label (:conflicts @merge-result))]
+ (log/infof "Pull merge conflict on %d entit(ies): %s"
+ (count conflicts) (str/join ", " conflicts))
+ {:status :conflict
+ :version (source.p/version snapshot)
+ :conflicts conflicts
+ :message "Import blocked: the same content was changed both locally and on the remote branch."})
+
+ :else
+ ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are
+ ;; disjoint from the remote changes, so restoring them reproduces exactly the local diff vs remote.
+ ;; Restore + set-version run inside the load's transaction so that a crash can't leave the dirty
+ ;; markers overwritten by the load (which would lose them — a retry captures nothing).
+ (let [{:keys [merged summary]} @merge-result
+ dirty-objects (capture-dirty-objects)]
+ (load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp
+ :finalize! (fn []
+ (restore-dirty-objects! dirty-objects sync-timestamp)
+ (remote-sync.task/set-version! task-id (source.p/version snapshot))))
+ (log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"
+ (apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)
+ (count dirty-objects))
+ {:status :success
+ :version (source.p/version snapshot)
+ :merge-summary summary}))))
(defn import!
"Imports and reloads Metabase entities from a remote snapshot.
- Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
+ Takes a non-nil SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
- :force? - forces import even when the snapshot version matches the last imported version or has first-import conflicts
- :force-deletion? - forces past deletion conflicts (unsynced local transforms the import would delete); defaults
to :force?. Kept separate so the setup/enable flow can force past version/dirty guards (force? true) while still
@@ -401,51 +503,77 @@
(log/info "Reloading remote entities from the remote source")
(analytics/inc! :metabase-remote-sync/imports)
(let [sync-timestamp (t/instant)]
- (if snapshot
- (try
- (if merge?
+ (try
+ (let [snapshot-version (source.p/version snapshot)
+ last-imported-version (remote-sync.task/last-version)
+ first-import? (nil? last-imported-version)
+ ;; force-deletion? defaults to force? when a caller doesn't pass it.
+ force-deletion? (if (nil? force-deletion?) force? force-deletion?)
+ finalize! #(remote-sync.task/set-version! task-id snapshot-version)
+ path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
+ ;; First-import conflicts only block the first import; deletion conflicts block every import (an
+ ;; already-configured instance must not silently delete unsynced transforms). The get-conflicts tree scan
+ ;; is itself deferred so it only runs when one of the gates is open (e.g. a forced import with
+ ;; force-deletion? true skips it entirely).
+ blocking-conflicts (delay
+ (let [conflicts (delay (get-conflicts (source.p/->ingestable snapshot {:path-filters path-filters})
+ first-import?))]
+ (cond-> []
+ (and first-import? (not force?))
+ (into (:first-import-conflicts @conflicts))
+
+ (not force-deletion?)
+ (into (:deletion-conflicts @conflicts)))))
+ incremental-plan (delay (incremental-import-plan snapshot last-imported-version))]
+ (cond
+ merge?
(import-merged! snapshot base-snapshot task-id sync-timestamp)
- (let [snapshot-version (source.p/version snapshot)
- last-imported-version (remote-sync.task/last-version)
- first-import? (nil? last-imported-version)
- ;; force-deletion? is optional; when a caller doesn't pass it, deletion follows force?
- force-deletion? (if (nil? force-deletion?) force? force-deletion?)
- path-filters (mapv #(re-pattern (str % "/.*")) serialization/legal-top-level-paths)
- base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})
- {:keys [first-import-conflicts deletion-conflicts]} (get-conflicts base-ingestable first-import?)
- ;; First-import conflicts only block on the first import; deletion conflicts block on every
- ;; import (an already-configured instance must not silently delete unsynced transforms).
- blocking-conflicts (vec (concat (when (and first-import? (not force?)) first-import-conflicts)
- (when (not force-deletion?) deletion-conflicts)))]
- (cond
- (seq blocking-conflicts)
- (u/prog1 {:status :conflict
- :version (source.p/version snapshot)
- :conflicts (into #{} (map :category) blocking-conflicts) ; Backward compat: set of category names
- :conflict-details blocking-conflicts ; Detailed conflict info
- :message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}
- (log/infof (:message <>)))
-
- (and (not force?) (= last-imported-version snapshot-version))
- (u/prog1 {:status :success
- :version (source.p/version snapshot)
- :message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}
- (log/infof (:message <>)))
- :else
- (do
- (load-snapshot! snapshot task-id sync-timestamp
- :finalize! #(remote-sync.task/set-version! task-id (source.p/version snapshot)))
- (log/info "Successfully reloaded entities from git repository")
- {:status :success
- :version (source.p/version snapshot)
- :message "Successfully reloaded from git repository"}))))
- (catch Exception e
- (handle-import-exception e snapshot))
- (finally
- (analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))
- {:status :error
- :message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."})))
+ ;; Cheap no-op pull: nothing changed remotely, so nothing is loaded or deleted
+ (and (not force?) (= last-imported-version snapshot-version))
+ (let [message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)]
+ (log/info message)
+ {:status :success
+ :version snapshot-version
+ :message message})
+
+ ;; Incremental fast-path: not forced, no local drift, and the change is incrementally loadable.
+ ;; Touches only changed files, so it can't wholesale-delete unsynced transforms — no conflict scan,
+ ;; and only the cheap O(changes) diff is computed (never the full-tree get-conflicts read).
+ (and (not force?) (not (remote-sync.object/dirty?)) (not= incremental-not-possible @incremental-plan))
+ (incremental-load-snapshot! @incremental-plan snapshot-version task-id sync-timestamp :finalize! finalize!)
+
+ (seq @blocking-conflicts)
+ (let [message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)]
+ (log/info message)
+ {:status :conflict
+ :version snapshot-version
+ :conflicts (into #{} (map :category) @blocking-conflicts) ; Backward compat: set of category names
+ :conflict-details @blocking-conflicts
+ :message message})
+
+ ;; No blocking conflicts: do the full reload.
+ :else
+ (do
+ (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)
+ (log/info "Successfully reloaded entities from git repository")
+ {:status :success
+ :version snapshot-version
+ :message "Successfully reloaded from git repository"})))
+ (catch Exception e
+ ;; A cancellation isn't a failure: log it and return nil. Otherwise log the error, count it, and
+ ;; return a user-friendly :error result.
+ (if (:cancelled? (ex-data e))
+ (log/info "Import from git repository was cancelled")
+ (do
+ (log/errorf e "Failed to reload from git repository: %s" (ex-message e))
+ (analytics/inc! :metabase-remote-sync/imports-failed)
+ {:status :error
+ :message (source-error-message e)
+ :version (source.p/version snapshot)
+ :details {:error-type (type e)}})))
+ (finally
+ (analytics/observe! :metabase-remote-sync/import-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))
(defn- export-merged!
"Export path taken when the remote branch has advanced beyond the last synced version (`base-snapshot`
@@ -459,13 +587,14 @@
(source/merge-and-store! models snapshot base-snapshot task-id message)]
(case status
:conflict
- (u/prog1 {:status :conflict
- :version (source.p/version snapshot)
- :conflicts (mapv remote-sync.merge/conflict-label conflicts)
- :merge-summary summary
- :message "Export blocked: the same content was changed both locally and on the remote branch."}
+ (let [conflicts (mapv remote-sync.merge/conflict-label conflicts)]
(log/infof "Export merge conflict on %d entit(ies): %s"
- (count conflicts) (str/join ", " (:conflicts <>))))
+ (count conflicts) (str/join ", " conflicts))
+ {:status :conflict
+ :version (source.p/version snapshot)
+ :conflicts conflicts
+ :merge-summary summary
+ :message "Export blocked: the same content was changed both locally and on the remote branch."})
:success
;; Fold-in pull: the merge brought remote changes that aren't in the local app DB yet. Load the
@@ -1151,5 +1280,6 @@
;; force? true bypasses the version/dirty guards for setup, but force-deletion? false keeps unsynced
;; local transforms from being silently destroyed — they surface as a conflict instead (GHY-3900).
(:id (async-import! (settings/remote-sync-branch) true {} :force-deletion? false))))
- (u/prog1 nil
- (collection/clear-remote-synced-collection!))))
+ (do
+ (collection/clear-remote-synced-collection!)
+ nil)))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index ca32b5cbaaa..e453ef46cba 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -16,11 +16,12 @@
(org.eclipse.jgit.api Git GitCommand TransportCommand)
(org.eclipse.jgit.dircache DirCache DirCacheEditor$DeletePath DirCacheEditor$DeleteTree
DirCacheEditor$PathEdit DirCacheEntry)
- (org.eclipse.jgit.lib CommitBuilder Constants FileMode PersonIdent Ref)
+ (org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)
(org.eclipse.jgit.revwalk RevCommit RevWalk)
(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate
RemoteRefUpdate$Status UsernamePasswordCredentialsProvider)
- (org.eclipse.jgit.treewalk TreeWalk)))
+ (org.eclipse.jgit.treewalk TreeWalk)
+ (org.eclipse.jgit.treewalk.filter TreeFilter)))
(set! *warn-on-reflection* true)
@@ -217,6 +218,38 @@
(let [loader (.open repo object-id)]
(String. (.getBytes loader) "UTF-8")))))
+(defn changed-files
+ "Paths whose blob differs between commit `from-version` and this snapshot's `version`, classified into
+ `{:added #{} :modified #{} :deleted #{}}`. jgit prunes unchanged subtrees as it walks, so the cost is
+ proportional to the number of changed entries, not the size of the tree.
+
+ Takes a GitSnapshot (:git instance and current :version) and a `from-version` commit-ish to diff against.
+
+ Returns nil when `from-version` cannot be resolved or is no longer present in the local object store
+ (e.g. orphaned by a force-push or rebase), signalling the caller to fall back to a full import."
+ [{:keys [^Git git ^String version]} ^String from-version]
+ (let [^Repository repo (.getRepository git)
+ objects (.getObjectDatabase repo)
+ old-id (.resolve repo from-version)
+ new-id (.resolve repo version)]
+ (when (and old-id new-id (.has objects old-id) (.has objects new-id))
+ (with-open [rw (RevWalk. repo)
+ ^TreeWalk tw (TreeWalk. repo)]
+ (.addTree tw (.getTree (.parseCommit rw old-id)))
+ (.addTree tw (.getTree (.parseCommit rw new-id)))
+ (.setRecursive tw true)
+ (.setFilter tw TreeFilter/ANY_DIFF)
+ (let [zero (ObjectId/zeroId)]
+ (loop [acc {:added #{} :modified #{} :deleted #{}}]
+ (if (.next tw)
+ (let [in-old? (not (.equals zero (.getObjectId tw 0)))
+ in-new? (not (.equals zero (.getObjectId tw 1)))
+ bucket (cond (not in-old?) :added
+ (not in-new?) :deleted
+ :else :modified)]
+ (recur (update acc bucket conj (.getPathString tw))))
+ acc)))))))
+
(defn push-branch!
"Pushes a local branch to the remote repository.
@@ -403,7 +436,11 @@
(apply-changes! this message upserts delete-paths))
(version [this]
- (:version this)))
+ (:version this))
+
+ source.p/Diffable
+ (changed-files* [this from-version]
+ (changed-files this from-version)))
(def ^:private jgit (atom {}))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index aed07ef26ed..f035a58d8a5 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -84,6 +84,23 @@
Returns a version identifier string (e.g., a git SHA)."))
+(defprotocol Diffable
+ "Optional capability for snapshots that can cheaply report which files changed since a prior version.
+ Implemented by sources backed by real history (git, in-memory versioned test sources); snapshots that
+ can't diff simply don't implement it and `changed-files` returns nil for them."
+ (changed-files* [snapshot from-version]
+ "Returns `{:added #{} :modified #{} :deleted #{}}` path sets between `from-version` and this
+ snapshot's version, or nil when `from-version` can't be resolved. Prefer calling [[changed-files]]."))
+
+(defn changed-files
+ "Paths that changed between `from-version` and `snapshot`'s version, as
+ `{:added #{} :modified #{} :deleted #{}}`, or nil when an incremental diff isn't available — the base
+ can't be resolved (force-push/rebase), or the snapshot's source type doesn't support diffing. A nil
+ result means diffing is not possible."
+ [snapshot from-version]
+ (when (satisfies? Diffable snapshot)
+ (changed-files* snapshot from-version)))
+
(methodical/defmulti ->ingestable
"Creates an ingestable snapshot for remote sync operations.
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index ee7ed3b801a..f7e853d41bd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -30,13 +30,6 @@
;; import! tests
-(deftest import!-with-no-source-configured-test
- (testing "import! with no snapshot configured"
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
- result (impl/import! nil task-id)]
- (is (= :error (:status result)))
- (is (re-find #"Remote sync source is not enabled" (:message result))))))
-
(deftest import!-successful-without-collections-test
(testing "import! successful without collections (imports all remote-synced)"
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
index 4a7ad72c3ba..7e2538b03cd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
@@ -16,7 +16,7 @@
[toucan2.core :as t2]))
(use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each rs.test/clean-remote-sync-state)
+(use-fixtures :each rs.test/clean-remote-sync-state rs.test/commit-with-temp)
(defn- venues-query []
{:database (mt/id) :type :query :query {:source-table (mt/id :venues)}})
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
new file mode 100644
index 00000000000..8ade5adf547
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
@@ -0,0 +1,279 @@
+(ns metabase-enterprise.remote-sync.incremental-import-test
+ "Differential equivalence harness for incremental import/pull (GHY-3779).
+
+ The full reload (`load-snapshot!`) is the specification: a correct incremental import must leave the app
+ DB, the RemoteSyncObject table, the remote-sync-transforms setting, and the version pointer in exactly
+ the state a full import would. So rather than assert against hand-computed expectations, we run BOTH
+ paths against the same change and assert their resulting state is identical.
+
+ `import!` already gives us the seam: `:force? true` always takes the full path (the oracle), while
+ `:force? false` is the path under test (full today; incremental once GHY-3779 lands). The harness:
+
+ import(v0, force) ; establish local == v0
+ import(v1, force) ; ORACLE: capture full-import state of v1
+ import(v0, force) ; reset local back to v0
+ import(v1, NO force) ; UNDER TEST: capture state of v1
+ assert the two states are equal
+
+ Until incremental lands, `force? false` == `force? true`, so these tests validate the harness and the
+ oracle itself (full import is a deterministic function of the snapshot, and the state comparison is
+ pk-independent and stable). When incremental lands behind `force? false`, the same tests become the
+ equivalence proof."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
+ [metabase-enterprise.remote-sync.settings :as settings]
+ [metabase-enterprise.remote-sync.source :as source]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.spec :as spec]
+ [metabase-enterprise.remote-sync.test-helpers :as rs.test]
+ [metabase.search.core :as search]
+ [metabase.search.test-util :as search.tu]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+(use-fixtures :each rs.test/clean-remote-sync-state rs.test/commit-with-temp)
+
+;;; ------------------------------------------------- State capture --------------------------------------------------
+
+(defn- synced-tree
+ "The current remote-synced set serialized to a {path content} map — what a fresh export would write.
+ After a correct import this equals the imported snapshot's content (round-trip), so it's a faithful,
+ pk-independent fingerprint of the loaded app-DB entities."
+ []
+ (into {} (map (juxt :path :content)) (source/serialize-specs (spec/extract-entities-for-export) nil)))
+
+(defn- rso-state
+ "The RemoteSyncObject table as a pk-independent set of [model_type file_path status]. Keyed on file_path
+ (derived from names, stable across delete+recreate) rather than the autoincrement model_id, so the same
+ logical entity compares equal even if its primary key changed between runs."
+ []
+ (into #{}
+ (map (juxt :model_type :file_path :status))
+ (t2/select :model/RemoteSyncObject)))
+
+(defn- state-vector
+ "Everything an import is responsible for reconciling. Two imports of the same snapshot must produce equal
+ state vectors."
+ []
+ {:files (synced-tree)
+ :rso (rso-state)
+ :transforms (settings/remote-sync-transforms)
+ :version (remote-sync.task/last-version)})
+
+(defn- assert-equivalent [oracle under-test]
+ (is (= (:files oracle) (:files under-test))
+ "synced app-DB entities serialize identically")
+ (is (= (:rso oracle) (:rso under-test))
+ "RemoteSyncObject table matches")
+ (is (= (:transforms oracle) (:transforms under-test))
+ "remote-sync-transforms setting matches")
+ (is (= (:version oracle) (:version under-test))
+ "version pointer matches"))
+
+;;; --------------------------------------------------- Harness ------------------------------------------------------
+
+(defn- import-at!
+ "Runs `import!` against the source's snapshot at `version`, then completes the task (so `last-version`
+ picks up its version for the next import). Returns the import result."
+ [src version & {:keys [force?] :or {force? false}}]
+ (let [task (t2/insert-returning-pk! :model/RemoteSyncTask
+ {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ result (impl/import! (source.p/snapshot-at src version) task :force? force?)]
+ (impl/handle-task-result! result task)
+ result))
+
+(defn- import-v1-under-test!
+ "Runs the under-test import of v1 (force? false), spying on incremental-load-snapshot! to report which
+ path it took. Returns [result path], where path is :incremental (the fast-path ran) or :fallback (it
+ declined and import! ran the full load-snapshot!)."
+ [src]
+ (let [real @#'impl/incremental-load-snapshot!
+ path (atom :fallback)]
+ (with-redefs [impl/incremental-load-snapshot!
+ (fn [& args] (let [r (apply real args)]
+ (reset! path (if (= r @#'impl/incremental-not-possible) :fallback :incremental))
+ r))]
+ [(import-at! src "v1" :force? false) @path])))
+
+(defn- run-differential!
+ "Imports `f0` as the baseline, then runs the full (oracle) and under-test imports of `f1` and asserts the
+ resulting state is identical. `f0`/`f1` are {path content} trees. Returns the path the under-test import
+ took (:incremental or :fallback) so the caller can assert the v1 scope boundary."
+ [f0 f1]
+ ;; These scenarios assert DB/RSO/setting/version equivalence, not search — disable index maintenance so
+ ;; their (async) search ingestion can't bleed into the search-index integration test in this namespace.
+ (search.tu/with-index-disabled
+ (let [src (rs.test/versioned-source :trees {"v0" f0 "v1" f1} :current "v0")]
+ (is (= :success (:status (import-at! src "v0" :force? true))) "baseline import of v0 succeeds")
+ (is (= :success (:status (import-at! src "v1" :force? true))) "oracle full import of v1 succeeds")
+ (let [oracle (state-vector)]
+ (is (= :success (:status (import-at! src "v0" :force? true))) "reset back to v0 succeeds")
+ (let [[result path] (import-v1-under-test! src)]
+ (is (= :success (:status result)) "under-test import of v1 succeeds")
+ (assert-equivalent oracle (state-vector))
+ path)))))
+
+(defn- do-with-bench!
+ "Sets up a remote-synced `Bench` collection with cards A and B, then calls `f` with the V0 tree
+ ({path content}; the two cards' paths carry `card_a` / `card_b` slugs). Cleans up any entities
+ recreated during the differential churn (delete+recreate gives fresh primary keys)."
+ [f]
+ ;; with-temp commits (not rollback) via the `commit-with-temp` :each fixture, so the import's progress
+ ;; updates on a separate connection don't block on MySQL — see that fixture's docstring.
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Bench" :is_remote_synced true :location "/"}
+ :model/Card _a {:name "Card A" :collection_id coll-id}
+ :model/Card _b {:name "Card B" :collection_id coll-id}]
+ (mt/with-model-cleanup [:model/Card :model/Collection]
+ (f (synced-tree))))))
+
+(defn- path-with [tree slug]
+ (some (fn [p] (when (str/includes? p slug) p)) (keys tree)))
+
+;;; --------------------------------------------------- Scenarios ----------------------------------------------------
+
+(deftest no-op-pull-equivalence-test
+ (testing "GHY-3779: a pull whose content is unchanged (only the version advanced) reconciles to the same
+ state via the full and under-test paths"
+ (do-with-bench!
+ (fn [f0]
+ ;; identical content under a new version — exercises the empty-diff fast path
+ (is (= :incremental (run-differential! f0 f0)))))))
+
+(deftest edit-card-equivalence-test
+ (testing "GHY-3779: editing a single card's content imports equivalently full vs. under-test"
+ (do-with-bench!
+ (fn [f0]
+ (let [b-path (path-with f0 "card_b")
+ f1 (update f0 b-path str/replace "display: table" "display: line")]
+ (is (= :incremental (run-differential! f0 f1))))))))
+
+(deftest edit-multiple-cards-equivalence-test
+ (testing "GHY-3779: editing several cards in one pull loads just those cards, equivalently to a full import"
+ (do-with-bench!
+ (fn [f0]
+ (let [f1 (-> f0
+ (update (path-with f0 "card_a") str/replace "display: table" "display: line")
+ (update (path-with f0 "card_b") str/replace "display: table" "display: bar"))]
+ (is (= :incremental (run-differential! f0 f1))))))))
+
+(deftest add-card-equivalence-test
+ (testing "GHY-3779: adding a new card imports equivalently full vs. under-test"
+ (do-with-bench!
+ (fn [f0]
+ (let [b-path (path-with f0 "card_b")
+ b-yaml (get f0 b-path)
+ b-eid (second (re-find #"entity_id: (\S+)" b-yaml))
+ c-yaml (-> b-yaml
+ (str/replace b-eid "Cnewcardnewcardnewca")
+ (str/replace "name: Card B" "name: Card C")
+ (str/replace "label: card_b" "label: card_c"))
+ c-path (str/replace b-path "card_b" "card_c")
+ f1 (assoc f0 c-path c-yaml)]
+ (is (= :incremental (run-differential! f0 f1))))))))
+
+(deftest delete-card-equivalence-test
+ (testing "GHY-3779: deleting a card remotely imports equivalently full vs. under-test"
+ (do-with-bench!
+ (fn [f0]
+ (let [b-path (path-with f0 "card_b")
+ f1 (dissoc f0 b-path)]
+ (is (= :incremental (run-differential! f0 f1))))))))
+
+(deftest incremental-search-update-only-changed-test
+ (testing "GHY-3779: an incremental edit re-indexes only the changed entity (we skip the full reindex),
+ keeping search-index cost proportional to the change"
+ ;; Index maintenance is disabled so the (un-stubbed) baseline reindex can't bleed into the
+ ;; search-index integration test; the with-redefs recorder still captures the hook's update! call.
+ (search.tu/with-index-disabled
+ (do-with-bench!
+ (fn [f0]
+ (let [a-eid (second (re-find #"entity_id: (\S+)" (get f0 (path-with f0 "card_a"))))
+ b-path (path-with f0 "card_b")
+ b-eid (second (re-find #"entity_id: (\S+)" (get f0 b-path)))
+ f1 (update f0 b-path str/replace "display: table" "display: line")
+ src (rs.test/versioned-source :trees {"v0" f0 "v1" f1} :current "v0")
+ indexed (atom #{})]
+ (import-at! src "v0" :force? true) ; baseline (full) — local == v0
+ (with-redefs [search/update! (fn [inst] (swap! indexed conj (:entity_id inst)))
+ search/delete! (fn [& _] nil)]
+ (import-at! src "v1")) ; incremental edit of card_b only
+ (is (contains? @indexed b-eid) "the edited card is re-indexed")
+ (is (not (contains? @indexed a-eid)) "the unchanged card is NOT re-indexed")))))))
+
+(deftest incremental-delete-removes-from-search-index-test
+ (testing "GHY-3779: a remote delete must explicitly remove the entity from the search index — unlike
+ updates (handled by the load's after-insert/after-update hooks), there is no after-delete
+ hook, so the incremental path calls search/delete! for each removed entity."
+ ;; Asserted at the call level (search/delete! invoked with the right model + id); the live appdb index
+ ;; isn't queried here because its tracking state is order-sensitive across tests in a shared JVM.
+ (search.tu/with-index-disabled
+ (do-with-bench!
+ (fn [f0]
+ (let [b-path (path-with f0 "card_b")
+ b-eid (second (re-find #"entity_id: (\S+)" (get f0 b-path)))
+ f1 (dissoc f0 b-path)
+ src (rs.test/versioned-source :trees {"v0" f0 "v1" f1} :current "v0")
+ deleted (atom [])]
+ (import-at! src "v0" :force? true) ; baseline (full) — local == v0
+ (let [b-id (t2/select-one-pk :model/Card :entity_id b-eid)]
+ (with-redefs [search/delete! (fn [model ids] (swap! deleted conj [model (vec ids)]))]
+ (import-at! src "v1")) ; incremental delete of card_b
+ (is (some (fn [[model ids]] (and (= :model/Card model) (some #{b-id} ids))) @deleted)
+ "the removed card is deleted from the search index by id"))))))))
+
+(deftest rename-card-equivalence-test
+ (testing "GHY-3779: renaming a card (same entity_id at a new path) imports equivalently — the old
+ path's delete must be recognized as a rename, not remove the entity"
+ (do-with-bench!
+ (fn [f0]
+ (let [b-path (path-with f0 "card_b")
+ ;; same entity_id, new name → new slug/path: a rename shows up as delete(old) + add(new)
+ renamed (-> (get f0 b-path)
+ (str/replace "name: Card B" "name: Card B Renamed")
+ (str/replace "label: card_b" "label: card_b_renamed"))
+ f1 (-> f0 (dissoc b-path) (assoc (str/replace b-path "card_b" "card_b_renamed") renamed))]
+ (is (= :incremental (run-differential! f0 f1))))))))
+
+;;; --------------------------------------- Structural changes: fall back to full ---------------------------------------
+
+(deftest collection-rename-falls-back-test
+ (testing "GHY-3779: a Collection change falls back to the full import (a rename moves descendant paths),
+ and still reconciles to the same state as the full oracle"
+ (do-with-bench!
+ (fn [f0]
+ (let [c-path (path-with f0 "bench.yaml")
+ f1 (update f0 c-path str/replace "name: Bench" "name: Workbench")]
+ (is (= :fallback (run-differential! f0 f1))))))))
+
+(deftest collection-delete-with-contents-falls-back-test
+ (testing "GHY-3779: deleting a collection and its contents falls back to the full import (avoids the
+ cascade trap), and still reconciles to the same state as the full oracle"
+ (do-with-bench!
+ (fn [f0]
+ ;; remove the whole bench subtree (collection + both cards)
+ (is (= :fallback (run-differential! f0 {})))))))
+
+;;; ------------------------------------------ First import: never incremental ------------------------------------------
+
+(deftest first-import-no-force-uses-full-load-test
+ (testing "GHY-3779: a first import (no prior version, so last-version is nil) with force? false must NOT
+ attempt the incremental fast-path. incremental-load-snapshot! assumes local state equals
+ last-version, which cannot hold on a first import — driving it with a nil base diffs against an
+ unresolvable version (a real git source NPEs on resolve(nil)). The first import must take the
+ full load-snapshot! and succeed."
+ (do-with-bench!
+ (fn [f0]
+ (let [src (rs.test/versioned-source :trees {"v0" f0} :current "v0")
+ real @#'impl/incremental-load-snapshot!
+ called? (atom false)]
+ (with-redefs [impl/incremental-load-snapshot! (fn [& args] (reset! called? true) (apply real args))]
+ ;; no baseline import has run, so this is the first import: first-import? is true
+ (is (= :success (:status (import-at! src "v0" :force? false))))
+ (is (not @called?)
+ "incremental-load-snapshot! must not run on a first import")))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index d4054d6057c..69cc8774109 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -326,6 +326,36 @@
(is (= "new" (source.p/read-file snap "collections/a/new.yaml")))
(is (= "keep" (source.p/read-file snap "collections/a/keep.yaml"))))))))
+(deftest changed-files-test
+ (testing "changed-files classifies the paths whose blob differs between two commits"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"keep.txt" "unchanged"
+ "edit.txt" "old content"
+ "remove.txt" "delete me"
+ "deep/nested/keep.txt" "deep unchanged"})
+ from-version (:version (source.p/snapshot master))]
+ (source.p/apply-changes! (source.p/snapshot master) "Change set"
+ [{:path "edit.txt" :content "new content"}
+ {:path "add.txt" :content "brand new"}]
+ ["remove.txt"])
+ (let [snap (source.p/snapshot master)]
+ (testing "added / modified / deleted are reported in their own buckets"
+ (is (= {:added #{"add.txt"}
+ :modified #{"edit.txt"}
+ :deleted #{"remove.txt"}}
+ (git/changed-files snap from-version))))
+ (testing "unchanged files — including deep untouched subtrees — are not reported"
+ (let [{:keys [added modified deleted]} (git/changed-files snap from-version)
+ touched (reduce into #{} [added modified deleted])]
+ (is (not (contains? touched "keep.txt")))
+ (is (not (contains? touched "deep/nested/keep.txt")))))
+ (testing "comparing a version against itself reports no changes"
+ (is (= {:added #{} :modified #{} :deleted #{}}
+ (git/changed-files snap (:version snap)))))
+ (testing "an unresolvable from-version returns nil, signalling a full import"
+ (is (nil? (git/changed-files snap "no-such-ref-or-sha")))))))))
+
(deftest write-special-collections
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
(mt/with-temp-dir [remote-dir nil]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index cba79d8ad5b..3afbf5b947e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -6,6 +6,7 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.transforms-python.core :as transforms-python]
+ [metabase.test.util.thread-local :as tu.thread-local]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -293,8 +294,22 @@ width: fixed
:or {current "v-remote" branch "main" managed-dirs ingest/legal-top-level-paths}}]
(let [managed (set managed-dirs)
state (atom {:current current :trees (or trees {}) :counter 0})
+ diff-trees (fn [old-tree new-tree]
+ (let [oks (set (keys old-tree)) nks (set (keys new-tree))]
+ {:added (into #{} (remove oks) nks)
+ :deleted (into #{} (remove nks) oks)
+ :modified (into #{} (filter #(and (contains? old-tree %) (not= (old-tree %) (new-tree %)))) nks)}))
mk-snapshot (fn mk-snapshot [version]
- (reify source.p/SourceSnapshot
+ (reify
+ source.p/Diffable
+ ;; In-memory mirror of git's tree diff: nil when the base version is unknown (models
+ ;; a base orphaned by force-push), so the importer falls back to a full import.
+ (changed-files* [_ from-version]
+ (let [trees (:trees @state)]
+ (when (and (contains? trees from-version) (contains? trees version))
+ (diff-trees (get trees from-version) (get trees version)))))
+
+ source.p/SourceSnapshot
(list-files [_] (vec (keys (get-in @state [:trees version] {}))))
(read-file [_ path] (get-in @state [:trees version path]))
(write-files! [_ _message files]
@@ -400,6 +415,20 @@ width: fixed
model tables (Transform, TransformTag, PythonLibrary) are clean."
(t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))
+(defn commit-with-temp
+ "Test fixture (`:each`) that makes `with-temp` COMMIT its rows instead of wrapping the test body in a
+ `:rollback-only` transaction (by binding `*thread-local*` false).
+
+ Required by remote-sync tests that call `import!`: the import reports progress on a separate pool
+ connection (`wrap-progress-ingestable`), and on MySQL that connection blocks on the uncommitted
+ `RemoteSyncTask` row for the full 50s `innodb_lock_wait_timeout` — once per ingested entity — turning
+ each such test into minutes (and timing out the EE MySQL app-db job). Committing the temp rows avoids
+ the cross-connection lock wait. Safe because these tests are non-parallel and clean up via other
+ fixtures. No effect on H2 (same-connection/MVCC). Compose after `clean-remote-sync-state`."
+ [thunk]
+ (binding [tu.thread-local/*thread-local* false]
+ (thunk)))
+
(defn generate-table-yaml
"Generate YAML content for a table with the given `table-name` and `db-name`.
Optionally accepts `:schema`, `:is-published` (defaults to true), and `:description`."
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 0336919a277..09ac4c2ef28 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -22,9 +22,10 @@
(use-fixtures :once (fixtures/initialize :db))
#_{:clj-kondo/ignore [:metabase/validate-deftest]}
-(use-fixtures :each (fn [f]
- (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
- (test-helpers/clean-remote-sync-state f))))
+(use-fixtures :each test-helpers/commit-with-temp
+ (fn [f]
+ (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]
+ (test-helpers/clean-remote-sync-state f))))
(deftest transform-event-creates-sync-object-when-setting-enabled-test
(testing "Creating a transform creates a RemoteSyncObject entry when remote-sync-transforms is enabled"