Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3725 incremental export fast-path for remote sync (#75057)

Eric Normanddb96fadd290b
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index 4310dc1ebcb..64c8d3526e3 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -21,6 +21,7 @@
[metabase.util :as u]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
+ [metabase.util.yaml :as yaml]
[toucan2.core :as t2])
(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))
@@ -221,14 +222,21 @@
{:conflicts (vec all-conflicts)
:summary (into #{} (map :category) all-conflicts)}))
-(defn- branch-changed-since-scheduling?
- "Returns true if `pre-task-branch` was captured by the async-* function and the
- `remote-sync-branch` setting has since drifted to a different value. Used as a
- defense-in-depth check against any future code path that bypasses the operation-level
- guards and mutates the setting between scheduling and the work running."
- [pre-task-branch]
- (and (some? pre-task-branch)
- (not= pre-task-branch (settings/remote-sync-branch))))
+(defn- record-exported-paths!
+ "Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and
+ deletes can resolve the old path. `entries` is a seq of {:model_type :entity_id :path}; only
+ entity-id models are recorded. Correlates entity_id -> model_id per model type."
+ [entries]
+ (doseq [[model-type es] (group-by :model_type entries)
+ :let [spec (spec/spec-for-model-type model-type)]
+ :when (and spec (= :entity-id (:identity spec)))]
+ (let [eid->path (into {} (map (juxt :entity_id :path)) es)
+ id->eid (t2/select-pk->fn :entity_id (:model-key spec) :entity_id [:in (vec (keys eid->path))])]
+ (doseq [[id eid] id->eid
+ :let [path (eid->path eid)]
+ :when path]
+ (t2/update! :model/RemoteSyncObject :model_type model-type :model_id id
+ {:file_path path})))))
(defn import!
"Imports and reloads Metabase entities from a remote snapshot.
@@ -245,7 +253,8 @@
Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may be
thrown during import and are caught and converted to error status maps."
[^SourceSnapshot snapshot task-id & {:keys [force? pre-task-branch]}]
- (when (branch-changed-since-scheduling? pre-task-branch)
+ (when (and (some? pre-task-branch)
+ (not= pre-task-branch (settings/remote-sync-branch)))
(log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"
pre-task-branch (settings/remote-sync-branch))
(throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
@@ -268,8 +277,8 @@
(and first-import? (not force?) (seq conflicts))
(u/prog1 {:status :conflict
:version (source.p/version snapshot)
- :conflicts summary ; Keep backward compatibility: return set of category names
- :conflict-details conflicts ; New: detailed conflict info
+ :conflicts summary ; Keep backward compatibility: return set of category names
+ :conflict-details conflicts ; New: detailed conflict info
:message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}
(log/infof (:message <>)))
@@ -291,7 +300,10 @@
(settings/remote-sync-transforms! true))
(t2/with-transaction [_conn]
(remove-unsynced! (spec/all-syncable-collection-ids) imported-data)
- (sync-objects! sync-timestamp imported-data))
+ (sync-objects! sync-timestamp imported-data)
+ ;; Record the actual repo path each entity was read from, so later renames/deletes
+ ;; resolve the real file and stay on the incremental export fast-path.
+ (record-exported-paths! (source.ingestable/cached-file-paths base-ingestable)))
(when (and (not has-transforms?)
(settings/remote-sync-transforms))
(log/info "No transforms in remote source, disabling remote-sync-transforms setting")
@@ -311,6 +323,193 @@
{:status :error
:message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."})))
+;;; ------------------------------------------- Incremental Export Fast-Path -------------------------------------------
+
+(def ^:private closure-opts
+ {:include-field-values false :include-database-secrets false
+ :continue-on-error false :skip-archived true})
+
+(defn- export-closure
+ "All `[model-type id]` entities a full export would pull for the entity `[model-type model-id]`
+ (its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."
+ [model-type model-id]
+ (keys (merge-with into
+ (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))
+ (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %))))))
+
+(defn- untracked-content-deps
+ "The `[model-type id]` entities in the change's export closure that are remote-sync content but have
+ no RemoteSyncObject row — dependencies a full export would pull (via `serdes/descendants`/`required`)
+ yet that aren't independently tracked (e.g. a card in a non-synced collection referenced by a synced
+ card, and that card's collection). These get written alongside the change so the incremental export
+ matches what a full export would emit. Excludes the entity itself and non-content deps (e.g.
+ databases, which are resolved by reference at import)."
+ [model-type model-id]
+ (into #{}
+ (filter (fn [[mt id]]
+ (and (not (and (= mt model-type) (= id model-id)))
+ (spec/spec-for-model-type mt)
+ (not (t2/exists? :model/RemoteSyncObject :model_type mt :model_id id)))))
+ (export-closure model-type model-id)))
+
+(defn- dep-upserts
+ "Serializes the untracked dependency entities `dep-ids` (a set of `[model-type id]`) into file-specs
+ to upsert, reusing the storage context `opts` so paths dedupe consistently with the changed entities.
+ Returns `[]` when there are no deps, a vector of file-specs on success, or nil if any dependency's
+ path is already occupied by a different entity (a collision that requires a full export)."
+ [snapshot opts dep-ids]
+ (if (empty? dep-ids)
+ []
+ (let [specs (->> dep-ids
+ (map (fn [[mt id]] {:model_type mt :model_id id}))
+ spec/extract-entities-for-rows
+ (map (fn [e] [(source/entity->file-spec opts e) (:entity_id e)])))]
+ ;; keep going only if every dependency's target path is free: absent, or already holding that
+ ;; same entity. A present file whose entity_id can't be read counts as occupied — don't clobber it.
+ (when (every? (fn [[spec eid]]
+ (if-let [content (source.p/read-file snapshot (:path spec))]
+ (= eid (try (:entity_id (yaml/parse-string content))
+ (catch Exception _ nil)))
+ true))
+ specs)
+ (mapv first specs)))))
+
+(defn- incremental-updates-for-row
+ "The updates a single dirty `row` contributes to an incremental plan: a map of
+ {:upserts :delete-paths :synced :removed-ids :pull} (any subset), or `:remote-sync/unsyncable-record`
+ if the row can't be synced incrementally and the whole batch must fall back to a full export."
+ [opts snapshot {:keys [status file_path] :as row}]
+ (try
+ (let [info (delay (when-let [entity (first (spec/extract-entities-for-rows [row]))]
+ (let [spec (source/entity->file-spec opts entity)
+ content (source.p/read-file snapshot (:path spec))
+ yaml (when content
+ ;; errors parsing should throw an invalidate the row
+ (yaml/parse-string content))]
+ {:entity entity
+ :spec spec
+ :eid (:entity_id entity)
+ :new-path (:path spec)
+ :file-exists? (boolean content)
+ :file-eid (:entity_id yaml)})))]
+ (cond
+ (not= :entity-id ;; only entity-id models can be synced incrementally
+ (:identity (spec/spec-for-model-type (:model_type row))))
+ :remote-sync/unsyncable-record
+
+ (not (#{"create" "update" "removed" "delete"} status))
+ :remote-sync/unsyncable-record
+
+ (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export
+ (str/blank? file_path))
+ :remote-sync/unsyncable-record
+
+ (#{"removed" "delete"} status)
+ {:delete-paths [file_path]
+ :removed-ids [(:id row)]}
+
+ ;; past here, we're seeing create and update statuses
+ (not @info) ;; entity no longer exists
+ :remote-sync/unsyncable-record
+
+ ;; create: brand-new file, no old path to delete.
+ ;; Target must be free or same entity id
+ (and (= "create" status)
+ (or (not (:file-exists? @info))
+ (= (:eid @info) (:file-eid @info))))
+ {:pull (untracked-content-deps (:model_type row) (:model_id row))
+ :upserts [(:spec @info)]
+ :synced [{:id (:id row) :file_path (:new-path @info)}]}
+
+ ;; in-place update: at its stored path, or (no stored path) the repo file at
+ ;; new-path is already this entity — overwrite.
+ (and (= "update" status)
+ (or (= file_path (:new-path @info))
+ (and (str/blank? file_path)
+ (= (:eid @info) (:file-eid @info)))))
+ {:pull (untracked-content-deps (:model_type row) (:model_id row))
+ :upserts [(:spec @info)]
+ :synced [{:id (:id row) :file_path (:new-path @info)}]}
+
+ ;; rename: update whose stored path differs from new path. Write the
+ ;; new file and delete the old one.
+ (and (= "update" status)
+ (not (str/blank? file_path))
+ (not= file_path (:new-path @info))
+ (or (not (:file-exists? @info))
+ (= (:eid @info) (:file-eid @info))))
+ {:pull (untracked-content-deps (:model_type row) (:model_id row))
+ :upserts [(:spec @info)]
+ :synced [{:id (:id row) :file_path (:new-path @info)}]
+ :delete-paths [file_path]}
+
+ ;; any other create/update case (e.g. the target path is occupied by a different entity) needs
+ ;; a full export
+ :else
+ :remote-sync/unsyncable-record))
+ (catch Exception _
+ :remote-sync/unsyncable-record)))
+
+(defn- incremental-plan
+ "Builds a plan for a safe incremental export of the current `dirty-rows`, or `:remote-sync/unsyncable-batch` if any
+ row can't be handled incrementally.
+
+ Every row must be on an entity-id model and be one of:
+ - `create`/`update` — re-serialize the entity and upsert its file. A create, or an update whose path
+ is unchanged, is an overwrite-in-place; an update whose stored `file_path` differs is a rename, so
+ the old file is also deleted. The target path must be free or already hold this same entity, and
+ the change must not reference remote-sync content that isn't already in the repo (else a full
+ export, which pulls that dependency via `serdes/descendants`, is required).
+ - `delete`/`removed` — delete the entity's stored `file_path`. Requires a stored `file_path`
+ (after a fresh import none is recorded yet, so we fall back to a full export, which self-heals).
+
+ Returns one of:
+ - {:upserts [file-spec] :delete-paths [path] :synced [{:id :file_path}] :removed-ids [id]} — a safe
+ incremental plan;
+ - `:remote-sync/unsyncable-batch` — a row (or a dependency path collision) forces a full export."
+ [snapshot dirty-rows]
+ (let [opts (serdes/storage-base-context)
+ plan (->> dirty-rows
+ (map #(incremental-updates-for-row opts snapshot %))
+ (reduce (fn [plan updates]
+ (if (= updates :remote-sync/unsyncable-record)
+ (reduced :remote-sync/unsyncable-batch)
+ (merge-with into plan updates)))
+ {:upserts [] :delete-paths [] :synced [] :removed-ids [] :pull #{}}))]
+ (if (= plan :remote-sync/unsyncable-batch)
+ :remote-sync/unsyncable-batch
+ (if-let [deps (dep-upserts snapshot opts (:pull plan))]
+ (-> plan (update :upserts into deps) (dissoc :pull))
+ ;; a dependency's path is occupied by a different entity — needs a full export
+ :remote-sync/unsyncable-batch))))
+
+(defn- path-top-level-dir [^String path]
+ (let [i (str/index-of path "/")]
+ (if i (subs path 0 i) path)))
+
+(defn- disabled-content-dirs
+ "Top-level repo directories whose content is disabled by the current settings."
+ []
+ (cond-> #{}
+ (not (settings/remote-sync-transforms)) (into ["transforms" "python-libraries" "python_libraries"])
+ (not (settings/library-is-remote-synced?)) (conj "snippets")))
+
+(defn- full-export!
+ "Re-serializes the entire remote-synced set into `snapshot` and commits it, then marks every
+ RemoteSyncObject row synced (as of `sync-timestamp`) and records each entity's file_path. Used when
+ the pending changes can't be applied incrementally (name collisions, disabled content needing a
+ reconcile, etc.). Throws if there is no remote-syncable content. Returns {:status :success}."
+ [snapshot task-id message sync-timestamp]
+ (let [models (spec/extract-entities-for-export)]
+ (when (not models)
+ (throw (ex-info "No remote-syncable content available." {})))
+ (remote-sync.task/update-progress! task-id 0.3)
+ (let [{:keys [version entries]} (source/store! models snapshot task-id message)]
+ (remote-sync.task/set-version! task-id version)
+ (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
+ (record-exported-paths! entries))
+ {:status :success}))
+
(defn export!
"Exports remote-synced collections to a remote source repository.
@@ -320,47 +519,70 @@
from the current setting at task start, the export aborts with `:error` (defense-in-depth
against any path that mutates the setting between scheduling and the work running).
- Extracts all remote-synced collections, serializes their content, writes the files to the source, and
- updates all RemoteSyncObject statuses to 'synced'.
+ Takes the incremental fast-path when every pending change can be applied incrementally (see
+ `incremental-plan` — creates, in-place updates, renames, and deletes of entity-id models). Otherwise does a
+ full export: re-serializes the entire synced set, writes it, marks all RemoteSyncObject rows synced,
+ and records each entity's `file_path` so future renames/deletes can go incremental.
- Returns a map with :status (either :success or :error), :version, and optionally :message keys. Various
- exceptions may be thrown during export and are caught and converted to error status maps."
+ Returns a map with {:status :success}, {:status :error}, or throws an exception with an appropriate message."
[^SourceSnapshot snapshot task-id message & {:keys [pre-task-branch]}]
- (when (branch-changed-since-scheduling? pre-task-branch)
+ ;; Defense-in-depth: refuse to run if the branch setting drifted between scheduling and now.
+ (when (and (some? pre-task-branch)
+ (not= pre-task-branch (settings/remote-sync-branch)))
(log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"
pre-task-branch (settings/remote-sync-branch))
(throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
{:pre-task-branch pre-task-branch
:current-branch (settings/remote-sync-branch)})))
- (if snapshot
- (let [sync-timestamp (t/instant)]
- (try
- (analytics/inc! :metabase-remote-sync/exports)
- (serdes/with-cache
- (if-let [models (spec/extract-entities-for-export)]
+
+ (when (not snapshot)
+ (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable." {})))
+
+ (let [sync-timestamp (t/instant)]
+ (try
+ (analytics/inc! :metabase-remote-sync/exports)
+ (serdes/with-cache
+ (let [disabled-files (filterv (comp (disabled-content-dirs) path-top-level-dir)
+ (source.p/list-files snapshot))
+ dirty-rows (seq (remote-sync.object/dirty-rows))
+ plan (when dirty-rows (incremental-plan snapshot dirty-rows))]
+ (cond
+ ;; nothing to do: no pending changes and no stale files in now-disabled content dirs
+ (and (empty? dirty-rows) (empty? disabled-files))
(do
+ (log/info "Remote sync export: no changes to export")
+ {:status :success})
+
+ ;; a dirty row can't be applied incrementally → full re-serialize
+ (= plan :remote-sync/unsyncable-batch)
+ (full-export! snapshot task-id message sync-timestamp)
+
+ ;; Incremental fast-path: write only the changed entities, and delete their old paths plus
+ ;; any files left behind in now-disabled content dirs — preserving every other file. Avoids
+ ;; re-serializing the entire synced set.
+ :else
+ (let [{:keys [upserts delete-paths synced removed-ids]} plan
+ delete-paths (into (vec delete-paths) disabled-files)]
(remote-sync.task/update-progress! task-id 0.3)
- (let [written-version (source/store! models snapshot task-id message)]
+ (let [written-version (source.p/apply-changes! snapshot message upserts delete-paths)]
(remote-sync.task/set-version! task-id written-version))
- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
- {:status :success
- :version (source.p/version snapshot)})
- {:status :error
- :message "No remote-syncable content available."}))
- (catch Exception e
- (if (:cancelled? (ex-data e))
- (log/info "Export to git repository was cancelled")
- (do
- (log/errorf e "Failed to export to git repository: %s" (ex-message e))
- (analytics/inc! :metabase-remote-sync/exports-failed)
- (remote-sync.task/fail-sync-task! task-id (ex-message e))
- {:status :error
- :version (source.p/version snapshot)
- :message (format "Failed to export to git repository: %s" (ex-message e))})))
- (finally
- (analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis)))))
- {:status :error
- :message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."}))
+ (doseq [{:keys [id file_path]} synced]
+ (t2/update! :model/RemoteSyncObject :id id
+ {:status "synced" :file_path file_path :status_changed_at sync-timestamp}))
+ (when (seq removed-ids)
+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
+ (log/infof "Remote sync incremental export: wrote %d, deleted %d"
+ (count upserts) (count delete-paths))
+ {:status :success}))))
+ (catch Exception e
+ ;; handle-task-result! records the failure on this result, and skips entirely when the task
+ ;; was already cancelled (ended_at set) — so cancellation needs no special case here.
+ (log/errorf e "Failed to export to git repository: %s" (ex-message e))
+ (analytics/inc! :metabase-remote-sync/exports-failed)
+ {:status :error
+ :message (format "Failed to export to git repository: %s" (ex-message e))})
+ (finally
+ (analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))
(defn create-task-with-lock!
"Takes a cluster-wide lock and either returns an existing in-progress RemoteSyncTask ID or creates a new one.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
index b06098c60ce..e8604fea5a0 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj
@@ -31,28 +31,33 @@
:status [:not= "synced"]
:model_type [:not-in excluded]))))
+(defn dirty-rows
+ "Returns the raw RemoteSyncObject rows that are not yet synced (status != 'synced'),
+ excluding disabled model types (e.g. transforms when transform sync is off)."
+ []
+ (let [excluded (spec/excluded-model-types)]
+ (if (empty? excluded)
+ (t2/select :model/RemoteSyncObject :status [:not= "synced"])
+ (t2/select :model/RemoteSyncObject
+ :status [:not= "synced"]
+ :model_type [:not-in excluded]))))
+
(defn dirty-objects
"Gets all models in any collection that are dirty with their sync status.
Returns a sequence of model maps that have changed since the last remote sync,
including details about their current state and sync status.
Excludes transform model types when transform sync is disabled."
[]
- (let [excluded (spec/excluded-model-types)
- query (if (empty? excluded)
- (t2/select :model/RemoteSyncObject :status [:not= "synced"])
- (t2/select :model/RemoteSyncObject
- :status [:not= "synced"]
- :model_type [:not-in excluded]))]
- (->> query
- (map #(-> %
- (dissoc :id :status_changed_at)
- (set/rename-keys {:model_id :id
- :model_name :name
- :model_type :model
- :model_collection_id :collection_id
- :model_display :display
- :model_table_id :table_id
- :model_table_name :table_name
- :status :sync_status})
- (update :model u/lower-case-en)))
- (into []))))
+ (->> (dirty-rows)
+ (map #(-> %
+ (dissoc :id :status_changed_at)
+ (set/rename-keys {:model_id :id
+ :model_name :name
+ :model_type :model
+ :model_collection_id :collection_id
+ :model_display :display
+ :model_table_id :table_id
+ :model_table_name :table_name
+ :status :sync_status})
+ (update :model u/lower-case-en)))
+ (into [])))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 1602ae72bf0..c55aa157276 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -8,7 +8,6 @@
[metabase-enterprise.serialization.core :as serialization]
[metabase.models.serialization :as serdes]
[metabase.settings.core :as setting]
- [metabase.util :as u]
[metabase.util.yaml :as yaml]
[methodical.core :as methodical])
(:import
@@ -16,8 +15,9 @@
(set! *warn-on-reflection* true)
-;; Wrapping snapshot accepts a list of path regexes to apply to paths in the source returning
-;; nil when they do not match
+;; A read-only, path-filtered view over a snapshot, used to scope ingestion to a set of path regexes:
+;; files (and reads) outside the filters are omitted. It is not a write target — exports write to the
+;; unfiltered source snapshot — so the write methods throw rather than silently filtering writes.
(defrecord WrappingSnapshot [original-snapshot path-filters]
source.p/SourceSnapshot
@@ -30,11 +30,11 @@
(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)
(source.p/read-file original-snapshot path)))
- (write-files! [_ message files]
- (source.p/write-files! original-snapshot message
- (filter (fn [file-spec]
- (some (fn [path-filter] (re-matches path-filter (:path file-spec))) path-filters))
- files)))
+ (write-files! [_ _message _files]
+ (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))
+
+ (apply-changes! [_ _message _upserts _delete-paths]
+ (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))
(version [_]
(source.p/version original-snapshot)))
@@ -53,15 +53,13 @@
basename (str (last resolved) ".yaml")]
(str/join File/separator (concat dirnames [basename]))))
-(defn- ->file-spec
- "Converts entity from serdes stream into file spec for source write-files!"
- [task-id count opts idx entity]
- (when (instance? Exception entity)
- (throw entity))
- (u/prog1 {:path (remote-sync-path opts entity)
- :content (yaml/generate-string (serialization/serialization-deep-sort entity)
- {:dumper-options {:flow-style :block :split-lines false}})}
- (remote-sync.task/update-progress! task-id (-> (inc idx) (/ count) (* 0.65) (+ 0.3)))))
+(defn entity->file-spec
+ "Serializes a single extracted entity into a `{:path :content}` file spec, using storage context
+ `opts` (from `serdes/storage-base-context`)."
+ [opts entity]
+ {:path (remote-sync-path opts entity)
+ :content (yaml/generate-string (serialization/serialization-deep-sort entity)
+ {:dumper-options {:flow-style :block :split-lines false}})})
(defn store!
"Stores serialized entities from a stream to a remote source and commits the changes.
@@ -71,15 +69,22 @@
identifier used to track progress updates), and a message (the commit message to use when writing
files to the source).
- Returns the version written to the source.
-
- Throws Exception if any entity in the stream is an Exception instance."
+ Returns `{:version <written-version> :entries [{:model_type :entity_id :path} ...]}`."
[stream snapshot task-id message]
- (let [opts (serdes/storage-base-context)
- stream-count (bounded-count 10000 stream)]
- (->> stream
- (map-indexed #(->file-spec task-id stream-count opts %1 %2))
- (source.p/write-files! snapshot message))))
+ (let [opts (serdes/storage-base-context)
+ stream-count (bounded-count 10000 stream)
+ entries (volatile! [])
+ version (->> stream
+ (map-indexed (fn [idx entity]
+ (let [spec (entity->file-spec opts entity)]
+ (vswap! entries conj {:model_type (-> entity :serdes/meta last :model)
+ :entity_id (:entity_id entity)
+ :path (:path spec)})
+ (remote-sync.task/update-progress!
+ task-id (-> (min (inc idx) stream-count) (/ stream-count) (* 0.65) (+ 0.3)))
+ spec)))
+ (source.p/write-files! snapshot message))]
+ {:version version :entries @entries}))
(defn source-from-settings
"Creates a git source from the current remote sync settings.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index 9e34570487d..138c544f7ff 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -14,7 +14,8 @@
(java.net URI)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git GitCommand TransportCommand)
- (org.eclipse.jgit.dircache DirCache DirCacheEntry)
+ (org.eclipse.jgit.dircache DirCache DirCacheEditor$DeletePath DirCacheEditor$DeleteTree
+ DirCacheEditor$PathEdit DirCacheEntry)
(org.eclipse.jgit.lib CommitBuilder Constants FileMode PersonIdent Ref)
(org.eclipse.jgit.revwalk RevCommit RevWalk)
(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate
@@ -233,13 +234,6 @@
(throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))
push-response))
-(defn- top-level-dir
- "Extracts the first path segment from a file path.
- E.g., \"databases/my_db/my_db.yaml\" => \"databases\""
- [path]
- (when-let [idx (str/index-of path "/")]
- (subs path 0 idx)))
-
(defn default-branch
"Retrieves the default branch name of the git repository.
@@ -257,60 +251,43 @@
(str/replace-first (.getName ^Ref target) "refs/heads/" ""))))
(throw (ex-info "Failed to get a default branch for git repository." {:head-ref head-ref})))))
-(defn write-files!
- "Writes multiple files to the git repository and commits the changes.
-
- Takes a snapshot map containing a :git Git instance, :version, and :managed-dirs,
- a commit message string, and a sequence of file specs (maps with :path and :content keys,
- paths should be relative to the repository root).
-
- All existing files within managed directories that are not in the write set are removed.
- This ensures stale files (from renames, moves, or deletions) are cleaned up automatically.
- Files outside managed directories are always preserved.
-
- Returns the version written. Throws ExceptionInfo if the write or push
- operation fails."
- [{:keys [^Git git ^String version ^String branch managed-dirs] :as snapshot} ^String message files]
- (let [repo (.getRepository git)
+(defn- commit-edits!
+ "Builds a commit by editing the branch tip's tree in place and pushes it. Seeds an in-core index from
+ the parent commit's tree — so every unchanged entry and subtree is carried forward by object id — then
+ applies only the edits: whole subtrees named in `delete-dirs` are removed (used to fully replace a
+ managed directory), paths in `delete-paths` are removed, and `upserts` (file specs with :path/:content)
+ are written as new blobs. `writeTree` only rewrites the subtrees on the path to an edit, so the work is
+ proportional to the number of changes rather than the size of the repo.
+
+ Returns the new commit sha. Throws ExceptionInfo if the write or push fails."
+ [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message upserts delete-paths delete-dirs]
+ (let [repo (.getRepository git)
branch-ref (qualify-branch branch)
- parent-id (.resolve repo version)]
- (with-open [inserter (.newObjectInserter repo)]
- (let [index (DirCache/newInCore)
- builder (.builder index)
- ;; Set of all paths being written in this commit
- write-paths (into #{}
- (comp (map :path)
- (remove str/blank?))
- files)]
- ;; Add new/updated files to the index
- (doseq [{:keys [path content]} files
- :when (not (str/blank? path))]
- (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))
- entry (doto (DirCacheEntry. ^String path)
- (.setFileMode FileMode/REGULAR_FILE)
- (.setObjectId blob-id))]
- (.add builder entry)))
- ;; Copy existing tree entries that should be preserved:
- ;; - Outside managed directories AND not being overwritten by the write set
- ;; Files in managed dirs not in write-paths are dropped (stale file cleanup)
- ;; Files in write-paths are skipped here (already added above with new content)
- (when parent-id
- (with-open [rev-walk (RevWalk. repo)
- tree-walk (TreeWalk. repo)]
- (let [commit (.parseCommit rev-walk parent-id)]
- (.addTree tree-walk (.getTree commit))
- (.setRecursive tree-walk true)
- (while (.next tree-walk)
- (let [path (.getPathString tree-walk)]
- (when (and (not (contains? write-paths path))
- (not (contains? managed-dirs (top-level-dir path))))
- (let [entry (doto (DirCacheEntry. path)
- (.setFileMode (.getFileMode tree-walk 0))
- (.setObjectId (.getObjectId tree-walk 0)))]
- (.add builder entry))))))))
- (.finish builder)
- ;; Create commit
- (let [tree-id (.writeTree index inserter)
+ parent-id (.resolve repo version)]
+ (with-open [inserter (.newObjectInserter repo)
+ reader (.newObjectReader repo)
+ rev-walk (RevWalk. repo)]
+ (let [index (DirCache/newInCore)]
+ ;; Seed the index from the parent tree; unchanged entries and subtrees carry forward by id.
+ (let [builder (.builder index)]
+ (when parent-id
+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader
+ (.getTree (.parseCommit rev-walk parent-id))))
+ (.finish builder))
+ ;; Apply only the edits.
+ (let [editor (.editor index)]
+ (doseq [^String dir delete-dirs]
+ (.add editor (DirCacheEditor$DeleteTree. dir)))
+ (doseq [^String path delete-paths]
+ (.add editor (DirCacheEditor$DeletePath. path)))
+ (doseq [{:keys [^String path content]} upserts]
+ (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]
+ (.add editor (proxy [DirCacheEditor$PathEdit] [path]
+ (apply [^DirCacheEntry entry]
+ (.setFileMode entry FileMode/REGULAR_FILE)
+ (.setObjectId entry blob-id))))))
+ (.finish editor))
+ (let [tree-id (.writeTree index inserter)
commit-builder (doto (CommitBuilder.)
(.setTreeId tree-id)
(.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))
@@ -326,6 +303,17 @@
(push-branch! snapshot)
(.name commit-id)))))))
+(defn- write-files!
+ "Full export: replace every managed dir wholesale with `files` (managed-dir files not in `files` are
+ removed), preserving everything outside the managed dirs."
+ [{:keys [managed-dirs] :as snapshot} ^String message files]
+ (commit-edits! snapshot message files nil managed-dirs))
+
+(defn- apply-changes!
+ "Incremental patch: write `upserts`, remove `delete-paths`, and preserve every other file."
+ [snapshot ^String message upserts delete-paths]
+ (commit-edits! snapshot message upserts delete-paths nil))
+
(defn branches
"Retrieves all branch names from the remote repository.
@@ -406,6 +394,9 @@
(write-files! [this message files]
(write-files! this message files))
+ (apply-changes! [this message upserts delete-paths]
+ (apply-changes! this message upserts delete-paths))
+
(version [this]
(:version this)))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
index 7e9fbec8c4b..7db471cafe5 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj
@@ -17,8 +17,10 @@
(serialization/read-timestamps (yaml/parse-string file-content {:key-fn serialization/parse-key})))
(defn- ingest-all
- "Returns {:entities {stripped-hierarchy [hierarchy content]}, :errors [Exception...]}.
- Dotfiles are silently skipped (editor temp files, see #41567).
+ "Returns {:entities {stripped-hierarchy {:content <yaml-string> :path <repo-path>}}, :errors [Exception...]}.
+ The repo `:path` is the actual file the entity was read from (including any dedup suffix), so callers
+ can record where each entity lives without recomputing — recomputation would diverge on name
+ collisions and slug changes. Dotfiles are silently skipped (editor temp files, see #41567).
Non-dotfile YAML parse/read failures are collected in :errors."
[snapshot]
(let [errors (atom [])]
@@ -39,7 +41,7 @@
(swap! errors conj (ex-info (format "Failed to parse file: %s" path) {:file path} e))
nil))]
:when loaded]
- [(serialization/strip-labels loaded) [loaded content]]))
+ [(serialization/strip-labels loaded) {:content content :path path}]))
:errors @errors}))
;; Wraps another Ingestable calling a callback when a file is ingested
@@ -69,8 +71,17 @@
(letfn [(progress-callback [item _]
(when item
(let [current-calls (swap! calls inc)]
- (t2/with-connection [_conn (app-db/app-db)]
- (remote-sync.task/update-progress! task-id (* (/ current-calls total) normalize))))))]
+ ;; Progress reporting must never abort the ingestion it tracks. The update runs on a
+ ;; separate connection, which on some app DBs can contend with the in-flight load (e.g.
+ ;; a MySQL lock-wait timeout), so swallow DB failures and keep going — but still honor a
+ ;; cancellation signal, which `update-progress!` raises to stop the task.
+ (try
+ (t2/with-connection [_conn (app-db/app-db)]
+ (remote-sync.task/update-progress! task-id (* (/ current-calls total) normalize)))
+ (catch Exception e
+ (if (:cancelled? (ex-data e))
+ (throw e)
+ (log/warn (u/strip-error e "Failed to report import progress; continuing"))))))))]
(->CallbackIngestable ingestable progress-callback))))
;; Wraps another Ingestable and filters the `list-files` content to only content that has the specified
@@ -128,9 +139,19 @@
(populate-cache! cache errors-atom #(ingest-all snapshot))
(when-let [target (get @cache (serialization/strip-labels serdes-path))]
(try
- (ingest-content (second target))
+ (ingest-content (:content target))
(catch Exception e
(throw (ex-info "Unable to ingest file" {:abs-path serdes-path} e))))))
(ingest-errors [_]
(or @errors-atom [])))
+
+(defn cached-file-paths
+ "Given an `IngestableSnapshot` whose cache has been populated by a prior ingestion, returns a seq of
+ {:model_type :entity_id :path} — the actual repo file each entity was read from. Lets the importer
+ record `file_path` so later renames and deletes resolve the real file."
+ [{:keys [cache]}]
+ (for [[hierarchy {:keys [path]}] @cache
+ :let [{:keys [model id]} (last hierarchy)]
+ :when (and model id path)]
+ {:model_type model :entity_id id :path path}))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index 9de80c1daaa..d5ae7b4b6af 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -58,6 +58,16 @@
Returns the version of the written files.")
+ (apply-changes! [snapshot message upserts delete-paths]
+ "Incrementally updates the source: writes/overwrites `upserts`, removes `delete-paths`,
+ and PRESERVES every other existing file (unlike `write-files!`, which deletes managed-dir
+ files absent from the write set).
+
+ Takes a SourceSnapshot instance, a commit message, `upserts` (a sequence of file specs, each
+ a map with :path and :content), and `delete-paths` (a sequence of path strings to remove).
+
+ Returns the version of the written files.")
+
(version [snapshot]
"Gets a version identifier for the current state of the snapshot.
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 87863feba0d..3fbf6bfeebe 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -1145,3 +1145,18 @@
:skip-archived true})))
cat
(u/group-by first second (keys targets))))))
+
+(defn extract-entities-for-rows
+ "Serializes the entities named by `rows`, grouped by model type. Each row is a map with a
+ `:model_type` (the serdes model name) and a `:model_id` (the local DB primary key). Returns a lazy
+ sequence of serialized entities."
+ [rows]
+ (let [by-model (reduce (fn [m {:keys [model_type model_id]}]
+ (update m model_type (fnil conj #{}) model_id))
+ {}
+ rows)]
+ (eduction (map (fn [[model ids]]
+ (serdes/extract-all model {:where [:in :id ids]
+ :skip-archived true})))
+ cat
+ by-model)))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index 853274882e9..358ee828d18 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -334,8 +334,11 @@
(deftest export-handles-write-errors-test
(testing "POST /api/ee/remote-sync/export handles write errors"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
- (mt/with-temp [:model/Collection _ {:is_remote_synced true :name "Test Collection" :location "/"}]
- (let [mock-main (test-helpers/create-mock-source :fail-mode :write-files-error)]
+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :name "Test Collection" :location "/"}]
+ ;; A pending create makes the export take the incremental write path, where the error fires.
+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id coll-id :model_name "Test Collection"
+ :status "create" :status_changed_at (t/offset-date-time)})
+ (let [mock-main (test-helpers/create-mock-source :fail-mode :apply-changes-error)]
(mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"
remote-sync-token "test-token"
remote-sync-branch "main"]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 9eeb5fa7237..5abd51b37c8 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -166,23 +166,33 @@
;; export! tests
(deftest export!-with-no-source-configured-test
- (testing "export! with no source configured"
+ (testing "export! with no source configured throws"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
- result (impl/export! nil task-id "Test commit")]
- (is (= :error (:status result)))
- (is (re-find #"Remote sync source is not enabled" (:message result)))))))
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Remote sync source is not enabled"
+ (impl/export! nil task-id "Test commit")))))))
(deftest export!-with-no-remote-synced-collections-test
- (testing "export! errors when there are no remote-synced collections"
+ (testing "export! is a no-op success when nothing is dirty (no remote-synced content to export)"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}]
(mt/with-temporary-setting-values [remote-sync-transforms false]
(let [mock-source (test-helpers/create-mock-source)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
- (is (= :error (:status result)))
- (is (= "No remote-syncable content available." (:message result))))))))))
+ (is (= :success (:status result))))))))))
+
+(deftest full-export!-errors-with-no-content-test
+ (testing "full-export! throws when there is no remote-syncable content"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}]
+ (mt/with-temporary-setting-values [remote-sync-transforms false]
+ (let [mock-source (test-helpers/create-mock-source)]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"No remote-syncable content available"
+ (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant)))))))))))
(deftest export!-successful-with-default-collections-test
(testing "export! successful with default collections"
@@ -194,20 +204,23 @@
(is (= :success (:status result)))))))))
(deftest export!-handles-store-failure-test
- (testing "export! handles store failure"
+ (testing "full-export! surfaces a store failure"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
(mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]
- (let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)
- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
- (is (= :error (:status result)))
- (is (re-find #"Failed to export to git repository" (:message result)))))))))
+ (let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)]
+ (is (thrown-with-msg? Exception #"Store failed"
+ (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit message" (t/instant))))))))))
(deftest export!-handles-network-errors-during-write-test
- (testing "export! handles network errors during write"
+ (testing "export! catches a write failure and returns an :error result"
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]
+ ;; A pending create makes the export take the incremental write path; the network error there
+ ;; must surface as an :error result.
+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id coll-id :model_name "Test Collection"
+ :status "create" :status_changed_at (t/offset-date-time)})
(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)
result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]
(is (= :error (:status result)))
@@ -220,6 +233,12 @@
(mt/with-temporary-setting-values [remote-sync-type :read-write]
(mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}
:model/Card {card-id :id} {:name "Test Card" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}]
+ ;; Pending creates so the export writes the entities (in production these come from events).
+ (t2/insert! :model/RemoteSyncObject
+ [{:model_type "Collection" :model_id coll-id :model_name "Test Collection"
+ :status "create" :status_changed_at (t/offset-date-time)}
+ {:model_type "Card" :model_id card-id :model_name "Test Card"
+ :status "create" :status_changed_at (t/offset-date-time)}])
(let [mock-main (test-helpers/create-mock-source :branch "test-branch")
export-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
export-result (impl/export! (source.p/snapshot mock-main) (:id export-task) "Test export")]
@@ -265,6 +284,24 @@
(is (not (t2/exists? :model/Collection :id coll2-id)))
(is (not (t2/exists? :model/Card :id card2-id))))))))
+(deftest import!-records-file-path-test
+ (testing "import! records each entity's actual repo file_path on its RemoteSyncObject row, so later
+ renames/deletes resolve the real file and stay on the incremental export fast-path"
+ (let [import-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ coll-path "collections/main/test_collection_1/test_collection_1.yaml"
+ card-path "collections/main/test_collection_1/test_card_1.yaml"]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection 1" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}
+ :model/Card {card-id :id} {:name "Test Card 1" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}]
+ (let [test-files {"test-branch" {coll-path (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection 1")
+ card-path (test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card 1" "test-collection-1xxxx")}}
+ mock-main (test-helpers/create-mock-source :initial-files test-files :branch "test-branch")
+ result (impl/import! (source.p/snapshot mock-main) (:id import-task))]
+ (is (= :success (:status result)))
+ (is (= coll-path (t2/select-one-fn :file_path :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))
+ "the collection's row records the file it was imported from")
+ (is (= card-path (t2/select-one-fn :file_path :model/RemoteSyncObject :model_type "Card" :model_id card-id))
+ "the card's row records the file it was imported from"))))))
+
(deftest error-handling-propagation-through-private-functions-test
(testing "error handling propagation through private functions"
(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)
@@ -305,7 +342,7 @@
(mt/with-dynamic-fn-redefs [remote-sync.task/update-progress!
(fn [task-id progress]
(swap! progress-calls conj {:task-id task-id :progress progress}))]
- (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ (let [result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))]
(is (= :success (:status result)))
(is (pos? (count @progress-calls)))
(is (= task-id (:task-id (first @progress-calls))))
@@ -737,10 +774,8 @@
:filter [:> [:field 2 nil] 0]}
:entity_id "archived-segment-xxxx"
:archived true}]
- (t2/insert! :model/RemoteSyncObject
- [{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])
(let [mock-source (test-helpers/create-mock-source)
- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
@@ -1015,10 +1050,8 @@
:table_id table-id
:entity_id "archived-measure-xxxx"
:archived true}]
- (t2/insert! :model/RemoteSyncObject
- [{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])
(let [mock-source (test-helpers/create-mock-source)
- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]
+ result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))]
(is (= :success (:status result)))
(let [files-after-export (get @(:files-atom mock-source) "main")
file-keys (keys files-after-export)]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
new file mode 100644
index 00000000000..d4dc1d0bc20
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj
@@ -0,0 +1,507 @@
+(ns metabase-enterprise.remote-sync.incremental-export-test
+ "Tests for the incremental export fast-path (GHY-3725). The fast-path writes only the changed
+ entities (and deletes only the removed ones) when every pending change can be applied
+ incrementally — in-place updates, renames, and deletes of entity-id models whose old path is
+ known via the RemoteSyncObject file_path column. Anything else falls back to the full export."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.test-helpers :as rs.test]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.util.yaml :as yaml]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+(use-fixtures :each rs.test/clean-remote-sync-state)
+
+(defn- venues-query []
+ {:database (mt/id) :type :query :query {:source-table (mt/id :venues)}})
+
+(defn- card-source-query [card-id]
+ {:database (mt/id) :type :query :query {:source-table (str "card__" card-id)}})
+
+(defn- new-task! []
+ (t2/delete! :model/RemoteSyncTask)
+ (t2/insert-returning-pk! :model/RemoteSyncTask
+ {:sync_task_type "export" :initiated_by (mt/user->id :rasta)}))
+
+(defn- written-version [task-id]
+ (t2/select-one-fn :version :model/RemoteSyncTask :id task-id))
+
+(defn- seed-synced-row! [model-type model-id]
+ (t2/insert! :model/RemoteSyncObject
+ {:model_type model-type
+ :model_id model-id
+ :model_name (str model-type " " model-id)
+ :status "synced"
+ :status_changed_at (t/offset-date-time)}))
+
+(defn- set-status! [model-type model-id status]
+ (t2/update! :model/RemoteSyncObject :model_type model-type :model_id model-id
+ {:status status :status_changed_at (t/offset-date-time)}))
+
+(defn- seed-create-row!
+ "Seeds a `create`-status RemoteSyncObject row, so the next export writes the entity (and records its
+ file_path). Use when a test needs to get an entity into the repo."
+ [model-type model-id]
+ (seed-synced-row! model-type model-id)
+ (set-status! model-type model-id "create"))
+
+(defn- rso [model-type model-id]
+ (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id))
+
+(defn- files [mock] (get @(:files-atom mock) "main"))
+
+(defn- path-for-eid
+ "The repo path of the file whose content contains `eid` (filenames are slug-only)."
+ [mock eid]
+ (some (fn [[p c]] (when (re-find (re-pattern eid) c) p)) (files mock)))
+
+(defn- entity-exported?
+ "True if some repo file is the serialized entity with top-level `eid` (i.e. the entity has its own
+ file). Distinct from `path-for-eid`, which also matches a mere reference to `eid` inside another
+ entity's file."
+ [mock eid]
+ (boolean (some (fn [[_ c]]
+ (try (= eid (:entity_id (yaml/parse-string c)))
+ (catch Exception _ false)))
+ (files mock))))
+
+(defn- with-exported-collection!
+ "Sets up a remote-synced collection with two cards, marks the three entities as new (`create`), and
+ runs an initial export into a fresh empty MockSource — writing the files and recording each entity's
+ file_path. Then calls `f` with `{:mock :coll-id :card-a :card-b}`."
+ [f]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Bench" :is_remote_synced true :location "/"}
+ :model/Card {card-a :id} {:name "Card A" :collection_id coll-id}
+ :model/Card {card-b :id} {:name "Card B" :collection_id coll-id}]
+ ;; Reset RSO and seed `create` rows for the three entities (in production these come from events /
+ ;; enabling sync) so the initial export writes them and records their file_path.
+ (t2/delete! :model/RemoteSyncObject)
+ (doseq [[model-type model-id] [["Collection" coll-id] ["Card" card-a] ["Card" card-b]]]
+ (seed-create-row! model-type model-id))
+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})
+ init-task (new-task!)
+ init-result (impl/export! (source.p/snapshot mock) init-task "init")]
+ (is (= :success (:status init-result)) "initial export succeeds")
+ (is (= "apply-changes-version" (written-version init-task))
+ "the three new entities are written incrementally")
+ (f {:mock mock :coll-id coll-id :card-a card-a :card-b card-b})))))
+
+(deftest full-export-records-file-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]
+ (is (= (path-for-eid mock a-eid) (:file_path (rso "Card" card-a)))
+ "full export records each entity's file_path on its RemoteSyncObject row")))))
+
+(deftest in-place-update-uses-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ a-path (path-for-eid mock a-eid)
+ files-before (files mock)]
+ (t2/update! :model/Card card-a {:description "edited in place"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "edit")
+ files-after (files mock)
+ other-path (some (fn [[p _]] (when (not= p a-path) p)) files-before)]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task)) "in-place update is incremental")
+ (is (= (set (keys files-before)) (set (keys files-after))) "no files added or removed")
+ (is (re-find #"edited in place" (get files-after a-path)) "edit written")
+ (is (= (get files-before other-path) (get files-after other-path)) "other files preserved")
+ (is (= a-path (:file_path (rso "Card" card-a))) "file_path unchanged")
+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))))))))
+
+(deftest multiple-in-place-updates-use-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a card-b]}]
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (t2/update! :model/Card card-b {:description "edit B"})
+ (set-status! "Card" card-a "update")
+ (set-status! "Card" card-b "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "edit both")
+ files-after (files mock)]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task)))
+ (is (some (fn [[_ c]] (re-find #"edit A" c)) files-after))
+ (is (some (fn [[_ c]] (re-find #"edit B" c)) files-after))))))
+
+(deftest collection-in-place-edit-uses-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock coll-id]}]
+ (t2/update! :model/Collection coll-id {:description "collection note"})
+ (set-status! "Collection" coll-id "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "edit collection")]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task))
+ "an in-place collection edit (no rename) is incremental")))))
+
+(deftest rename-uses-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ old-path (path-for-eid mock a-eid)]
+ (t2/update! :model/Card card-a {:name "Card A Renamed"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "rename")
+ files-after (files mock)
+ new-path (path-for-eid mock a-eid)]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task)) "rename is incremental")
+ (is (not (contains? files-after old-path)) "old file removed")
+ (is (not= old-path new-path) "file moved to a new path")
+ (is (re-find #"Card A Renamed" (get files-after new-path)) "new file has the new name")
+ (is (= 1 (count (filter (fn [[_ c]] (re-find (re-pattern a-eid) c)) files-after)))
+ "exactly one file for the entity (no stale duplicate)")
+ (is (= new-path (:file_path (rso "Card" card-a))) "file_path updated to the new path"))))))
+
+(deftest delete-uses-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a card-b]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ b-eid (t2/select-one-fn :entity_id :model/Card :id card-b)
+ b-path (path-for-eid mock b-eid)]
+ ;; A real delete means the entity is archived, so it can't be re-serialized — the incremental
+ ;; delete must work from the stored file_path alone, without re-extracting the entity.
+ (t2/update! :model/Card card-b {:archived true})
+ (set-status! "Card" card-b "delete")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "delete B")
+ files-after (files mock)]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task)) "delete is incremental")
+ (is (not (contains? files-after b-path)) "deleted entity's file removed")
+ (is (some (fn [[_ c]] (re-find (re-pattern a-eid) c)) files-after) "card A preserved")
+ (is (nil? (rso "Card" card-b)) "deleted entity's RemoteSyncObject row removed"))))))
+
+(deftest rename-without-stored-path-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ ;; Simulate the post-import state where file_path hasn't been recorded yet: a rename then can't
+ ;; resolve the old path, so we fall back to the full export (which reconciles stale files).
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})
+ (t2/update! :model/Card card-a {:name "Card A Renamed"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "rename")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "rename without a stored file_path falls back to full")))))
+
+(deftest delete-without-stored-path-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-b]}]
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})
+ (set-status! "Card" card-b "delete")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "delete")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "delete without a stored file_path falls back to full")))))
+
+(deftest create-uses-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock coll-id]}]
+ (mt/with-temp [:model/Card {card-c :id} {:name "Card C" :collection_id coll-id}]
+ (seed-synced-row! "Card" card-c)
+ (set-status! "Card" card-c "create")
+ (let [files-before (files mock)
+ c-eid (t2/select-one-fn :entity_id :model/Card :id card-c)
+ task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "add card C")
+ (is (= "apply-changes-version" (written-version task)) "a create is incremental")
+ (is (entity-exported? mock c-eid) "the new card's file is written")
+ (is (some? (:file_path (rso "Card" card-c))) "file_path recorded for the new card")
+ (is (every? (fn [[p c]] (= c (get (files mock) p))) files-before)
+ "existing files are preserved unchanged")
+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))))))))
+
+(deftest create-with-name-collision-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock coll-id]}]
+ ;; A new card whose name collides with an existing card resolves (in isolation) to the existing
+ ;; card's path. The safety check sees a different occupant and falls back to full, which dedups.
+ (mt/with-temp [:model/Card {card-c :id} {:name "Card A" :collection_id coll-id}]
+ (seed-synced-row! "Card" card-c)
+ (set-status! "Card" card-c "create")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "collision create")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "a create whose path collides with a different entity falls back to full")
+ (is (entity-exported? mock (t2/select-one-fn :entity_id :model/Card :id card-c))
+ "the colliding new card is still exported (at a deduped path) by the full export"))))))
+
+(deftest mixed-batch-with-unhandled-row-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a card-b]}]
+ ;; One incremental-able update batched with a delete that has no stored path (so it can't be
+ ;; handled incrementally) -> the whole batch falls back to a full export.
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (set-status! "Card" card-a "update")
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})
+ (set-status! "Card" card-b "delete")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "mixed")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "a single non-incremental row forces the whole batch to full export")))))
+
+(deftest name-collision-backfill-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ a-path (path-for-eid mock a-eid)]
+ ;; No stored file_path (post-import), and the repo file at the computed path holds a DIFFERENT
+ ;; entity (a dedup name collision). The safety check must reject the clobber → full export.
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})
+ (swap! (:files-atom mock) assoc-in ["main" a-path]
+ (str/replace (get (files mock) a-path) a-eid "different0entity0id0x"))
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "collision")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "a path holding a different entity fails the safety check and falls back to full"))))))
+
+(deftest unparseable-occupant-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ a-path (path-for-eid mock a-eid)]
+ ;; No stored file_path (post-import), and the repo file at the computed path is present but has
+ ;; no readable entity_id. We can't tell whose file it is, so the safety check must treat it as
+ ;; occupied — not free — and fall back to full rather than clobber an unidentifiable entity.
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})
+ (swap! (:files-atom mock) assoc-in ["main" a-path] "name: Not A Serialized Entity\nother: value\n")
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "unparseable occupant")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "a present file with no entity_id fails the safety check and falls back to full"))))))
+
+(deftest malformed-yaml-occupant-falls-back-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ a-path (path-for-eid mock a-eid)]
+ ;; No stored file_path, and the repo file at the computed path is malformed YAML that can't be
+ ;; parsed. Reading it throws — the row must be treated as unsyncable and fall back to a full
+ ;; export rather than crashing the whole export.
+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})
+ (swap! (:files-atom mock) assoc-in ["main" a-path] "a: b: c")
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (set-status! "Card" card-a "update")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "malformed occupant")]
+ (is (= :success (:status result)))
+ (is (= "write-files-version" (written-version task))
+ "a malformed YAML file at the target path falls back to full export instead of crashing"))))))
+
+(deftest create-onto-same-entity-stays-incremental-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ ;; card-a is already exported: its file sits at its computed path holding its own entity_id. A
+ ;; fresh "create" whose target path already holds THIS same entity is a safe overwrite, so it must
+ ;; stay incremental rather than fall back (the path is occupied, but by us).
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]
+ (set-status! "Card" card-a "create")
+ (let [task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "re-create A")
+ (is (= "apply-changes-version" (written-version task))
+ "a create whose path already holds the same entity stays incremental")
+ (is (entity-exported? mock a-eid) "card A is still exported"))))))
+
+;;; ------------------------------------ Disabled content cleanup (GHY-3725) ------------------------------------
+;;; When a content type is disabled (transforms is off in `with-exported-collection!`), stale files left
+;;; behind in its repo dir must be removed — but that shouldn't drop us off the incremental fast-path.
+;;; They're appended to the incremental delete-paths instead of forcing a full re-serialize.
+
+(deftest disabled-content-cleaned-up-incrementally-test
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ ;; A stale file in a now-disabled content dir, alongside an ordinary card edit. The edit stays on
+ ;; the incremental path AND the stale transforms file is deleted in the same commit.
+ (swap! (:files-atom mock) assoc-in ["main" "transforms/old/old.yaml"] "stale transform")
+ (t2/update! :model/Card card-a {:description "edit A"})
+ (set-status! "Card" card-a "update")
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)
+ task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "edit with stale transform")
+ (is (= "apply-changes-version" (written-version task))
+ "disabled content no longer forces a full export")
+ (is (nil? (get (files mock) "transforms/old/old.yaml"))
+ "the stale file in the disabled dir is deleted incrementally")
+ (is (entity-exported? mock a-eid) "the edited card is still exported")))))
+
+(deftest disabled-content-cleaned-up-with-no-dirty-rows-test
+ (with-exported-collection!
+ (fn [{:keys [mock]}]
+ ;; A stale file in a disabled dir with nothing else dirty: a delete-only incremental commit removes
+ ;; it (rather than a no-op that would leave it behind, or a full re-serialize).
+ (swap! (:files-atom mock) assoc-in ["main" "transforms/old/old.yaml"] "stale transform")
+ (let [task (new-task!)
+ result (impl/export! (source.p/snapshot mock) task "remove stale transform")]
+ (is (= :success (:status result)))
+ (is (= "apply-changes-version" (written-version task))
+ "a delete-only incremental commit removes the stale file")
+ (is (nil? (get (files mock) "transforms/old/old.yaml")))))))
+
+;;; ---------------------------------- Required-closure regression (GHY-3725) ------------------------------------
+;;; A full export pulls a card's source cards via `serdes/descendants` (transitively, through
+;;; `resolve-targets`), even when the source card lives outside the synced collection. The incremental
+;;; fast-path writes only the changed entities, so a change that introduces such a cross-scope reference
+;;; must still export the referenced dependency — otherwise the reference dangles on import.
+
+(defn- with-cross-scope-setup!
+ "Synced collection C plus a card Y in a NON-synced collection. Runs an initial full export into a
+ fresh MockSource. Calls `f` with `{:mock :c-id :ext-id :y-id :y-eid}`. Y is not referenced yet, so
+ it isn't in the repo after the initial export."
+ [f]
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {c-id :id} {:name "Synced" :is_remote_synced true :location "/"}
+ :model/Collection {ext-id :id} {:name "External" :is_remote_synced false :location "/"}
+ :model/Card {y-id :id} {:name "Outside Card" :collection_id ext-id
+ :database_id (mt/id) :dataset_query (venues-query)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (seed-create-row! "Collection" c-id)
+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})
+ y-eid (t2/select-one-fn :entity_id :model/Card :id y-id)]
+ (impl/export! (source.p/snapshot mock) (new-task!) "init")
+ (is (not (entity-exported? mock y-eid)) "Y is not in the repo until something references it")
+ (f {:mock mock :c-id c-id :ext-id ext-id :y-id y-id :y-eid y-eid})))))
+
+(deftest update-adding-external-reference-exports-dependency-test
+ (testing "an in-place update that newly references a card outside the synced collection pulls that dependency incrementally"
+ (with-cross-scope-setup!
+ (fn [{:keys [mock c-id y-id y-eid]}]
+ (mt/with-temp [:model/Card {x-id :id} {:name "Synced Card" :collection_id c-id
+ :database_id (mt/id) :dataset_query (venues-query)}]
+ (seed-create-row! "Card" x-id)
+ (impl/export! (source.p/snapshot mock) (new-task!) "add x") ; X now in repo, Y still not
+ ;; Edit X to reference the external card Y, then export the single in-place change.
+ (t2/update! :model/Card x-id {:dataset_query (card-source-query y-id)})
+ (set-status! "Card" x-id "update")
+ (let [task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "reference Y")
+ (is (= "apply-changes-version" (written-version task))
+ "the change stays incremental — the dependency is pulled, not a full export")
+ (is (entity-exported? mock y-eid)
+ "the newly-referenced external dependency is exported alongside the change")))))))
+
+(deftest create-with-external-reference-exports-dependency-test
+ (testing "a new card that references a card outside the synced collection pulls that dependency incrementally"
+ (with-cross-scope-setup!
+ (fn [{:keys [mock c-id y-id y-eid]}]
+ (mt/with-temp [:model/Card {x-id :id} {:name "New Synced Card" :collection_id c-id
+ :database_id (mt/id)
+ :dataset_query (card-source-query y-id)}]
+ (seed-synced-row! "Card" x-id)
+ (set-status! "Card" x-id "create")
+ (let [task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "create x")
+ (is (= "apply-changes-version" (written-version task))
+ "the create stays incremental — the dependency is pulled, not a full export")
+ (is (entity-exported? mock y-eid)
+ "the external dependency referenced by the new card is exported alongside it")))))))
+
+(deftest update-referencing-synced-card-stays-incremental-test
+ (testing "an in-place edit of a card that references a card in another SYNCED collection stays incremental"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {c-id :id} {:name "C" :is_remote_synced true :location "/"}
+ :model/Collection {d-id :id} {:name "D" :is_remote_synced true :location "/"}
+ :model/Card {y-id :id} {:name "Synced Dep" :collection_id d-id
+ :database_id (mt/id) :dataset_query (venues-query)}
+ :model/Card {x-id :id} {:name "Referencing Card" :collection_id c-id
+ :database_id (mt/id)
+ :dataset_query (card-source-query y-id)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (doseq [[model-type model-id] [["Collection" c-id] ["Collection" d-id] ["Card" x-id] ["Card" y-id]]]
+ (seed-create-row! model-type model-id))
+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})
+ y-eid (t2/select-one-fn :entity_id :model/Card :id y-id)]
+ (impl/export! (source.p/snapshot mock) (new-task!) "init")
+ (is (entity-exported? mock y-eid) "the referenced synced card is in the repo")
+ (t2/update! :model/Card x-id {:description "edit"})
+ (set-status! "Card" x-id "update")
+ (let [task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "edit")
+ (is (= "apply-changes-version" (written-version task))
+ "the dependency is already tracked, so the guard allows the incremental path")))))))
+
+;;; ---------------------------------------- Import round-trip (GHY-3725) ----------------------------------------
+;;; The tests above assert the right files land in the repo. These assert the resulting repo actually imports
+;;; back to the intended state — the real point of keeping the repo consistent. `:force? true` bypasses the
+;;; first-import conflict check (the entities already exist locally with these entity_ids).
+
+(deftest incremental-edit-round-trips-through-import-test
+ (testing "a repo produced by an incremental in-place edit imports back to the edited content"
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (t2/update! :model/Card card-a {:description "edited via incremental"})
+ (set-status! "Card" card-a "update")
+ (impl/export! (source.p/snapshot mock) (new-task!) "edit")
+ ;; diverge the DB from the repo, then import the repo back over it
+ (t2/update! :model/Card card-a {:description "local divergence"})
+ (is (= :success (:status (impl/import! (source.p/snapshot mock) (new-task!) :force? true)))
+ "import succeeds")
+ (is (= "edited via incremental" (t2/select-one-fn :description :model/Card :id card-a))
+ "import restored the description the incremental export wrote")))))
+
+(deftest incremental-rename-round-trips-through-import-test
+ (testing "a repo produced by an incremental rename imports to the renamed entity, with no stale duplicate"
+ (with-exported-collection!
+ (fn [{:keys [mock card-a]}]
+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]
+ (t2/update! :model/Card card-a {:name "Renamed A"})
+ (set-status! "Card" card-a "update")
+ (impl/export! (source.p/snapshot mock) (new-task!) "rename")
+ ;; rename back locally, then import the repo
+ (t2/update! :model/Card card-a {:name "Card A"})
+ (is (= :success (:status (impl/import! (source.p/snapshot mock) (new-task!) :force? true)))
+ "import succeeds")
+ (is (= "Renamed A" (t2/select-one-fn :name :model/Card :id card-a))
+ "import applied the renamed name from the repo")
+ (is (= 1 (t2/count :model/Card :entity_id a-eid))
+ "exactly one card for the entity — no duplicate from a stale file"))))))
+
+(deftest two-same-named-creates-use-incremental-path-test
+ (with-exported-collection!
+ (fn [{:keys [mock coll-id]}]
+ ;; Two NEW cards with the same name in one batch. The shared storage context dedups them to
+ ;; distinct paths (foo + foo_2), so neither collides and both are written incrementally — matching
+ ;; a full export. (No prior foo.yaml exists, so path-free-for? can't catch a collision; the shared
+ ;; generator is what keeps them apart.)
+ (mt/with-temp [:model/Card {c1 :id} {:name "Foo" :collection_id coll-id}
+ :model/Card {c2 :id} {:name "Foo" :collection_id coll-id}]
+ (seed-synced-row! "Card" c1)
+ (seed-synced-row! "Card" c2)
+ (set-status! "Card" c1 "create")
+ (set-status! "Card" c2 "create")
+ (let [e1 (t2/select-one-fn :entity_id :model/Card :id c1)
+ e2 (t2/select-one-fn :entity_id :model/Card :id c2)
+ task (new-task!)]
+ (impl/export! (source.p/snapshot mock) task "two foos")
+ (is (= "apply-changes-version" (written-version task))
+ "two same-named creates stay on the incremental path")
+ (is (entity-exported? mock e1) "first Foo exported")
+ (is (entity-exported? mock e2) "second Foo exported")
+ (is (not= (path-for-eid mock e1) (path-for-eid mock e2))
+ "the two Foos get distinct, deduped paths"))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 47395611223..e050062bf14 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -230,6 +230,86 @@
"master2.txt"]
(git/list-files (assoc remote :version "master")))))))))
+(deftest apply-changes
+ (let [subdir (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master remote] (init-source! "master" remote-dir
+ :files {"master.txt" "root file"
+ (str subdir "keep.yaml") "keep me"
+ (str subdir "edit.yaml") "old content"
+ (str subdir "remove.yaml") "delete me"})]
+ (testing "apply-changes! overwrites/adds upserts, removes delete-paths, and PRESERVES every other file"
+ (source.p/apply-changes! (source.p/snapshot master) "Incremental"
+ [{:path (str subdir "edit.yaml") :content "new content"}
+ {:path (str subdir "new.yaml") :content "brand new"}]
+ [(str subdir "remove.yaml")])
+ (is (= ["Incremental" "Initial commit"] (map :message (git/log master))))
+ (let [snap (source.p/snapshot master)]
+ (is (= [(str subdir "edit.yaml")
+ (str subdir "keep.yaml")
+ (str subdir "new.yaml")
+ "master.txt"]
+ (source.p/list-files snap))
+ "edit overwritten + new added, remove deleted; keep.yaml (managed, untouched) and master.txt preserved")
+ (is (= "new content" (source.p/read-file snap (str subdir "edit.yaml"))))
+ (is (= "brand new" (source.p/read-file snap (str subdir "new.yaml"))))
+ (is (= "keep me" (source.p/read-file snap (str subdir "keep.yaml")))
+ "a managed-dir file not in the write set is preserved (unlike write-files!)")
+ (is (= "root file" (source.p/read-file snap "master.txt")))
+ (is (nil? (source.p/read-file snap (str subdir "remove.yaml")))))
+ (testing "the commit was pushed to the remote"
+ (is (= ["Incremental" "Initial commit"]
+ (map :message (git/log (assoc remote :branch "master")))))))))))
+
+(deftest apply-changes-preserves-deep-unchanged-subtree-test
+ (testing "an incremental upsert leaves deeply-nested, unrelated subtrees untouched (carried forward by id)"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"collections/a/deep/nested/keep.yaml" "deep keep"
+ "collections/a/deep/sibling.yaml" "deep sibling"
+ "collections/b/edit.yaml" "old"
+ "notes.txt" "root note"})]
+ (source.p/apply-changes! (source.p/snapshot master) "Incremental deep"
+ [{:path "collections/b/edit.yaml" :content "new"}]
+ [])
+ (let [snap (source.p/snapshot master)]
+ (is (= "new" (source.p/read-file snap "collections/b/edit.yaml")))
+ (is (= "deep keep" (source.p/read-file snap "collections/a/deep/nested/keep.yaml"))
+ "a deeply-nested file in an unrelated subtree is carried forward unchanged")
+ (is (= "deep sibling" (source.p/read-file snap "collections/a/deep/sibling.yaml")))
+ (is (= "root note" (source.p/read-file snap "notes.txt"))))))))
+
+(deftest write-files-reconciles-every-managed-dir-test
+ (testing "a full export wipes every managed dir not covered by the write set, keeps non-managed files, and re-adds an upsert inside a managed dir"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"collections/old/old.yaml" "old collection"
+ "transforms/t1/t.yaml" "a transform"
+ "transforms/t2/t.yaml" "another transform"
+ "notes.txt" "root note"})]
+ (source.p/write-files! (source.p/snapshot master) "Full"
+ [{:path "collections/new/new.yaml" :content "new collection"}])
+ (let [snap (source.p/snapshot master)]
+ (is (= ["collections/new/new.yaml" "notes.txt"] (source.p/list-files snap))
+ "transforms/ (managed, no upserts) fully removed; collections/ reconciled to the write set; non-managed notes.txt preserved")
+ (is (= "new collection" (source.p/read-file snap "collections/new/new.yaml")))
+ (is (nil? (source.p/read-file snap "collections/old/old.yaml")))
+ (is (nil? (source.p/read-file snap "transforms/t1/t.yaml")))
+ (is (= "root note" (source.p/read-file snap "notes.txt"))))))))
+
+(deftest apply-changes-tolerates-missing-delete-path-test
+ (testing "apply-changes! tolerates a delete-path that doesn't exist — the upsert still applies, no error"
+ (mt/with-temp-dir [remote-dir nil]
+ (let [[master _remote] (init-source! "master" remote-dir
+ :files {"collections/a/keep.yaml" "keep"})]
+ (source.p/apply-changes! (source.p/snapshot master) "Delete missing + add"
+ [{:path "collections/a/new.yaml" :content "new"}]
+ ["collections/a/gone.yaml"])
+ (let [snap (source.p/snapshot master)]
+ (is (= ["collections/a/keep.yaml" "collections/a/new.yaml"] (source.p/list-files snap)))
+ (is (= "new" (source.p/read-file snap "collections/a/new.yaml")))
+ (is (= "keep" (source.p/read-file snap "collections/a/keep.yaml"))))))))
+
(deftest write-special-collections
(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]
(mt/with-temp-dir [remote-dir nil]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
index 7bc3bea0a34..62f84a95265 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
@@ -16,6 +16,9 @@
(into {} (map (juxt :path :content) new-files))
"written-files-version")
+ (apply-changes! [_ _message _upserts _delete-paths]
+ "written-files-version")
+
(version [_]
"mock-version"))
@@ -93,67 +96,12 @@
(is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))
"Should return nil for file not matching any pattern"))))
-(deftest wrapping-source-write-files-single-filter-test
- (testing "WrappingSource filters write-files! based on path-filters"
- (let [written-files (atom [])
- mock-snap (reify source.p/SourceSnapshot
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files files)
- nil)
- (version [_]
- "mock-version"))
- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])
- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "collections/bar.yaml" :content "bar-content"}
- {:path "databases/db1.yaml" :content "db1-content"}]]
- (source.p/write-files! wrapped-snap "test commit" files-to-write)
- (is (= [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "collections/bar.yaml" :content "bar-content"}]
- @written-files)
- "Should only write files matching the filter"))))
-
-(deftest wrapping-source-write-files-multiple-filters-test
- (testing "WrappingSource write-files! with multiple filters"
- (let [written-files (atom [])
- mock-snap (reify source.p/SourceSnapshot
- (list-files [_]
- [])
- (read-file [_ _path]
- nil)
- (write-files! [_ _message files]
- (reset! written-files files)
- nil)
- (version [_]
- "mock-version"))
- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*" #"dashboards/.*"])
- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "databases/db1.yaml" :content "db1-content"}
- {:path "dashboards/dash1.yaml" :content "dash1-content"}]]
- (source.p/write-files! wrapped-snap "test commit" files-to-write)
- (is (= [{:path "collections/foo.yaml" :content "foo-content"}
- {:path "dashboards/dash1.yaml" :content "dash1-content"}]
- @written-files)
- "Should write files matching any pattern"))))
-
-(deftest wrapping-source-write-files-no-match-test
- (testing "WrappingSource write-files! with no matching files"
- (let [written-files (atom [])
- mock-snapshot (reify source.p/SourceSnapshot
- (list-files [_]
- [])
- (read-file [_this _path]
- nil)
- (write-files! [_this _message _files]
- nil)
- (version [_this]
- nil))
- files-to-write [{:path "databases/db1.yaml" :content "db1-content"}
- {:path "dashboards/dash1.yaml" :content "dash1-content"}]]
- (source.p/write-files! mock-snapshot "test commit" files-to-write)
- (is (= []
- @written-files)
- "Should write no files when none match"))))
+(deftest wrapping-source-is-read-only-test
+ (testing "WrappingSnapshot is a read-only ingestion view: write operations throw rather than write"
+ (let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"})
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])
+ file-spec {:path "collections/foo.yaml" :content "foo-content"}]
+ (is (thrown? UnsupportedOperationException
+ (source.p/write-files! wrapped-snap "test commit" [file-spec])))
+ (is (thrown? UnsupportedOperationException
+ (source.p/apply-changes! wrapped-snap "test commit" [file-spec] []))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index 16f74a82123..6dbee30f270 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -26,6 +26,10 @@
(reset! written-files {:message message :files realized-files}))
"mock-written-version")
+ (apply-changes! [_ message upserts _delete-paths]
+ (reset! written-files {:message message :files (vec upserts)})
+ "mock-written-version")
+
(version [_]
"mock-version"))
@@ -62,7 +66,7 @@
mock-source (->MockSource written-files)
test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")
(create-test-entity "test-id-2" "entity-two" "Card")]]
- (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))
+ (is (= "mock-written-version" (:version (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message"))))
(testing "write-files! was called with correct message"
(is (= "Test commit message" (:message @written-files))))
(testing "write-files! was called with correct number of files"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 4007d403a99..b2677576450 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -189,8 +189,9 @@ width: fixed
:auth-error (throw (Exception. "Authentication failed"))
:repo-not-found (throw (Exception. "Repository not found"))
:branch-error (throw (Exception. "Invalid branch specified"))
- ;; Default success case - return file content from atom
- (get-in @files-atom [branch path] "")))
+ ;; Default success case - return file content from atom, or nil if absent (matches the real
+ ;; git read-file contract).
+ (get-in @files-atom [branch path])))
(write-files! [_this _message files]
(case fail-mode
@@ -212,6 +213,20 @@ width: fixed
(swap! files-atom assoc branch final-files)))
"write-files-version")
+ (apply-changes! [_this _message upserts delete-paths]
+ (case fail-mode
+ :apply-changes-error (throw (Exception. "Failed to apply changes"))
+ :network-error (throw (java.net.UnknownHostException. "Remote host not found"))
+ ;; Default: overwrite/add upserts, remove delete-paths, preserve every other file.
+ (let [write-entries (remove #(str/blank? (:path %)) upserts)
+ delete-set (into #{} (remove str/blank?) delete-paths)]
+ (swap! files-atom update branch
+ (fn [current]
+ (as-> (or current {}) files
+ (apply dissoc files delete-set)
+ (into files (map (juxt :path :content)) write-entries))))))
+ "apply-changes-version")
+
(version [_this]
"mock-version"))
diff --git a/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yaml b/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yaml
new file mode 100644
index 00000000000..413080d9152
--- /dev/null
+++ b/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yaml
@@ -0,0 +1,22 @@
+## Add file_path column to remote_sync_object for incremental export rename/delete handling (GHY-3725)
+
+databaseChangeLog:
+
+ - changeSet:
+ id: v63.2026-06-01T00:00:00
+ author: ericnormand
+ comment: Add remote_sync_object.file_path to record each entity's last-exported serialized path, so incremental export can resolve old paths for renames and deletes
+ changes:
+ - addColumn:
+ tableName: remote_sync_object
+ columns:
+ - column:
+ name: file_path
+ type: ${text.type}
+ remarks: Repo-relative path of this entity's serialized file as of the last export; used to resolve old paths for incremental renames/deletes
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: remote_sync_object
+ columnName: file_path