The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3725 incremental export fast-path for remote sync (#75057)
Eric Normanddb96fadd290b
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 4310dc1ebcb..64c8d3526e3 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -21,6 +21,7 @@[metabase.util :as u][metabase.util.jvm :as u.jvm][metabase.util.log :as log]+ [metabase.util.yaml :as yaml][toucan2.core :as t2])(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))@@ -221,14 +222,21 @@{:conflicts (vec all-conflicts):summary (into #{} (map :category) all-conflicts)}))-(defn- branch-changed-since-scheduling?- "Returns true if `pre-task-branch` was captured by the async-* function and the- `remote-sync-branch` setting has since drifted to a different value. Used as a- defense-in-depth check against any future code path that bypasses the operation-level- guards and mutates the setting between scheduling and the work running."- [pre-task-branch]- (and (some? pre-task-branch)- (not= pre-task-branch (settings/remote-sync-branch))))+(defn- record-exported-paths!+ "Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and+ deletes can resolve the old path. `entries` is a seq of {:model_type :entity_id :path}; only+ entity-id models are recorded. Correlates entity_id -> model_id per model type."+ [entries]+ (doseq [[model-type es] (group-by :model_type entries)+ :let [spec (spec/spec-for-model-type model-type)]+ :when (and spec (= :entity-id (:identity spec)))]+ (let [eid->path (into {} (map (juxt :entity_id :path)) es)+ id->eid (t2/select-pk->fn :entity_id (:model-key spec) :entity_id [:in (vec (keys eid->path))])]+ (doseq [[id eid] id->eid+ :let [path (eid->path eid)]+ :when path]+ (t2/update! :model/RemoteSyncObject :model_type model-type :model_id id+ {:file_path path})))))(defn import!"Imports and reloads Metabase entities from a remote snapshot.@@ -245,7 +253,8 @@Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may bethrown during import and are caught and converted to error status maps."[^SourceSnapshot snapshot task-id & {:keys [force? pre-task-branch]}]- (when (branch-changed-since-scheduling? pre-task-branch)+ (when (and (some? pre-task-branch)+ (not= pre-task-branch (settings/remote-sync-branch)))(log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"pre-task-branch (settings/remote-sync-branch))(throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"@@ -268,8 +277,8 @@(and first-import? (not force?) (seq conflicts))(u/prog1 {:status :conflict:version (source.p/version snapshot)- :conflicts summary ; Keep backward compatibility: return set of category names- :conflict-details conflicts ; New: detailed conflict info+ :conflicts summary ; Keep backward compatibility: return set of category names+ :conflict-details conflicts ; New: detailed conflict info:message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)}(log/infof (:message <>)))@@ -291,7 +300,10 @@(settings/remote-sync-transforms! true))(t2/with-transaction [_conn](remove-unsynced! (spec/all-syncable-collection-ids) imported-data)- (sync-objects! sync-timestamp imported-data))+ (sync-objects! sync-timestamp imported-data)+ ;; Record the actual repo path each entity was read from, so later renames/deletes+ ;; resolve the real file and stay on the incremental export fast-path.+ (record-exported-paths! (source.ingestable/cached-file-paths base-ingestable)))(when (and (not has-transforms?)(settings/remote-sync-transforms))(log/info "No transforms in remote source, disabling remote-sync-transforms setting")@@ -311,6 +323,193 @@{:status :error:message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."})))+;;; ------------------------------------------- Incremental Export Fast-Path -------------------------------------------++(def ^:private closure-opts+ {:include-field-values false :include-database-secrets false+ :continue-on-error false :skip-archived true})++(defn- export-closure+ "All `[model-type id]` entities a full export would pull for the entity `[model-type model-id]`+ (its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."+ [model-type model-id]+ (keys (merge-with into+ (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))+ (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %))))))++(defn- untracked-content-deps+ "The `[model-type id]` entities in the change's export closure that are remote-sync content but have+ no RemoteSyncObject row — dependencies a full export would pull (via `serdes/descendants`/`required`)+ yet that aren't independently tracked (e.g. a card in a non-synced collection referenced by a synced+ card, and that card's collection). These get written alongside the change so the incremental export+ matches what a full export would emit. Excludes the entity itself and non-content deps (e.g.+ databases, which are resolved by reference at import)."+ [model-type model-id]+ (into #{}+ (filter (fn [[mt id]]+ (and (not (and (= mt model-type) (= id model-id)))+ (spec/spec-for-model-type mt)+ (not (t2/exists? :model/RemoteSyncObject :model_type mt :model_id id)))))+ (export-closure model-type model-id)))++(defn- dep-upserts+ "Serializes the untracked dependency entities `dep-ids` (a set of `[model-type id]`) into file-specs+ to upsert, reusing the storage context `opts` so paths dedupe consistently with the changed entities.+ Returns `[]` when there are no deps, a vector of file-specs on success, or nil if any dependency's+ path is already occupied by a different entity (a collision that requires a full export)."+ [snapshot opts dep-ids]+ (if (empty? dep-ids)+ []+ (let [specs (->> dep-ids+ (map (fn [[mt id]] {:model_type mt :model_id id}))+ spec/extract-entities-for-rows+ (map (fn [e] [(source/entity->file-spec opts e) (:entity_id e)])))]+ ;; keep going only if every dependency's target path is free: absent, or already holding that+ ;; same entity. A present file whose entity_id can't be read counts as occupied — don't clobber it.+ (when (every? (fn [[spec eid]]+ (if-let [content (source.p/read-file snapshot (:path spec))]+ (= eid (try (:entity_id (yaml/parse-string content))+ (catch Exception _ nil)))+ true))+ specs)+ (mapv first specs)))))++(defn- incremental-updates-for-row+ "The updates a single dirty `row` contributes to an incremental plan: a map of+ {:upserts :delete-paths :synced :removed-ids :pull} (any subset), or `:remote-sync/unsyncable-record`+ if the row can't be synced incrementally and the whole batch must fall back to a full export."+ [opts snapshot {:keys [status file_path] :as row}]+ (try+ (let [info (delay (when-let [entity (first (spec/extract-entities-for-rows [row]))]+ (let [spec (source/entity->file-spec opts entity)+ content (source.p/read-file snapshot (:path spec))+ yaml (when content+ ;; errors parsing should throw an invalidate the row+ (yaml/parse-string content))]+ {:entity entity+ :spec spec+ :eid (:entity_id entity)+ :new-path (:path spec)+ :file-exists? (boolean content)+ :file-eid (:entity_id yaml)})))]+ (cond+ (not= :entity-id ;; only entity-id models can be synced incrementally+ (:identity (spec/spec-for-model-type (:model_type row))))+ :remote-sync/unsyncable-record++ (not (#{"create" "update" "removed" "delete"} status))+ :remote-sync/unsyncable-record++ (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export+ (str/blank? file_path))+ :remote-sync/unsyncable-record++ (#{"removed" "delete"} status)+ {:delete-paths [file_path]+ :removed-ids [(:id row)]}++ ;; past here, we're seeing create and update statuses+ (not @info) ;; entity no longer exists+ :remote-sync/unsyncable-record++ ;; create: brand-new file, no old path to delete.+ ;; Target must be free or same entity id+ (and (= "create" status)+ (or (not (:file-exists? @info))+ (= (:eid @info) (:file-eid @info))))+ {:pull (untracked-content-deps (:model_type row) (:model_id row))+ :upserts [(:spec @info)]+ :synced [{:id (:id row) :file_path (:new-path @info)}]}++ ;; in-place update: at its stored path, or (no stored path) the repo file at+ ;; new-path is already this entity — overwrite.+ (and (= "update" status)+ (or (= file_path (:new-path @info))+ (and (str/blank? file_path)+ (= (:eid @info) (:file-eid @info)))))+ {:pull (untracked-content-deps (:model_type row) (:model_id row))+ :upserts [(:spec @info)]+ :synced [{:id (:id row) :file_path (:new-path @info)}]}++ ;; rename: update whose stored path differs from new path. Write the+ ;; new file and delete the old one.+ (and (= "update" status)+ (not (str/blank? file_path))+ (not= file_path (:new-path @info))+ (or (not (:file-exists? @info))+ (= (:eid @info) (:file-eid @info))))+ {:pull (untracked-content-deps (:model_type row) (:model_id row))+ :upserts [(:spec @info)]+ :synced [{:id (:id row) :file_path (:new-path @info)}]+ :delete-paths [file_path]}++ ;; any other create/update case (e.g. the target path is occupied by a different entity) needs+ ;; a full export+ :else+ :remote-sync/unsyncable-record))+ (catch Exception _+ :remote-sync/unsyncable-record)))++(defn- incremental-plan+ "Builds a plan for a safe incremental export of the current `dirty-rows`, or `:remote-sync/unsyncable-batch` if any+ row can't be handled incrementally.++ Every row must be on an entity-id model and be one of:+ - `create`/`update` — re-serialize the entity and upsert its file. A create, or an update whose path+ is unchanged, is an overwrite-in-place; an update whose stored `file_path` differs is a rename, so+ the old file is also deleted. The target path must be free or already hold this same entity, and+ the change must not reference remote-sync content that isn't already in the repo (else a full+ export, which pulls that dependency via `serdes/descendants`, is required).+ - `delete`/`removed` — delete the entity's stored `file_path`. Requires a stored `file_path`+ (after a fresh import none is recorded yet, so we fall back to a full export, which self-heals).++ Returns one of:+ - {:upserts [file-spec] :delete-paths [path] :synced [{:id :file_path}] :removed-ids [id]} — a safe+ incremental plan;+ - `:remote-sync/unsyncable-batch` — a row (or a dependency path collision) forces a full export."+ [snapshot dirty-rows]+ (let [opts (serdes/storage-base-context)+ plan (->> dirty-rows+ (map #(incremental-updates-for-row opts snapshot %))+ (reduce (fn [plan updates]+ (if (= updates :remote-sync/unsyncable-record)+ (reduced :remote-sync/unsyncable-batch)+ (merge-with into plan updates)))+ {:upserts [] :delete-paths [] :synced [] :removed-ids [] :pull #{}}))]+ (if (= plan :remote-sync/unsyncable-batch)+ :remote-sync/unsyncable-batch+ (if-let [deps (dep-upserts snapshot opts (:pull plan))]+ (-> plan (update :upserts into deps) (dissoc :pull))+ ;; a dependency's path is occupied by a different entity — needs a full export+ :remote-sync/unsyncable-batch))))++(defn- path-top-level-dir [^String path]+ (let [i (str/index-of path "/")]+ (if i (subs path 0 i) path)))++(defn- disabled-content-dirs+ "Top-level repo directories whose content is disabled by the current settings."+ []+ (cond-> #{}+ (not (settings/remote-sync-transforms)) (into ["transforms" "python-libraries" "python_libraries"])+ (not (settings/library-is-remote-synced?)) (conj "snippets")))++(defn- full-export!+ "Re-serializes the entire remote-synced set into `snapshot` and commits it, then marks every+ RemoteSyncObject row synced (as of `sync-timestamp`) and records each entity's file_path. Used when+ the pending changes can't be applied incrementally (name collisions, disabled content needing a+ reconcile, etc.). Throws if there is no remote-syncable content. Returns {:status :success}."+ [snapshot task-id message sync-timestamp]+ (let [models (spec/extract-entities-for-export)]+ (when (not models)+ (throw (ex-info "No remote-syncable content available." {})))+ (remote-sync.task/update-progress! task-id 0.3)+ (let [{:keys [version entries]} (source/store! models snapshot task-id message)]+ (remote-sync.task/set-version! task-id version)+ (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})+ (record-exported-paths! entries))+ {:status :success}))+(defn export!"Exports remote-synced collections to a remote source repository.@@ -320,47 +519,70 @@from the current setting at task start, the export aborts with `:error` (defense-in-depthagainst any path that mutates the setting between scheduling and the work running).- Extracts all remote-synced collections, serializes their content, writes the files to the source, and- updates all RemoteSyncObject statuses to 'synced'.+ Takes the incremental fast-path when every pending change can be applied incrementally (see+ `incremental-plan` — creates, in-place updates, renames, and deletes of entity-id models). Otherwise does a+ full export: re-serializes the entire synced set, writes it, marks all RemoteSyncObject rows synced,+ and records each entity's `file_path` so future renames/deletes can go incremental.- Returns a map with :status (either :success or :error), :version, and optionally :message keys. Various- exceptions may be thrown during export and are caught and converted to error status maps."+ Returns a map with {:status :success}, {:status :error}, or throws an exception with an appropriate message."[^SourceSnapshot snapshot task-id message & {:keys [pre-task-branch]}]- (when (branch-changed-since-scheduling? pre-task-branch)+ ;; Defense-in-depth: refuse to run if the branch setting drifted between scheduling and now.+ (when (and (some? pre-task-branch)+ (not= pre-task-branch (settings/remote-sync-branch)))(log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"pre-task-branch (settings/remote-sync-branch))(throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"{:pre-task-branch pre-task-branch:current-branch (settings/remote-sync-branch)})))- (if snapshot- (let [sync-timestamp (t/instant)]- (try- (analytics/inc! :metabase-remote-sync/exports)- (serdes/with-cache- (if-let [models (spec/extract-entities-for-export)]++ (when (not snapshot)+ (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable." {})))++ (let [sync-timestamp (t/instant)]+ (try+ (analytics/inc! :metabase-remote-sync/exports)+ (serdes/with-cache+ (let [disabled-files (filterv (comp (disabled-content-dirs) path-top-level-dir)+ (source.p/list-files snapshot))+ dirty-rows (seq (remote-sync.object/dirty-rows))+ plan (when dirty-rows (incremental-plan snapshot dirty-rows))]+ (cond+ ;; nothing to do: no pending changes and no stale files in now-disabled content dirs+ (and (empty? dirty-rows) (empty? disabled-files))(do+ (log/info "Remote sync export: no changes to export")+ {:status :success})++ ;; a dirty row can't be applied incrementally → full re-serialize+ (= plan :remote-sync/unsyncable-batch)+ (full-export! snapshot task-id message sync-timestamp)++ ;; Incremental fast-path: write only the changed entities, and delete their old paths plus+ ;; any files left behind in now-disabled content dirs — preserving every other file. Avoids+ ;; re-serializing the entire synced set.+ :else+ (let [{:keys [upserts delete-paths synced removed-ids]} plan+ delete-paths (into (vec delete-paths) disabled-files)](remote-sync.task/update-progress! task-id 0.3)- (let [written-version (source/store! models snapshot task-id message)]+ (let [written-version (source.p/apply-changes! snapshot message upserts delete-paths)](remote-sync.task/set-version! task-id written-version))- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})- {:status :success- :version (source.p/version snapshot)})- {:status :error- :message "No remote-syncable content available."}))- (catch Exception e- (if (:cancelled? (ex-data e))- (log/info "Export to git repository was cancelled")- (do- (log/errorf e "Failed to export to git repository: %s" (ex-message e))- (analytics/inc! :metabase-remote-sync/exports-failed)- (remote-sync.task/fail-sync-task! task-id (ex-message e))- {:status :error- :version (source.p/version snapshot)- :message (format "Failed to export to git repository: %s" (ex-message e))})))- (finally- (analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis)))))- {:status :error- :message "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."}))+ (doseq [{:keys [id file_path]} synced]+ (t2/update! :model/RemoteSyncObject :id id+ {:status "synced" :file_path file_path :status_changed_at sync-timestamp}))+ (when (seq removed-ids)+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))+ (log/infof "Remote sync incremental export: wrote %d, deleted %d"+ (count upserts) (count delete-paths))+ {:status :success}))))+ (catch Exception e+ ;; handle-task-result! records the failure on this result, and skips entirely when the task+ ;; was already cancelled (ended_at set) — so cancellation needs no special case here.+ (log/errorf e "Failed to export to git repository: %s" (ex-message e))+ (analytics/inc! :metabase-remote-sync/exports-failed)+ {:status :error+ :message (format "Failed to export to git repository: %s" (ex-message e))})+ (finally+ (analytics/observe! :metabase-remote-sync/export-duration-ms (t/as (t/duration sync-timestamp (t/instant)) :millis))))))(defn create-task-with-lock!"Takes a cluster-wide lock and either returns an existing in-progress RemoteSyncTask ID or creates a new one.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.cljindex b06098c60ce..e8604fea5a0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/models/remote_sync_object.clj@@ -31,28 +31,33 @@:status [:not= "synced"]:model_type [:not-in excluded]))))+(defn dirty-rows+ "Returns the raw RemoteSyncObject rows that are not yet synced (status != 'synced'),+ excluding disabled model types (e.g. transforms when transform sync is off)."+ []+ (let [excluded (spec/excluded-model-types)]+ (if (empty? excluded)+ (t2/select :model/RemoteSyncObject :status [:not= "synced"])+ (t2/select :model/RemoteSyncObject+ :status [:not= "synced"]+ :model_type [:not-in excluded]))))+(defn dirty-objects"Gets all models in any collection that are dirty with their sync status.Returns a sequence of model maps that have changed since the last remote sync,including details about their current state and sync status.Excludes transform model types when transform sync is disabled."[]- (let [excluded (spec/excluded-model-types)- query (if (empty? excluded)- (t2/select :model/RemoteSyncObject :status [:not= "synced"])- (t2/select :model/RemoteSyncObject- :status [:not= "synced"]- :model_type [:not-in excluded]))]- (->> query- (map #(-> %- (dissoc :id :status_changed_at)- (set/rename-keys {:model_id :id- :model_name :name- :model_type :model- :model_collection_id :collection_id- :model_display :display- :model_table_id :table_id- :model_table_name :table_name- :status :sync_status})- (update :model u/lower-case-en)))- (into []))))+ (->> (dirty-rows)+ (map #(-> %+ (dissoc :id :status_changed_at)+ (set/rename-keys {:model_id :id+ :model_name :name+ :model_type :model+ :model_collection_id :collection_id+ :model_display :display+ :model_table_id :table_id+ :model_table_name :table_name+ :status :sync_status})+ (update :model u/lower-case-en)))+ (into [])))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 1602ae72bf0..c55aa157276 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -8,7 +8,6 @@[metabase-enterprise.serialization.core :as serialization][metabase.models.serialization :as serdes][metabase.settings.core :as setting]- [metabase.util :as u][metabase.util.yaml :as yaml][methodical.core :as methodical])(:import@@ -16,8 +15,9 @@(set! *warn-on-reflection* true)-;; Wrapping snapshot accepts a list of path regexes to apply to paths in the source returning-;; nil when they do not match+;; A read-only, path-filtered view over a snapshot, used to scope ingestion to a set of path regexes:+;; files (and reads) outside the filters are omitted. It is not a write target — exports write to the+;; unfiltered source snapshot — so the write methods throw rather than silently filtering writes.(defrecord WrappingSnapshot [original-snapshot path-filters]source.p/SourceSnapshot@@ -30,11 +30,11 @@(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)(source.p/read-file original-snapshot path)))- (write-files! [_ message files]- (source.p/write-files! original-snapshot message- (filter (fn [file-spec]- (some (fn [path-filter] (re-matches path-filter (:path file-spec))) path-filters))- files)))+ (write-files! [_ _message _files]+ (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))++ (apply-changes! [_ _message _upserts _delete-paths]+ (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))(version [_](source.p/version original-snapshot)))@@ -53,15 +53,13 @@basename (str (last resolved) ".yaml")](str/join File/separator (concat dirnames [basename]))))-(defn- ->file-spec- "Converts entity from serdes stream into file spec for source write-files!"- [task-id count opts idx entity]- (when (instance? Exception entity)- (throw entity))- (u/prog1 {:path (remote-sync-path opts entity)- :content (yaml/generate-string (serialization/serialization-deep-sort entity)- {:dumper-options {:flow-style :block :split-lines false}})}- (remote-sync.task/update-progress! task-id (-> (inc idx) (/ count) (* 0.65) (+ 0.3)))))+(defn entity->file-spec+ "Serializes a single extracted entity into a `{:path :content}` file spec, using storage context+ `opts` (from `serdes/storage-base-context`)."+ [opts entity]+ {:path (remote-sync-path opts entity)+ :content (yaml/generate-string (serialization/serialization-deep-sort entity)+ {:dumper-options {:flow-style :block :split-lines false}})})(defn store!"Stores serialized entities from a stream to a remote source and commits the changes.@@ -71,15 +69,22 @@identifier used to track progress updates), and a message (the commit message to use when writingfiles to the source).- Returns the version written to the source.-- Throws Exception if any entity in the stream is an Exception instance."+ Returns `{:version <written-version> :entries [{:model_type :entity_id :path} ...]}`."[stream snapshot task-id message]- (let [opts (serdes/storage-base-context)- stream-count (bounded-count 10000 stream)]- (->> stream- (map-indexed #(->file-spec task-id stream-count opts %1 %2))- (source.p/write-files! snapshot message))))+ (let [opts (serdes/storage-base-context)+ stream-count (bounded-count 10000 stream)+ entries (volatile! [])+ version (->> stream+ (map-indexed (fn [idx entity]+ (let [spec (entity->file-spec opts entity)]+ (vswap! entries conj {:model_type (-> entity :serdes/meta last :model)+ :entity_id (:entity_id entity)+ :path (:path spec)})+ (remote-sync.task/update-progress!+ task-id (-> (min (inc idx) stream-count) (/ stream-count) (* 0.65) (+ 0.3)))+ spec)))+ (source.p/write-files! snapshot message))]+ {:version version :entries @entries}))(defn source-from-settings"Creates a git source from the current remote sync settings.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex 9e34570487d..138c544f7ff 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -14,7 +14,8 @@(java.net URI)(org.apache.commons.io FileUtils)(org.eclipse.jgit.api Git GitCommand TransportCommand)- (org.eclipse.jgit.dircache DirCache DirCacheEntry)+ (org.eclipse.jgit.dircache DirCache DirCacheEditor$DeletePath DirCacheEditor$DeleteTree+ DirCacheEditor$PathEdit DirCacheEntry)(org.eclipse.jgit.lib CommitBuilder Constants FileMode PersonIdent Ref)(org.eclipse.jgit.revwalk RevCommit RevWalk)(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate@@ -233,13 +234,6 @@(throw (ex-info (str "Failed to push branch " branch-name " to remote") {:failures failures})))push-response))-(defn- top-level-dir- "Extracts the first path segment from a file path.- E.g., \"databases/my_db/my_db.yaml\" => \"databases\""- [path]- (when-let [idx (str/index-of path "/")]- (subs path 0 idx)))-(defn default-branch"Retrieves the default branch name of the git repository.@@ -257,60 +251,43 @@(str/replace-first (.getName ^Ref target) "refs/heads/" ""))))(throw (ex-info "Failed to get a default branch for git repository." {:head-ref head-ref})))))-(defn write-files!- "Writes multiple files to the git repository and commits the changes.-- Takes a snapshot map containing a :git Git instance, :version, and :managed-dirs,- a commit message string, and a sequence of file specs (maps with :path and :content keys,- paths should be relative to the repository root).-- All existing files within managed directories that are not in the write set are removed.- This ensures stale files (from renames, moves, or deletions) are cleaned up automatically.- Files outside managed directories are always preserved.-- Returns the version written. Throws ExceptionInfo if the write or push- operation fails."- [{:keys [^Git git ^String version ^String branch managed-dirs] :as snapshot} ^String message files]- (let [repo (.getRepository git)+(defn- commit-edits!+ "Builds a commit by editing the branch tip's tree in place and pushes it. Seeds an in-core index from+ the parent commit's tree — so every unchanged entry and subtree is carried forward by object id — then+ applies only the edits: whole subtrees named in `delete-dirs` are removed (used to fully replace a+ managed directory), paths in `delete-paths` are removed, and `upserts` (file specs with :path/:content)+ are written as new blobs. `writeTree` only rewrites the subtrees on the path to an edit, so the work is+ proportional to the number of changes rather than the size of the repo.++ Returns the new commit sha. Throws ExceptionInfo if the write or push fails."+ [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message upserts delete-paths delete-dirs]+ (let [repo (.getRepository git)branch-ref (qualify-branch branch)- parent-id (.resolve repo version)]- (with-open [inserter (.newObjectInserter repo)]- (let [index (DirCache/newInCore)- builder (.builder index)- ;; Set of all paths being written in this commit- write-paths (into #{}- (comp (map :path)- (remove str/blank?))- files)]- ;; Add new/updated files to the index- (doseq [{:keys [path content]} files- :when (not (str/blank? path))]- (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))- entry (doto (DirCacheEntry. ^String path)- (.setFileMode FileMode/REGULAR_FILE)- (.setObjectId blob-id))]- (.add builder entry)))- ;; Copy existing tree entries that should be preserved:- ;; - Outside managed directories AND not being overwritten by the write set- ;; Files in managed dirs not in write-paths are dropped (stale file cleanup)- ;; Files in write-paths are skipped here (already added above with new content)- (when parent-id- (with-open [rev-walk (RevWalk. repo)- tree-walk (TreeWalk. repo)]- (let [commit (.parseCommit rev-walk parent-id)]- (.addTree tree-walk (.getTree commit))- (.setRecursive tree-walk true)- (while (.next tree-walk)- (let [path (.getPathString tree-walk)]- (when (and (not (contains? write-paths path))- (not (contains? managed-dirs (top-level-dir path))))- (let [entry (doto (DirCacheEntry. path)- (.setFileMode (.getFileMode tree-walk 0))- (.setObjectId (.getObjectId tree-walk 0)))]- (.add builder entry))))))))- (.finish builder)- ;; Create commit- (let [tree-id (.writeTree index inserter)+ parent-id (.resolve repo version)]+ (with-open [inserter (.newObjectInserter repo)+ reader (.newObjectReader repo)+ rev-walk (RevWalk. repo)]+ (let [index (DirCache/newInCore)]+ ;; Seed the index from the parent tree; unchanged entries and subtrees carry forward by id.+ (let [builder (.builder index)]+ (when parent-id+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader+ (.getTree (.parseCommit rev-walk parent-id))))+ (.finish builder))+ ;; Apply only the edits.+ (let [editor (.editor index)]+ (doseq [^String dir delete-dirs]+ (.add editor (DirCacheEditor$DeleteTree. dir)))+ (doseq [^String path delete-paths]+ (.add editor (DirCacheEditor$DeletePath. path)))+ (doseq [{:keys [^String path content]} upserts]+ (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]+ (.add editor (proxy [DirCacheEditor$PathEdit] [path]+ (apply [^DirCacheEntry entry]+ (.setFileMode entry FileMode/REGULAR_FILE)+ (.setObjectId entry blob-id))))))+ (.finish editor))+ (let [tree-id (.writeTree index inserter)commit-builder (doto (CommitBuilder.)(.setTreeId tree-id)(.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))@@ -326,6 +303,17 @@(push-branch! snapshot)(.name commit-id)))))))+(defn- write-files!+ "Full export: replace every managed dir wholesale with `files` (managed-dir files not in `files` are+ removed), preserving everything outside the managed dirs."+ [{:keys [managed-dirs] :as snapshot} ^String message files]+ (commit-edits! snapshot message files nil managed-dirs))++(defn- apply-changes!+ "Incremental patch: write `upserts`, remove `delete-paths`, and preserve every other file."+ [snapshot ^String message upserts delete-paths]+ (commit-edits! snapshot message upserts delete-paths nil))+(defn branches"Retrieves all branch names from the remote repository.@@ -406,6 +394,9 @@(write-files! [this message files](write-files! this message files))+ (apply-changes! [this message upserts delete-paths]+ (apply-changes! this message upserts delete-paths))+(version [this](:version this)))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.cljindex 7e9fbec8c4b..7db471cafe5 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/ingestable.clj@@ -17,8 +17,10 @@(serialization/read-timestamps (yaml/parse-string file-content {:key-fn serialization/parse-key})))(defn- ingest-all- "Returns {:entities {stripped-hierarchy [hierarchy content]}, :errors [Exception...]}.- Dotfiles are silently skipped (editor temp files, see #41567).+ "Returns {:entities {stripped-hierarchy {:content <yaml-string> :path <repo-path>}}, :errors [Exception...]}.+ The repo `:path` is the actual file the entity was read from (including any dedup suffix), so callers+ can record where each entity lives without recomputing — recomputation would diverge on name+ collisions and slug changes. Dotfiles are silently skipped (editor temp files, see #41567).Non-dotfile YAML parse/read failures are collected in :errors."[snapshot](let [errors (atom [])]@@ -39,7 +41,7 @@(swap! errors conj (ex-info (format "Failed to parse file: %s" path) {:file path} e))nil))]:when loaded]- [(serialization/strip-labels loaded) [loaded content]]))+ [(serialization/strip-labels loaded) {:content content :path path}])):errors @errors}));; Wraps another Ingestable calling a callback when a file is ingested@@ -69,8 +71,17 @@(letfn [(progress-callback [item _](when item(let [current-calls (swap! calls inc)]- (t2/with-connection [_conn (app-db/app-db)]- (remote-sync.task/update-progress! task-id (* (/ current-calls total) normalize))))))]+ ;; Progress reporting must never abort the ingestion it tracks. The update runs on a+ ;; separate connection, which on some app DBs can contend with the in-flight load (e.g.+ ;; a MySQL lock-wait timeout), so swallow DB failures and keep going — but still honor a+ ;; cancellation signal, which `update-progress!` raises to stop the task.+ (try+ (t2/with-connection [_conn (app-db/app-db)]+ (remote-sync.task/update-progress! task-id (* (/ current-calls total) normalize)))+ (catch Exception e+ (if (:cancelled? (ex-data e))+ (throw e)+ (log/warn (u/strip-error e "Failed to report import progress; continuing"))))))))](->CallbackIngestable ingestable progress-callback))));; Wraps another Ingestable and filters the `list-files` content to only content that has the specified@@ -128,9 +139,19 @@(populate-cache! cache errors-atom #(ingest-all snapshot))(when-let [target (get @cache (serialization/strip-labels serdes-path))](try- (ingest-content (second target))+ (ingest-content (:content target))(catch Exception e(throw (ex-info "Unable to ingest file" {:abs-path serdes-path} e))))))(ingest-errors [_](or @errors-atom [])))++(defn cached-file-paths+ "Given an `IngestableSnapshot` whose cache has been populated by a prior ingestion, returns a seq of+ {:model_type :entity_id :path} — the actual repo file each entity was read from. Lets the importer+ record `file_path` so later renames and deletes resolve the real file."+ [{:keys [cache]}]+ (for [[hierarchy {:keys [path]}] @cache+ :let [{:keys [model id]} (last hierarchy)]+ :when (and model id path)]+ {:model_type model :entity_id id :path path}))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex 9de80c1daaa..d5ae7b4b6af 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -58,6 +58,16 @@Returns the version of the written files.")+ (apply-changes! [snapshot message upserts delete-paths]+ "Incrementally updates the source: writes/overwrites `upserts`, removes `delete-paths`,+ and PRESERVES every other existing file (unlike `write-files!`, which deletes managed-dir+ files absent from the write set).++ Takes a SourceSnapshot instance, a commit message, `upserts` (a sequence of file specs, each+ a map with :path and :content), and `delete-paths` (a sequence of path strings to remove).++ Returns the version of the written files.")+(version [snapshot]"Gets a version identifier for the current state of the snapshot.diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 87863feba0d..3fbf6bfeebe 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -1145,3 +1145,18 @@:skip-archived true})))cat(u/group-by first second (keys targets))))))++(defn extract-entities-for-rows+ "Serializes the entities named by `rows`, grouped by model type. Each row is a map with a+ `:model_type` (the serdes model name) and a `:model_id` (the local DB primary key). Returns a lazy+ sequence of serialized entities."+ [rows]+ (let [by-model (reduce (fn [m {:keys [model_type model_id]}]+ (update m model_type (fnil conj #{}) model_id))+ {}+ rows)]+ (eduction (map (fn [[model ids]]+ (serdes/extract-all model {:where [:in :id ids]+ :skip-archived true})))+ cat+ by-model)))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.cljindex 853274882e9..358ee828d18 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj@@ -334,8 +334,11 @@(deftest export-handles-write-errors-test(testing "POST /api/ee/remote-sync/export handles write errors"(mt/with-temporary-setting-values [remote-sync-type :read-write]- (mt/with-temp [:model/Collection _ {:is_remote_synced true :name "Test Collection" :location "/"}]- (let [mock-main (test-helpers/create-mock-source :fail-mode :write-files-error)]+ (mt/with-temp [:model/Collection {coll-id :id} {:is_remote_synced true :name "Test Collection" :location "/"}]+ ;; A pending create makes the export take the incremental write path, where the error fires.+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id coll-id :model_name "Test Collection"+ :status "create" :status_changed_at (t/offset-date-time)})+ (let [mock-main (test-helpers/create-mock-source :fail-mode :apply-changes-error)](mt/with-temporary-setting-values [remote-sync-url "https://github.com/test/repo.git"remote-sync-token "test-token"remote-sync-branch "main"]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 9eeb5fa7237..5abd51b37c8 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -166,23 +166,33 @@;; export! tests(deftest export!-with-no-source-configured-test- (testing "export! with no source configured"+ (testing "export! with no source configured throws"(mt/with-temporary-setting-values [remote-sync-type :read-write]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})- result (impl/export! nil task-id "Test commit")]- (is (= :error (:status result)))- (is (re-find #"Remote sync source is not enabled" (:message result)))))))+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Remote sync source is not enabled"+ (impl/export! nil task-id "Test commit")))))))(deftest export!-with-no-remote-synced-collections-test- (testing "export! errors when there are no remote-synced collections"+ (testing "export! is a no-op success when nothing is dirty (no remote-synced content to export)"(mt/with-temporary-setting-values [remote-sync-type :read-write](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}](mt/with-temporary-setting-values [remote-sync-transforms false](let [mock-source (test-helpers/create-mock-source)result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]- (is (= :error (:status result)))- (is (= "No remote-syncable content available." (:message result))))))))))+ (is (= :success (:status result))))))))))++(deftest full-export!-errors-with-no-content-test+ (testing "full-export! throws when there is no remote-syncable content"+ (mt/with-temporary-setting-values [remote-sync-type :read-write]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}]+ (mt/with-temporary-setting-values [remote-sync-transforms false]+ (let [mock-source (test-helpers/create-mock-source)]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"No remote-syncable content available"+ (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant)))))))))))(deftest export!-successful-with-default-collections-test(testing "export! successful with default collections"@@ -194,20 +204,23 @@(is (= :success (:status result)))))))))(deftest export!-handles-store-failure-test- (testing "export! handles store failure"+ (testing "full-export! surfaces a store failure"(mt/with-temporary-setting-values [remote-sync-type :read-write](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})](mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]- (let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")]- (is (= :error (:status result)))- (is (re-find #"Failed to export to git repository" (:message result)))))))))+ (let [mock-source (test-helpers/create-mock-source :fail-mode :store-error)]+ (is (thrown-with-msg? Exception #"Store failed"+ (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit message" (t/instant))))))))))(deftest export!-handles-network-errors-during-write-test- (testing "export! handles network errors during write"+ (testing "export! catches a write failure and returns an :error result"(mt/with-temporary-setting-values [remote-sync-type :read-write](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {_coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}]+ ;; A pending create makes the export take the incremental write path; the network error there+ ;; must surface as an :error result.+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id coll-id :model_name "Test Collection"+ :status "create" :status_changed_at (t/offset-date-time)})(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)result (impl/export! (source.p/snapshot mock-source) task-id "Test commit message")](is (= :error (:status result)))@@ -220,6 +233,12 @@(mt/with-temporary-setting-values [remote-sync-type :read-write](mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}:model/Card {card-id :id} {:name "Test Card" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}]+ ;; Pending creates so the export writes the entities (in production these come from events).+ (t2/insert! :model/RemoteSyncObject+ [{:model_type "Collection" :model_id coll-id :model_name "Test Collection"+ :status "create" :status_changed_at (t/offset-date-time)}+ {:model_type "Card" :model_id card-id :model_name "Test Card"+ :status "create" :status_changed_at (t/offset-date-time)}])(let [mock-main (test-helpers/create-mock-source :branch "test-branch")export-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})export-result (impl/export! (source.p/snapshot mock-main) (:id export-task) "Test export")]@@ -265,6 +284,24 @@(is (not (t2/exists? :model/Collection :id coll2-id)))(is (not (t2/exists? :model/Card :id card2-id))))))))+(deftest import!-records-file-path-test+ (testing "import! records each entity's actual repo file_path on its RemoteSyncObject row, so later+ renames/deletes resolve the real file and stay on the incremental export fast-path"+ (let [import-task (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ coll-path "collections/main/test_collection_1/test_collection_1.yaml"+ card-path "collections/main/test_collection_1/test_card_1.yaml"]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Test Collection 1" :is_remote_synced true :entity_id "test-collection-1xxxx" :location "/"}+ :model/Card {card-id :id} {:name "Test Card 1" :collection_id coll-id :entity_id "test-card-1xxxxxxxxxx"}]+ (let [test-files {"test-branch" {coll-path (test-helpers/generate-collection-yaml "test-collection-1xxxx" "Test Collection 1")+ card-path (test-helpers/generate-card-yaml "test-card-1xxxxxxxxxx" "Test Card 1" "test-collection-1xxxx")}}+ mock-main (test-helpers/create-mock-source :initial-files test-files :branch "test-branch")+ result (impl/import! (source.p/snapshot mock-main) (:id import-task))]+ (is (= :success (:status result)))+ (is (= coll-path (t2/select-one-fn :file_path :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))+ "the collection's row records the file it was imported from")+ (is (= card-path (t2/select-one-fn :file_path :model/RemoteSyncObject :model_type "Card" :model_id card-id))+ "the card's row records the file it was imported from"))))))+(deftest error-handling-propagation-through-private-functions-test(testing "error handling propagation through private functions"(let [mock-source (test-helpers/create-mock-source :fail-mode :network-error)@@ -305,7 +342,7 @@(mt/with-dynamic-fn-redefs [remote-sync.task/update-progress!(fn [task-id progress](swap! progress-calls conj {:task-id task-id :progress progress}))]- (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]+ (let [result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))](is (= :success (:status result)))(is (pos? (count @progress-calls)))(is (= task-id (:task-id (first @progress-calls))))@@ -737,10 +774,8 @@:filter [:> [:field 2 nil] 0]}:entity_id "archived-segment-xxxx":archived true}]- (t2/insert! :model/RemoteSyncObject- [{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])(let [mock-source (test-helpers/create-mock-source)- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]+ result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))](is (= :success (:status result)))(let [files-after-export (get @(:files-atom mock-source) "main")file-keys (keys files-after-export)]@@ -1015,10 +1050,8 @@:table_id table-id:entity_id "archived-measure-xxxx":archived true}]- (t2/insert! :model/RemoteSyncObject- [{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])(let [mock-source (test-helpers/create-mock-source)- result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]+ result (#'impl/full-export! (source.p/snapshot mock-source) task-id "Test commit" (t/instant))](is (= :success (:status result)))(let [files-after-export (get @(:files-atom mock-source) "main")file-keys (keys files-after-export)]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.cljnew file mode 100644index 00000000000..d4dc1d0bc20--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_export_test.clj@@ -0,0 +1,507 @@+(ns metabase-enterprise.remote-sync.incremental-export-test+ "Tests for the incremental export fast-path (GHY-3725). The fast-path writes only the changed+ entities (and deletes only the removed ones) when every pending change can be applied+ incrementally — in-place updates, renames, and deletes of entity-id models whose old path is+ known via the RemoteSyncObject file_path column. Anything else falls back to the full export."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.test-helpers :as rs.test]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.util.yaml :as yaml]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))+(use-fixtures :each rs.test/clean-remote-sync-state)++(defn- venues-query []+ {:database (mt/id) :type :query :query {:source-table (mt/id :venues)}})++(defn- card-source-query [card-id]+ {:database (mt/id) :type :query :query {:source-table (str "card__" card-id)}})++(defn- new-task! []+ (t2/delete! :model/RemoteSyncTask)+ (t2/insert-returning-pk! :model/RemoteSyncTask+ {:sync_task_type "export" :initiated_by (mt/user->id :rasta)}))++(defn- written-version [task-id]+ (t2/select-one-fn :version :model/RemoteSyncTask :id task-id))++(defn- seed-synced-row! [model-type model-id]+ (t2/insert! :model/RemoteSyncObject+ {:model_type model-type+ :model_id model-id+ :model_name (str model-type " " model-id)+ :status "synced"+ :status_changed_at (t/offset-date-time)}))++(defn- set-status! [model-type model-id status]+ (t2/update! :model/RemoteSyncObject :model_type model-type :model_id model-id+ {:status status :status_changed_at (t/offset-date-time)}))++(defn- seed-create-row!+ "Seeds a `create`-status RemoteSyncObject row, so the next export writes the entity (and records its+ file_path). Use when a test needs to get an entity into the repo."+ [model-type model-id]+ (seed-synced-row! model-type model-id)+ (set-status! model-type model-id "create"))++(defn- rso [model-type model-id]+ (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id))++(defn- files [mock] (get @(:files-atom mock) "main"))++(defn- path-for-eid+ "The repo path of the file whose content contains `eid` (filenames are slug-only)."+ [mock eid]+ (some (fn [[p c]] (when (re-find (re-pattern eid) c) p)) (files mock)))++(defn- entity-exported?+ "True if some repo file is the serialized entity with top-level `eid` (i.e. the entity has its own+ file). Distinct from `path-for-eid`, which also matches a mere reference to `eid` inside another+ entity's file."+ [mock eid]+ (boolean (some (fn [[_ c]]+ (try (= eid (:entity_id (yaml/parse-string c)))+ (catch Exception _ false)))+ (files mock))))++(defn- with-exported-collection!+ "Sets up a remote-synced collection with two cards, marks the three entities as new (`create`), and+ runs an initial export into a fresh empty MockSource — writing the files and recording each entity's+ file_path. Then calls `f` with `{:mock :coll-id :card-a :card-b}`."+ [f]+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms false]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Bench" :is_remote_synced true :location "/"}+ :model/Card {card-a :id} {:name "Card A" :collection_id coll-id}+ :model/Card {card-b :id} {:name "Card B" :collection_id coll-id}]+ ;; Reset RSO and seed `create` rows for the three entities (in production these come from events /+ ;; enabling sync) so the initial export writes them and records their file_path.+ (t2/delete! :model/RemoteSyncObject)+ (doseq [[model-type model-id] [["Collection" coll-id] ["Card" card-a] ["Card" card-b]]]+ (seed-create-row! model-type model-id))+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})+ init-task (new-task!)+ init-result (impl/export! (source.p/snapshot mock) init-task "init")]+ (is (= :success (:status init-result)) "initial export succeeds")+ (is (= "apply-changes-version" (written-version init-task))+ "the three new entities are written incrementally")+ (f {:mock mock :coll-id coll-id :card-a card-a :card-b card-b})))))++(deftest full-export-records-file-path-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]+ (is (= (path-for-eid mock a-eid) (:file_path (rso "Card" card-a)))+ "full export records each entity's file_path on its RemoteSyncObject row")))))++(deftest in-place-update-uses-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ a-path (path-for-eid mock a-eid)+ files-before (files mock)]+ (t2/update! :model/Card card-a {:description "edited in place"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "edit")+ files-after (files mock)+ other-path (some (fn [[p _]] (when (not= p a-path) p)) files-before)]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task)) "in-place update is incremental")+ (is (= (set (keys files-before)) (set (keys files-after))) "no files added or removed")+ (is (re-find #"edited in place" (get files-after a-path)) "edit written")+ (is (= (get files-before other-path) (get files-after other-path)) "other files preserved")+ (is (= a-path (:file_path (rso "Card" card-a))) "file_path unchanged")+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))))))))++(deftest multiple-in-place-updates-use-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock card-a card-b]}]+ (t2/update! :model/Card card-a {:description "edit A"})+ (t2/update! :model/Card card-b {:description "edit B"})+ (set-status! "Card" card-a "update")+ (set-status! "Card" card-b "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "edit both")+ files-after (files mock)]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task)))+ (is (some (fn [[_ c]] (re-find #"edit A" c)) files-after))+ (is (some (fn [[_ c]] (re-find #"edit B" c)) files-after))))))++(deftest collection-in-place-edit-uses-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock coll-id]}]+ (t2/update! :model/Collection coll-id {:description "collection note"})+ (set-status! "Collection" coll-id "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "edit collection")]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task))+ "an in-place collection edit (no rename) is incremental")))))++(deftest rename-uses-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ old-path (path-for-eid mock a-eid)]+ (t2/update! :model/Card card-a {:name "Card A Renamed"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "rename")+ files-after (files mock)+ new-path (path-for-eid mock a-eid)]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task)) "rename is incremental")+ (is (not (contains? files-after old-path)) "old file removed")+ (is (not= old-path new-path) "file moved to a new path")+ (is (re-find #"Card A Renamed" (get files-after new-path)) "new file has the new name")+ (is (= 1 (count (filter (fn [[_ c]] (re-find (re-pattern a-eid) c)) files-after)))+ "exactly one file for the entity (no stale duplicate)")+ (is (= new-path (:file_path (rso "Card" card-a))) "file_path updated to the new path"))))))++(deftest delete-uses-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock card-a card-b]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ b-eid (t2/select-one-fn :entity_id :model/Card :id card-b)+ b-path (path-for-eid mock b-eid)]+ ;; A real delete means the entity is archived, so it can't be re-serialized — the incremental+ ;; delete must work from the stored file_path alone, without re-extracting the entity.+ (t2/update! :model/Card card-b {:archived true})+ (set-status! "Card" card-b "delete")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "delete B")+ files-after (files mock)]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task)) "delete is incremental")+ (is (not (contains? files-after b-path)) "deleted entity's file removed")+ (is (some (fn [[_ c]] (re-find (re-pattern a-eid) c)) files-after) "card A preserved")+ (is (nil? (rso "Card" card-b)) "deleted entity's RemoteSyncObject row removed"))))))++(deftest rename-without-stored-path-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ ;; Simulate the post-import state where file_path hasn't been recorded yet: a rename then can't+ ;; resolve the old path, so we fall back to the full export (which reconciles stale files).+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})+ (t2/update! :model/Card card-a {:name "Card A Renamed"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "rename")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "rename without a stored file_path falls back to full")))))++(deftest delete-without-stored-path-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-b]}]+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})+ (set-status! "Card" card-b "delete")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "delete")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "delete without a stored file_path falls back to full")))))++(deftest create-uses-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock coll-id]}]+ (mt/with-temp [:model/Card {card-c :id} {:name "Card C" :collection_id coll-id}]+ (seed-synced-row! "Card" card-c)+ (set-status! "Card" card-c "create")+ (let [files-before (files mock)+ c-eid (t2/select-one-fn :entity_id :model/Card :id card-c)+ task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "add card C")+ (is (= "apply-changes-version" (written-version task)) "a create is incremental")+ (is (entity-exported? mock c-eid) "the new card's file is written")+ (is (some? (:file_path (rso "Card" card-c))) "file_path recorded for the new card")+ (is (every? (fn [[p c]] (= c (get (files mock) p))) files-before)+ "existing files are preserved unchanged")+ (is (not (t2/exists? :model/RemoteSyncObject :status [:not= "synced"]))))))))++(deftest create-with-name-collision-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock coll-id]}]+ ;; A new card whose name collides with an existing card resolves (in isolation) to the existing+ ;; card's path. The safety check sees a different occupant and falls back to full, which dedups.+ (mt/with-temp [:model/Card {card-c :id} {:name "Card A" :collection_id coll-id}]+ (seed-synced-row! "Card" card-c)+ (set-status! "Card" card-c "create")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "collision create")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "a create whose path collides with a different entity falls back to full")+ (is (entity-exported? mock (t2/select-one-fn :entity_id :model/Card :id card-c))+ "the colliding new card is still exported (at a deduped path) by the full export"))))))++(deftest mixed-batch-with-unhandled-row-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-a card-b]}]+ ;; One incremental-able update batched with a delete that has no stored path (so it can't be+ ;; handled incrementally) -> the whole batch falls back to a full export.+ (t2/update! :model/Card card-a {:description "edit A"})+ (set-status! "Card" card-a "update")+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-b {:file_path nil})+ (set-status! "Card" card-b "delete")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "mixed")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "a single non-incremental row forces the whole batch to full export")))))++(deftest name-collision-backfill-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ a-path (path-for-eid mock a-eid)]+ ;; No stored file_path (post-import), and the repo file at the computed path holds a DIFFERENT+ ;; entity (a dedup name collision). The safety check must reject the clobber → full export.+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})+ (swap! (:files-atom mock) assoc-in ["main" a-path]+ (str/replace (get (files mock) a-path) a-eid "different0entity0id0x"))+ (t2/update! :model/Card card-a {:description "edit A"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "collision")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "a path holding a different entity fails the safety check and falls back to full"))))))++(deftest unparseable-occupant-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ a-path (path-for-eid mock a-eid)]+ ;; No stored file_path (post-import), and the repo file at the computed path is present but has+ ;; no readable entity_id. We can't tell whose file it is, so the safety check must treat it as+ ;; occupied — not free — and fall back to full rather than clobber an unidentifiable entity.+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})+ (swap! (:files-atom mock) assoc-in ["main" a-path] "name: Not A Serialized Entity\nother: value\n")+ (t2/update! :model/Card card-a {:description "edit A"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "unparseable occupant")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "a present file with no entity_id fails the safety check and falls back to full"))))))++(deftest malformed-yaml-occupant-falls-back-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ a-path (path-for-eid mock a-eid)]+ ;; No stored file_path, and the repo file at the computed path is malformed YAML that can't be+ ;; parsed. Reading it throws — the row must be treated as unsyncable and fall back to a full+ ;; export rather than crashing the whole export.+ (t2/update! :model/RemoteSyncObject :model_type "Card" :model_id card-a {:file_path nil})+ (swap! (:files-atom mock) assoc-in ["main" a-path] "a: b: c")+ (t2/update! :model/Card card-a {:description "edit A"})+ (set-status! "Card" card-a "update")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "malformed occupant")]+ (is (= :success (:status result)))+ (is (= "write-files-version" (written-version task))+ "a malformed YAML file at the target path falls back to full export instead of crashing"))))))++(deftest create-onto-same-entity-stays-incremental-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ ;; card-a is already exported: its file sits at its computed path holding its own entity_id. A+ ;; fresh "create" whose target path already holds THIS same entity is a safe overwrite, so it must+ ;; stay incremental rather than fall back (the path is occupied, but by us).+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]+ (set-status! "Card" card-a "create")+ (let [task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "re-create A")+ (is (= "apply-changes-version" (written-version task))+ "a create whose path already holds the same entity stays incremental")+ (is (entity-exported? mock a-eid) "card A is still exported"))))))++;;; ------------------------------------ Disabled content cleanup (GHY-3725) ------------------------------------+;;; When a content type is disabled (transforms is off in `with-exported-collection!`), stale files left+;;; behind in its repo dir must be removed — but that shouldn't drop us off the incremental fast-path.+;;; They're appended to the incremental delete-paths instead of forcing a full re-serialize.++(deftest disabled-content-cleaned-up-incrementally-test+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ ;; A stale file in a now-disabled content dir, alongside an ordinary card edit. The edit stays on+ ;; the incremental path AND the stale transforms file is deleted in the same commit.+ (swap! (:files-atom mock) assoc-in ["main" "transforms/old/old.yaml"] "stale transform")+ (t2/update! :model/Card card-a {:description "edit A"})+ (set-status! "Card" card-a "update")+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)+ task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "edit with stale transform")+ (is (= "apply-changes-version" (written-version task))+ "disabled content no longer forces a full export")+ (is (nil? (get (files mock) "transforms/old/old.yaml"))+ "the stale file in the disabled dir is deleted incrementally")+ (is (entity-exported? mock a-eid) "the edited card is still exported")))))++(deftest disabled-content-cleaned-up-with-no-dirty-rows-test+ (with-exported-collection!+ (fn [{:keys [mock]}]+ ;; A stale file in a disabled dir with nothing else dirty: a delete-only incremental commit removes+ ;; it (rather than a no-op that would leave it behind, or a full re-serialize).+ (swap! (:files-atom mock) assoc-in ["main" "transforms/old/old.yaml"] "stale transform")+ (let [task (new-task!)+ result (impl/export! (source.p/snapshot mock) task "remove stale transform")]+ (is (= :success (:status result)))+ (is (= "apply-changes-version" (written-version task))+ "a delete-only incremental commit removes the stale file")+ (is (nil? (get (files mock) "transforms/old/old.yaml")))))))++;;; ---------------------------------- Required-closure regression (GHY-3725) ------------------------------------+;;; A full export pulls a card's source cards via `serdes/descendants` (transitively, through+;;; `resolve-targets`), even when the source card lives outside the synced collection. The incremental+;;; fast-path writes only the changed entities, so a change that introduces such a cross-scope reference+;;; must still export the referenced dependency — otherwise the reference dangles on import.++(defn- with-cross-scope-setup!+ "Synced collection C plus a card Y in a NON-synced collection. Runs an initial full export into a+ fresh MockSource. Calls `f` with `{:mock :c-id :ext-id :y-id :y-eid}`. Y is not referenced yet, so+ it isn't in the repo after the initial export."+ [f]+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms false]+ (mt/with-temp [:model/Collection {c-id :id} {:name "Synced" :is_remote_synced true :location "/"}+ :model/Collection {ext-id :id} {:name "External" :is_remote_synced false :location "/"}+ :model/Card {y-id :id} {:name "Outside Card" :collection_id ext-id+ :database_id (mt/id) :dataset_query (venues-query)}]+ (t2/delete! :model/RemoteSyncObject)+ (seed-create-row! "Collection" c-id)+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})+ y-eid (t2/select-one-fn :entity_id :model/Card :id y-id)]+ (impl/export! (source.p/snapshot mock) (new-task!) "init")+ (is (not (entity-exported? mock y-eid)) "Y is not in the repo until something references it")+ (f {:mock mock :c-id c-id :ext-id ext-id :y-id y-id :y-eid y-eid})))))++(deftest update-adding-external-reference-exports-dependency-test+ (testing "an in-place update that newly references a card outside the synced collection pulls that dependency incrementally"+ (with-cross-scope-setup!+ (fn [{:keys [mock c-id y-id y-eid]}]+ (mt/with-temp [:model/Card {x-id :id} {:name "Synced Card" :collection_id c-id+ :database_id (mt/id) :dataset_query (venues-query)}]+ (seed-create-row! "Card" x-id)+ (impl/export! (source.p/snapshot mock) (new-task!) "add x") ; X now in repo, Y still not+ ;; Edit X to reference the external card Y, then export the single in-place change.+ (t2/update! :model/Card x-id {:dataset_query (card-source-query y-id)})+ (set-status! "Card" x-id "update")+ (let [task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "reference Y")+ (is (= "apply-changes-version" (written-version task))+ "the change stays incremental — the dependency is pulled, not a full export")+ (is (entity-exported? mock y-eid)+ "the newly-referenced external dependency is exported alongside the change")))))))++(deftest create-with-external-reference-exports-dependency-test+ (testing "a new card that references a card outside the synced collection pulls that dependency incrementally"+ (with-cross-scope-setup!+ (fn [{:keys [mock c-id y-id y-eid]}]+ (mt/with-temp [:model/Card {x-id :id} {:name "New Synced Card" :collection_id c-id+ :database_id (mt/id)+ :dataset_query (card-source-query y-id)}]+ (seed-synced-row! "Card" x-id)+ (set-status! "Card" x-id "create")+ (let [task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "create x")+ (is (= "apply-changes-version" (written-version task))+ "the create stays incremental — the dependency is pulled, not a full export")+ (is (entity-exported? mock y-eid)+ "the external dependency referenced by the new card is exported alongside it")))))))++(deftest update-referencing-synced-card-stays-incremental-test+ (testing "an in-place edit of a card that references a card in another SYNCED collection stays incremental"+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms false]+ (mt/with-temp [:model/Collection {c-id :id} {:name "C" :is_remote_synced true :location "/"}+ :model/Collection {d-id :id} {:name "D" :is_remote_synced true :location "/"}+ :model/Card {y-id :id} {:name "Synced Dep" :collection_id d-id+ :database_id (mt/id) :dataset_query (venues-query)}+ :model/Card {x-id :id} {:name "Referencing Card" :collection_id c-id+ :database_id (mt/id)+ :dataset_query (card-source-query y-id)}]+ (t2/delete! :model/RemoteSyncObject)+ (doseq [[model-type model-id] [["Collection" c-id] ["Collection" d-id] ["Card" x-id] ["Card" y-id]]]+ (seed-create-row! model-type model-id))+ (let [mock (rs.test/create-mock-source :initial-files {"main" {}})+ y-eid (t2/select-one-fn :entity_id :model/Card :id y-id)]+ (impl/export! (source.p/snapshot mock) (new-task!) "init")+ (is (entity-exported? mock y-eid) "the referenced synced card is in the repo")+ (t2/update! :model/Card x-id {:description "edit"})+ (set-status! "Card" x-id "update")+ (let [task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "edit")+ (is (= "apply-changes-version" (written-version task))+ "the dependency is already tracked, so the guard allows the incremental path")))))))++;;; ---------------------------------------- Import round-trip (GHY-3725) ----------------------------------------+;;; The tests above assert the right files land in the repo. These assert the resulting repo actually imports+;;; back to the intended state — the real point of keeping the repo consistent. `:force? true` bypasses the+;;; first-import conflict check (the entities already exist locally with these entity_ids).++(deftest incremental-edit-round-trips-through-import-test+ (testing "a repo produced by an incremental in-place edit imports back to the edited content"+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (t2/update! :model/Card card-a {:description "edited via incremental"})+ (set-status! "Card" card-a "update")+ (impl/export! (source.p/snapshot mock) (new-task!) "edit")+ ;; diverge the DB from the repo, then import the repo back over it+ (t2/update! :model/Card card-a {:description "local divergence"})+ (is (= :success (:status (impl/import! (source.p/snapshot mock) (new-task!) :force? true)))+ "import succeeds")+ (is (= "edited via incremental" (t2/select-one-fn :description :model/Card :id card-a))+ "import restored the description the incremental export wrote")))))++(deftest incremental-rename-round-trips-through-import-test+ (testing "a repo produced by an incremental rename imports to the renamed entity, with no stale duplicate"+ (with-exported-collection!+ (fn [{:keys [mock card-a]}]+ (let [a-eid (t2/select-one-fn :entity_id :model/Card :id card-a)]+ (t2/update! :model/Card card-a {:name "Renamed A"})+ (set-status! "Card" card-a "update")+ (impl/export! (source.p/snapshot mock) (new-task!) "rename")+ ;; rename back locally, then import the repo+ (t2/update! :model/Card card-a {:name "Card A"})+ (is (= :success (:status (impl/import! (source.p/snapshot mock) (new-task!) :force? true)))+ "import succeeds")+ (is (= "Renamed A" (t2/select-one-fn :name :model/Card :id card-a))+ "import applied the renamed name from the repo")+ (is (= 1 (t2/count :model/Card :entity_id a-eid))+ "exactly one card for the entity — no duplicate from a stale file"))))))++(deftest two-same-named-creates-use-incremental-path-test+ (with-exported-collection!+ (fn [{:keys [mock coll-id]}]+ ;; Two NEW cards with the same name in one batch. The shared storage context dedups them to+ ;; distinct paths (foo + foo_2), so neither collides and both are written incrementally — matching+ ;; a full export. (No prior foo.yaml exists, so path-free-for? can't catch a collision; the shared+ ;; generator is what keeps them apart.)+ (mt/with-temp [:model/Card {c1 :id} {:name "Foo" :collection_id coll-id}+ :model/Card {c2 :id} {:name "Foo" :collection_id coll-id}]+ (seed-synced-row! "Card" c1)+ (seed-synced-row! "Card" c2)+ (set-status! "Card" c1 "create")+ (set-status! "Card" c2 "create")+ (let [e1 (t2/select-one-fn :entity_id :model/Card :id c1)+ e2 (t2/select-one-fn :entity_id :model/Card :id c2)+ task (new-task!)]+ (impl/export! (source.p/snapshot mock) task "two foos")+ (is (= "apply-changes-version" (written-version task))+ "two same-named creates stay on the incremental path")+ (is (entity-exported? mock e1) "first Foo exported")+ (is (entity-exported? mock e2) "second Foo exported")+ (is (not= (path-for-eid mock e1) (path-for-eid mock e2))+ "the two Foos get distinct, deduped paths"))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 47395611223..e050062bf14 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -230,6 +230,86 @@"master2.txt"](git/list-files (assoc remote :version "master")))))))))+(deftest apply-changes+ (let [subdir (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")]+ (mt/with-temp-dir [remote-dir nil]+ (let [[master remote] (init-source! "master" remote-dir+ :files {"master.txt" "root file"+ (str subdir "keep.yaml") "keep me"+ (str subdir "edit.yaml") "old content"+ (str subdir "remove.yaml") "delete me"})]+ (testing "apply-changes! overwrites/adds upserts, removes delete-paths, and PRESERVES every other file"+ (source.p/apply-changes! (source.p/snapshot master) "Incremental"+ [{:path (str subdir "edit.yaml") :content "new content"}+ {:path (str subdir "new.yaml") :content "brand new"}]+ [(str subdir "remove.yaml")])+ (is (= ["Incremental" "Initial commit"] (map :message (git/log master))))+ (let [snap (source.p/snapshot master)]+ (is (= [(str subdir "edit.yaml")+ (str subdir "keep.yaml")+ (str subdir "new.yaml")+ "master.txt"]+ (source.p/list-files snap))+ "edit overwritten + new added, remove deleted; keep.yaml (managed, untouched) and master.txt preserved")+ (is (= "new content" (source.p/read-file snap (str subdir "edit.yaml"))))+ (is (= "brand new" (source.p/read-file snap (str subdir "new.yaml"))))+ (is (= "keep me" (source.p/read-file snap (str subdir "keep.yaml")))+ "a managed-dir file not in the write set is preserved (unlike write-files!)")+ (is (= "root file" (source.p/read-file snap "master.txt")))+ (is (nil? (source.p/read-file snap (str subdir "remove.yaml")))))+ (testing "the commit was pushed to the remote"+ (is (= ["Incremental" "Initial commit"]+ (map :message (git/log (assoc remote :branch "master")))))))))))++(deftest apply-changes-preserves-deep-unchanged-subtree-test+ (testing "an incremental upsert leaves deeply-nested, unrelated subtrees untouched (carried forward by id)"+ (mt/with-temp-dir [remote-dir nil]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"collections/a/deep/nested/keep.yaml" "deep keep"+ "collections/a/deep/sibling.yaml" "deep sibling"+ "collections/b/edit.yaml" "old"+ "notes.txt" "root note"})]+ (source.p/apply-changes! (source.p/snapshot master) "Incremental deep"+ [{:path "collections/b/edit.yaml" :content "new"}]+ [])+ (let [snap (source.p/snapshot master)]+ (is (= "new" (source.p/read-file snap "collections/b/edit.yaml")))+ (is (= "deep keep" (source.p/read-file snap "collections/a/deep/nested/keep.yaml"))+ "a deeply-nested file in an unrelated subtree is carried forward unchanged")+ (is (= "deep sibling" (source.p/read-file snap "collections/a/deep/sibling.yaml")))+ (is (= "root note" (source.p/read-file snap "notes.txt"))))))))++(deftest write-files-reconciles-every-managed-dir-test+ (testing "a full export wipes every managed dir not covered by the write set, keeps non-managed files, and re-adds an upsert inside a managed dir"+ (mt/with-temp-dir [remote-dir nil]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"collections/old/old.yaml" "old collection"+ "transforms/t1/t.yaml" "a transform"+ "transforms/t2/t.yaml" "another transform"+ "notes.txt" "root note"})]+ (source.p/write-files! (source.p/snapshot master) "Full"+ [{:path "collections/new/new.yaml" :content "new collection"}])+ (let [snap (source.p/snapshot master)]+ (is (= ["collections/new/new.yaml" "notes.txt"] (source.p/list-files snap))+ "transforms/ (managed, no upserts) fully removed; collections/ reconciled to the write set; non-managed notes.txt preserved")+ (is (= "new collection" (source.p/read-file snap "collections/new/new.yaml")))+ (is (nil? (source.p/read-file snap "collections/old/old.yaml")))+ (is (nil? (source.p/read-file snap "transforms/t1/t.yaml")))+ (is (= "root note" (source.p/read-file snap "notes.txt"))))))))++(deftest apply-changes-tolerates-missing-delete-path-test+ (testing "apply-changes! tolerates a delete-path that doesn't exist — the upsert still applies, no error"+ (mt/with-temp-dir [remote-dir nil]+ (let [[master _remote] (init-source! "master" remote-dir+ :files {"collections/a/keep.yaml" "keep"})]+ (source.p/apply-changes! (source.p/snapshot master) "Delete missing + add"+ [{:path "collections/a/new.yaml" :content "new"}]+ ["collections/a/gone.yaml"])+ (let [snap (source.p/snapshot master)]+ (is (= ["collections/a/keep.yaml" "collections/a/new.yaml"] (source.p/list-files snap)))+ (is (= "new" (source.p/read-file snap "collections/a/new.yaml")))+ (is (= "keep" (source.p/read-file snap "collections/a/keep.yaml"))))))))+(deftest write-special-collections(let [subdir-path (str "collections/" "r" (subs (u/generate-nano-id "a") 1) "_subdir/")](mt/with-temp-dir [remote-dir nil]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.cljindex 7bc3bea0a34..62f84a95265 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj@@ -16,6 +16,9 @@(into {} (map (juxt :path :content) new-files))"written-files-version")+ (apply-changes! [_ _message _upserts _delete-paths]+ "written-files-version")+(version [_]"mock-version"))@@ -93,67 +96,12 @@(is (nil? (source.p/read-file wrapped-snap "databases/db1.yaml"))"Should return nil for file not matching any pattern"))))-(deftest wrapping-source-write-files-single-filter-test- (testing "WrappingSource filters write-files! based on path-filters"- (let [written-files (atom [])- mock-snap (reify source.p/SourceSnapshot- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files files)- nil)- (version [_]- "mock-version"))- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*"])- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}- {:path "collections/bar.yaml" :content "bar-content"}- {:path "databases/db1.yaml" :content "db1-content"}]]- (source.p/write-files! wrapped-snap "test commit" files-to-write)- (is (= [{:path "collections/foo.yaml" :content "foo-content"}- {:path "collections/bar.yaml" :content "bar-content"}]- @written-files)- "Should only write files matching the filter"))))--(deftest wrapping-source-write-files-multiple-filters-test- (testing "WrappingSource write-files! with multiple filters"- (let [written-files (atom [])- mock-snap (reify source.p/SourceSnapshot- (list-files [_]- [])- (read-file [_ _path]- nil)- (write-files! [_ _message files]- (reset! written-files files)- nil)- (version [_]- "mock-version"))- wrapped-snap (source/->WrappingSnapshot mock-snap [#"collections/.*" #"dashboards/.*"])- files-to-write [{:path "collections/foo.yaml" :content "foo-content"}- {:path "databases/db1.yaml" :content "db1-content"}- {:path "dashboards/dash1.yaml" :content "dash1-content"}]]- (source.p/write-files! wrapped-snap "test commit" files-to-write)- (is (= [{:path "collections/foo.yaml" :content "foo-content"}- {:path "dashboards/dash1.yaml" :content "dash1-content"}]- @written-files)- "Should write files matching any pattern"))))--(deftest wrapping-source-write-files-no-match-test- (testing "WrappingSource write-files! with no matching files"- (let [written-files (atom [])- mock-snapshot (reify source.p/SourceSnapshot- (list-files [_]- [])- (read-file [_this _path]- nil)- (write-files! [_this _message _files]- nil)- (version [_this]- nil))- files-to-write [{:path "databases/db1.yaml" :content "db1-content"}- {:path "dashboards/dash1.yaml" :content "dash1-content"}]]- (source.p/write-files! mock-snapshot "test commit" files-to-write)- (is (= []- @written-files)- "Should write no files when none match"))))+(deftest wrapping-source-is-read-only-test+ (testing "WrappingSnapshot is a read-only ingestion view: write operations throw rather than write"+ (let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"})+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])+ file-spec {:path "collections/foo.yaml" :content "foo-content"}]+ (is (thrown? UnsupportedOperationException+ (source.p/write-files! wrapped-snap "test commit" [file-spec])))+ (is (thrown? UnsupportedOperationException+ (source.p/apply-changes! wrapped-snap "test commit" [file-spec] []))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex 16f74a82123..6dbee30f270 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -26,6 +26,10 @@(reset! written-files {:message message :files realized-files}))"mock-written-version")+ (apply-changes! [_ message upserts _delete-paths]+ (reset! written-files {:message message :files (vec upserts)})+ "mock-written-version")+(version [_]"mock-version"))@@ -62,7 +66,7 @@mock-source (->MockSource written-files)test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")(create-test-entity "test-id-2" "entity-two" "Card")]]- (is (= "mock-written-version" (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message")))+ (is (= "mock-written-version" (:version (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message"))))(testing "write-files! was called with correct message"(is (= "Test commit message" (:message @written-files))))(testing "write-files! was called with correct number of files"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 4007d403a99..b2677576450 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -189,8 +189,9 @@ width: fixed:auth-error (throw (Exception. "Authentication failed")):repo-not-found (throw (Exception. "Repository not found")):branch-error (throw (Exception. "Invalid branch specified"))- ;; Default success case - return file content from atom- (get-in @files-atom [branch path] "")))+ ;; Default success case - return file content from atom, or nil if absent (matches the real+ ;; git read-file contract).+ (get-in @files-atom [branch path])))(write-files! [_this _message files](case fail-mode@@ -212,6 +213,20 @@ width: fixed(swap! files-atom assoc branch final-files)))"write-files-version")+ (apply-changes! [_this _message upserts delete-paths]+ (case fail-mode+ :apply-changes-error (throw (Exception. "Failed to apply changes"))+ :network-error (throw (java.net.UnknownHostException. "Remote host not found"))+ ;; Default: overwrite/add upserts, remove delete-paths, preserve every other file.+ (let [write-entries (remove #(str/blank? (:path %)) upserts)+ delete-set (into #{} (remove str/blank?) delete-paths)]+ (swap! files-atom update branch+ (fn [current]+ (as-> (or current {}) files+ (apply dissoc files delete-set)+ (into files (map (juxt :path :content)) write-entries))))))+ "apply-changes-version")+(version [_this]"mock-version"))diff --git a/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yaml b/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yamlnew file mode 100644index 00000000000..413080d9152--- /dev/null+++ b/resources/migrations/063/20260601_ghy_3725_remote_sync_object_file_path.yaml@@ -0,0 +1,22 @@+## Add file_path column to remote_sync_object for incremental export rename/delete handling (GHY-3725)++databaseChangeLog:++ - changeSet:+ id: v63.2026-06-01T00:00:00+ author: ericnormand+ comment: Add remote_sync_object.file_path to record each entity's last-exported serialized path, so incremental export can resolve old paths for renames and deletes+ changes:+ - addColumn:+ tableName: remote_sync_object+ columns:+ - column:+ name: file_path+ type: ${text.type}+ remarks: Repo-relative path of this entity's serialized file as of the last export; used to resolve old paths for incremental renames/deletes+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: remote_sync_object+ columnName: file_path