Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Track remote-sync dirty state by serialized content hash (GHY-3933) (#76098)

Eric Normand585f612d7492
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index df5ec237ba7..1f5511b7f07 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -17,6 +17,7 @@
- NativeQuerySnippet, snippets-namespace Collections (when Library is remote-synced)"
(:require
[java-time.api :as t]
+ [metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.spec :as spec]
[metabase.collections.core :as collections]
[metabase.events.core :as events]
@@ -24,42 +25,57 @@
[methodical.core :as methodical]
[toucan2.core :as t2]))
-(defn sync-snippet-tracking!
- "Called when the Library collection's remote sync status changes.
- When enabled: mark all existing snippets and snippets-namespace collections
- as 'create' for initial sync.
- When disabled: remove all snippet-related tracking entries."
- [enabled?]
- (let [timestamp (t/offset-date-time)]
- (if enabled?
- (do
- ;; Mark all snippets-namespace collections for initial sync
- (doseq [coll (t2/select [:model/Collection :id :name] :namespace "snippets")]
- (t2/insert! :model/RemoteSyncObject
- {:model_type "Collection"
- :model_id (:id coll)
- :model_name (:name coll)
- :status "create"
- :status_changed_at timestamp}))
- ;; Mark all existing snippets for initial sync
- (doseq [snippet (t2/select [:model/NativeQuerySnippet :id :name :collection_id])]
- (t2/insert! :model/RemoteSyncObject
- {:model_type "NativeQuerySnippet"
- :model_id (:id snippet)
- :model_name (:name snippet)
- :model_collection_id (:collection_id snippet)
- :status "create"
- :status_changed_at timestamp})))
- (let [snippet-coll-ids (t2/select-pks-set :model/Collection :namespace "snippets")]
- (t2/delete! :model/RemoteSyncObject
- :model_type "NativeQuerySnippet")
- (when (seq snippet-coll-ids)
- (t2/delete! :model/RemoteSyncObject
- :model_type "Collection"
- :model_id [:in snippet-coll-ids]))))))
+(defn enable-snippet-tracking!
+ "Mark all existing snippets and snippets-namespace collections as 'create' for initial sync."
+ []
+ (let [timestamp (t/offset-date-time)
+ rows (concat
+ (for [coll (t2/select [:model/Collection :id :name] :namespace "snippets")]
+ {:model_type "Collection"
+ :model_id (:id coll)
+ :model_name (:name coll)
+ :status "create"
+ :status_changed_at timestamp})
+ (for [snippet (t2/select [:model/NativeQuerySnippet :id :name :collection_id])]
+ {:model_type "NativeQuerySnippet"
+ :model_id (:id snippet)
+ :model_name (:name snippet)
+ :model_collection_id (:collection_id snippet)
+ :status "create"
+ :status_changed_at timestamp}))]
+ (when (seq rows)
+ (t2/insert! :model/RemoteSyncObject rows))))
+
+(defn disable-snippet-tracking!
+ "Remove all snippet-related tracking entries."
+ []
+ (let [snippet-coll-ids (t2/select-pks-set :model/Collection :namespace "snippets")]
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "NativeQuerySnippet")
+ (when (seq snippet-coll-ids)
+ (t2/delete! :model/RemoteSyncObject
+ :model_type "Collection"
+ :model_id [:in snippet-coll-ids]))))
;;; ----------------------------------------- Helper Functions ---------------------------------------------------------
+(defn- resolve-status
+ "Suppresses a no-op 'update' based on status and content_hash, otherwise keep status unchanged."
+ [model-type model-id status existing]
+ (cond
+ (not= "update" status)
+ status
+
+ (nil? (:content_hash existing))
+ status
+
+ (not= (:content_hash existing)
+ (source/row->content-hash {:model_type model-type :model_id model-id}))
+ status
+
+ :else ;; hash has not changed
+ "synced"))
+
(defn- create-or-update-remote-sync-object-entry!
"Creates or updates a remote sync object entry for a model change.
@@ -93,7 +109,7 @@
(not (= "create" (:status existing)))
(let [model-details (hydrate-details-fn model-id)]
(t2/update! :model/RemoteSyncObject (:id existing)
- {:status status
+ {:status (resolve-status model-type model-id status existing)
:status_changed_at (t/offset-date-time)
:model_name (:name model-details)
:model_collection_id (:collection_id model-details)
@@ -129,7 +145,7 @@
(let [model-details (spec/hydrate-model-details model-spec model-id)
fields (spec/build-sync-object-fields model-spec model-details)]
(t2/update! :model/RemoteSyncObject (:id existing)
- (merge {:status status
+ (merge {:status (resolve-status model-type model-id status existing)
:status_changed_at (t/offset-date-time)}
fields))))))
@@ -227,11 +243,11 @@
(and is-now-synced? (not snippets-already-tracked?))
(do
(log/info "Library collection became remote-synced, enabling snippet sync tracking")
- (sync-snippet-tracking! true))
+ (enable-snippet-tracking!))
(and (not is-now-synced?) snippets-already-tracked?)
(do
(log/info "Library collection is no longer remote-synced, disabling snippet sync tracking")
- (sync-snippet-tracking! false)))))
+ (disable-snippet-tracking!)))))
(methodical/defmethod events/publish-event! ::collection-change-event
[topic event]
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index a6218510f9d..07b27426cc7 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -55,19 +55,6 @@
(spec/transforms-namespace-collection? entity))))
serdes-paths))))
-(defn- sync-objects!
- "Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and
- inserts new ones for each imported entity, marking them as 'synced' with the given timestamp.
-
- Takes a timestamp instant and imported-data map from spec/extract-imported-entities.
-
- Uses the spec system to determine how to query and build sync objects for each model type."
- [timestamp imported-data]
- (t2/delete! :model/RemoteSyncObject)
- (let [all-inserts (spec/sync-all-entities! timestamp imported-data)]
- (when (seq all-inserts)
- (t2/insert! :model/RemoteSyncObject all-inserts))))
-
(defn- build-entity-id-where-clause
"Builds a HoneySQL WHERE clause for entity_id filtering.
Combines the imported entity-ids exclusion with any spec-level entity_id condition."
@@ -219,21 +206,65 @@
{:first-import-conflicts (vec first-import-conflicts)
:deletion-conflicts (spec/check-deletion-conflicts imported-data)}))
-(defn- record-exported-paths!
- "Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and
- deletes can resolve the old path. `entries` is a seq of {:model_type :entity_id :path}; only
- entity-id models are recorded. Correlates entity_id -> model_id per model type."
- [entries]
- (doseq [[model-type es] (group-by :model_type entries)
- :let [spec (spec/spec-for-model-type model-type)]
- :when (and spec (= :entity-id (:identity spec)))]
- (let [eid->path (into {} (map (juxt :entity_id :path)) es)
- id->eid (t2/select-pk->fn :entity_id (:model-key spec) :entity_id [:in (vec (keys eid->path))])]
- (doseq [[id eid] id->eid
- :let [path (eid->path eid)]
- :when path]
- (t2/update! :model/RemoteSyncObject :model_type model-type :model_id id
- {:file_path path})))))
+(def app-db-batch-size
+ "Max rows per select/update batch, to keep IN-lists and CASE expressions bounded."
+ 500)
+
+(defn- import-content-metadata
+ "Content-hash entries for imported `rows` ({:model_type :model_id}), re-serializing each entity once to hash
+ it. `repo-paths` supplies the real repo path for entity-id models (else the freshly computed one).
+
+ Returns:
+ - [{:model_type :model_id :path :content_hash}] (entities that fail to serialize are omitted)"
+ [rows repo-paths]
+ (let [storage-opts (serdes/storage-base-context)
+ repo-by-eid (into {} (map (fn [{:keys [model_type entity_id path]}] [[model_type entity_id] path])) repo-paths)
+ serialize (fn [model-type opts id->eid instance]
+ (try
+ (let [fspec (source/entity->file-spec storage-opts (serdes/extract-one model-type opts instance))
+ repo-path (some->> (some-> id->eid (get (:id instance)))
+ (vector model-type)
+ repo-by-eid)]
+ {:model_type model-type
+ :model_id (:id instance)
+ :path (or repo-path (:path fspec))
+ :content_hash (source/content-hash (:content fspec))})
+ (catch Exception e
+ (log/warnf e "Skipping %s %s: failed to serialize for content hash" model-type (:id instance))
+ nil)))]
+ ;; One transduction over the model groups: stream each model's extract-query through `serialize` via an
+ ;; eduction — extract-one runs while the ResultSet is open, with no intermediate per-model sequence.
+ (into []
+ (mapcat (fn [[model-type model-rows]]
+ (let [spec (spec/spec-for-model-type model-type)
+ model-key (:model-key spec)
+ opts {:where [:in :id (mapv :model_id model-rows)] :skip-archived true}
+ ;; entity-id models: map local id -> entity_id so we can look up the repo path
+ id->eid (when (and model-key (= :entity-id (:identity spec)))
+ (t2/select-pk->fn :entity_id model-key :id [:in (mapv :model_id model-rows)]))]
+ (eduction (keep #(serialize model-type opts id->eid %))
+ (serdes/extract-query model-type opts)))))
+ (group-by :model_type rows))))
+
+(defn- merge-content-metadata
+ "Folds `metadata` ({:model_type :model_id :path :content_hash}) into matching `rows` as :file_path +
+ :content_hash; rows with no metadata are unchanged."
+ [rows metadata]
+ (let [by-key (u/index-by (juxt :model_type :model_id) metadata)]
+ (mapv (fn [row]
+ (if-let [m (by-key [(:model_type row) (:model_id row)])]
+ (assoc row :file_path (:path m) :content_hash (:content_hash m))
+ row))
+ rows)))
+
+(defn- insert-with-metadata!
+ "Inserts RemoteSyncObject `rows` after an import, one `content-hash-batch-size` chunk at a time, folding
+ each chunk's file_path + content_hash (`repo-paths` gives entity-id models their real path) into its insert."
+ [rows repo-paths]
+ (serdes/with-cache
+ (doseq [chunk (partition-all app-db-batch-size rows)]
+ (t2/insert! :model/RemoteSyncObject
+ (merge-content-metadata chunk (import-content-metadata chunk repo-paths))))))
(defn- branch-changed-since-scheduling?
"Returns true if `pre-task-branch` was captured by the async-* function and the
@@ -273,10 +304,12 @@
(settings/remote-sync-transforms! true))
(t2/with-transaction [_conn]
(remove-unsynced! (spec/all-syncable-collection-ids) imported-data)
- (sync-objects! sync-timestamp imported-data)
- ;; Record the actual repo path each entity was read from, so later renames/deletes resolve the
- ;; real file and stay on the incremental export fast-path.
- (record-exported-paths! (source.ingestable/cached-file-paths base-ingestable))
+ ;; Replace the RemoteSyncObject table, folding each entity's repo file_path (so later renames/deletes
+ ;; resolve the real file) and serialized-content hash (so a post-pull no-op edit stays synced) into the
+ ;; insert. Chunked so insert/IN params and memory stay bounded.
+ (t2/delete! :model/RemoteSyncObject)
+ (insert-with-metadata! (spec/sync-all-entities! sync-timestamp imported-data)
+ (source.ingestable/cached-file-paths base-ingestable))
(when finalize! (finalize!)))
(when (and (not has-transforms?)
(settings/remote-sync-transforms))
@@ -287,11 +320,6 @@
;;; ------------------------------------------- Incremental Import Fast-Path -------------------------------------------
-(def ^:private incremental-not-possible
- "Sentinel returned by [[incremental-load-snapshot!]] when a change can't be applied incrementally and the
- caller must run the full [[load-snapshot!]] instead."
- :remote-sync/incremental-import-not-possible)
-
(def ^:private full-import-models
"Models whose change forces a full import on the incremental fast-path: Collection (a rename moves every
descendant's file, a delete cascades to its contents) and the feature models (their presence drives the
@@ -339,16 +367,16 @@
all-models (into add-models (map :model_type) deleted-rsos)]
(cond
(nil? changed) ;; first import, or diffing not possible (force-push/rebase, non-diffable source)
- incremental-not-possible
+ :remote-sync/incremental-not-possible
(some nil? deleted-rsos) ;; some deleted file not present in appdb
- incremental-not-possible
+ :remote-sync/incremental-not-possible
(some full-import-models all-models) ;; anything not incrementally importable by model type?
- incremental-not-possible
+ :remote-sync/incremental-not-possible
(some #(not= :entity-id (:identity (spec/spec-for-model-type %))) all-models) ;; anything not entity-id model?
- incremental-not-possible
+ :remote-sync/incremental-not-possible
:else
{:ingestable ingestable :deleted-rsos deleted-rsos})))
@@ -390,10 +418,9 @@
(when (seq deletes)
(t2/delete! :model/RemoteSyncObject {:where (rso-where-for deletes)}))
(when (seq sync-rows)
+ ;; fold file_path + content_hash into the insert so the touched rows are written once (chunked)
(t2/delete! :model/RemoteSyncObject {:where (rso-where-for sync-rows)})
- (t2/insert! :model/RemoteSyncObject sync-rows))
- (when ingestable
- (record-exported-paths! (source.ingestable/cached-file-paths ingestable)))
+ (insert-with-metadata! sync-rows (when ingestable (source.ingestable/cached-file-paths ingestable))))
(when finalize! (finalize!)))
;; We skip the whole-appdb reindex the full load runs. Added/modified entities are already
;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index
@@ -427,51 +454,27 @@
(-> row (dissoc :id) (assoc :status_changed_at timestamp))))))
(defn- import-merged!
- "Local-only merge for the pull path. Performs an entity-identity 3-way merge of local state against the
- remote tip and, on a clean merge, applies the merged result to the LOCAL app DB only — it does not push.
- Remote-originated changes end up 'synced'; the un-pushed local changes are preserved as dirty so they
- can be pushed later. Sets the last version to the remote tip (so the branch is no longer 'behind').
-
- On a genuine same-entity conflict, returns `:conflict` without touching local state.
-
- `base-snapshot` is nil only when no safe 3-way merge is possible (no prior sync, or the base commit was
- orphaned by a force-push/rebase) — that surfaces as a conflict. When the remote has not advanced past
- the base, there is nothing to fold in, so it is a no-op success that keeps local changes dirty."
+ "Import in merge mode. Should only be called when you have a base-snapshot and its version differs from snaphot's version.
+
+ Entity-identity 3-way merge of local state against the remote tip:
+ - on same-entity conflict, returns `:conflict` without touching local state;
+ - on a clean merge, applies the merged result to the LOCAL app DB only (no push).
+ - Marks remote changes synced
+ - Local changes stay dirty
+ - Sets version to remote tip"
[snapshot base-snapshot task-id sync-timestamp]
- (let [merge-result (delay (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id))]
- (cond
- (nil? base-snapshot)
- {:status :conflict
- :version (source.p/version snapshot)
- :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
- :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}
-
- ;; The remote hasn't advanced past the merge base — nothing to fold in. Keep local changes as-is (still
- ;; dirty, to be pushed later); a no-op success, not a spurious "history was rewritten" conflict.
- (= (source.p/version base-snapshot) (source.p/version snapshot))
- (do
- (log/info "Pull merge: remote has not advanced; keeping local changes unchanged")
- {:status :success
- :version (source.p/version snapshot)
- :merge-summary {:added 0 :updated 0 :removed 0}
- :outcome {:kind "pull-skipped"}})
-
- (seq (:conflicts @merge-result))
- (let [conflicts (mapv remote-sync.merge/conflict-label (:conflicts @merge-result))]
- (log/infof "Pull merge conflict on %d entit(ies): %s"
- (count conflicts) (str/join ", " conflicts))
+ (let [{:keys [conflicts merged summary]} (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id)]
+ (if (seq conflicts)
+ (let [labels (mapv remote-sync.merge/conflict-label conflicts)]
+ (log/infof "Pull merge conflict on %d entit(ies): %s" (count labels) (str/join ", " labels))
{:status :conflict
:version (source.p/version snapshot)
- :conflicts conflicts
+ :conflicts labels
:message "Import blocked: the same content was changed both locally and on the remote branch."})
-
- :else
- ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are
- ;; disjoint from the remote changes, so restoring them reproduces exactly the local diff vs remote.
- ;; Restore + set-version run inside the load's transaction so that a crash can't leave the dirty
- ;; markers overwritten by the load (which would lose them — a retry captures nothing).
- (let [{:keys [merged summary]} @merge-result
- dirty-objects (capture-dirty-objects)]
+ ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are disjoint
+ ;; from the remote changes, so restoring them reproduces exactly the local diff vs remote. Restore +
+ ;; set-version run inside the load's transaction so a crash can't leave the dirty markers overwritten.
+ (let [dirty-objects (capture-dirty-objects)]
(load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp
:finalize! (fn []
(restore-dirty-objects! dirty-objects sync-timestamp)
@@ -479,39 +482,29 @@
(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"
(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)
(count dirty-objects))
- {:status :success
- :version (source.p/version snapshot)
+ {:status :success
+ :version (source.p/version snapshot)
:merge-summary summary
- :outcome {:kind "pulled"
- :count (apply + (vals summary))
- :branch (settings/remote-sync-branch)}}))))
+ :outcome {:kind "pulled"
+ :count (apply + (vals summary))
+ :branch (settings/remote-sync-branch)}}))))
(defn import!
"Imports and reloads Metabase entities from a remote snapshot.
Takes a non-nil SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:
+ - :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local
+ - :base-snapshot - the merge base (last synced version), required when :merge? is set
- :force? - forces import even when the snapshot version matches the last imported version or has first-import conflicts
- :force-deletion? - forces past deletion conflicts (unsynced local transforms the import would delete); defaults
- to :force?. Kept separate so the setup/enable flow can force past version/dirty guards (force? true) while still
- surfacing transform-deletion as a conflict the admin must confirm (force-deletion? false) — see GHY-3900.
- - :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local
- - :base-snapshot - the merge base (last synced version), required when :merge? is set
- - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs
- from the current setting at task start, the import aborts with `:error` to protect data
- integrity (the load mutates the app DB, so we refuse to proceed when state has drifted)
+ to :force?.
Loads serialized entities, removes entities not in the import, syncs the remote-sync-object table, and
optionally creates a remote-synced collection.
Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may be
- thrown during import and are caught and converted to error status maps."
- [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot pre-task-branch]}]
- (when (branch-changed-since-scheduling? pre-task-branch)
- (log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"
- pre-task-branch (settings/remote-sync-branch))
- (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
- {:pre-task-branch pre-task-branch
- :current-branch (settings/remote-sync-branch)})))
+ thrown during import."
+ [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot]}]
(log/info "Reloading remote entities from the remote source")
(analytics/inc! :metabase-remote-sync/imports)
(let [sync-timestamp (t/instant)]
@@ -539,21 +532,64 @@
incremental-plan (delay (incremental-import-plan snapshot last-imported-version))
dirty? (delay (remote-sync.object/dirty?))]
(cond
+ ;; --- Merge mode: fold remote changes into local, keeping un-pushed local changes. ---
merge?
- (import-merged! snapshot base-snapshot task-id sync-timestamp)
+ (cond
+ ;; No safe merge base (no prior sync, or base orphaned by a force-push/rebase) — can't 3-way merge.
+ (nil? base-snapshot)
+ {:status :conflict
+ :version snapshot-version
+ :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
+ :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}
+
+ ;; Remote hasn't advanced past the merge base — nothing to fold in; keep local changes dirty.
+ (= (source.p/version base-snapshot) snapshot-version)
+ (do
+ (log/info "Pull merge: remote has not advanced; keeping local changes unchanged")
+ {:status :success
+ :version snapshot-version
+ :merge-summary {:added 0 :updated 0 :removed 0}
+ :outcome {:kind "pull-skipped"}})
+
+ :else
+ (import-merged! snapshot base-snapshot task-id sync-timestamp))
- ;; Cheap no-op pull: nothing changed remotely, so nothing is loaded or deleted
- (and (not force?) (= last-imported-version snapshot-version))
+ ;; --- Forced reload: bypasses the no-op/incremental guards. Deletion conflicts (when
+ ;; force-deletion? is false) still block; otherwise a full reload. ---
+ force?
+ (cond
+ (seq @blocking-conflicts)
+ (let [message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)]
+ (log/info message)
+ {:status :conflict
+ :version snapshot-version
+ :conflicts (into #{} (map :category) @blocking-conflicts)
+ :conflict-details @blocking-conflicts
+ :message message})
+
+ :else
+ (let [_ (log/info "Remote sync full import: forced")
+ imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]
+ (log/info "Successfully reloaded entities from git repository")
+ {:status :success
+ :version snapshot-version
+ :outcome {:kind "pulled"
+ :count (pulled-change-count imported-data)
+ :branch (settings/remote-sync-branch)}}))
+
+ ;; --- Normal pull ---
+ ;; Cheap no-op pull: nothing changed remotely, so nothing is loaded or deleted.
+ (= last-imported-version snapshot-version)
(do
(log/infof "Skipping import: snapshot version %s matches last imported version" snapshot-version)
{:status :success
:version snapshot-version
:outcome {:kind "pull-skipped"}})
- ;; Incremental fast-path: not forced, no local drift, and the change is incrementally loadable.
- ;; Touches only changed files, so it can't wholesale-delete unsynced transforms — no conflict scan,
- ;; and only the cheap O(changes) diff is computed (never the full-tree get-conflicts read).
- (and (not force?) (not @dirty?) (not= incremental-not-possible @incremental-plan))
+ ;; Incremental fast-path: no local drift and the change is incrementally loadable. Tested before the
+ ;; conflict. It touches only changed files, so it can't wholesale-delete unsynced transforms.
+ (and (not @dirty?)
+ (not= :remote-sync/incremental-not-possible @incremental-plan))
(incremental-load-snapshot! @incremental-plan snapshot-version task-id sync-timestamp :finalize! finalize!)
(seq @blocking-conflicts)
@@ -565,10 +601,8 @@
:conflict-details @blocking-conflicts
:message message})
- ;; No blocking conflicts: do the full reload.
- :else
+ :else ;; fall back to full import
(let [reason (cond
- force? "forced"
@dirty? "local changes pending"
first-import? "first import"
:else "changes not incrementally loadable")
@@ -637,7 +671,7 @@
:pushed pushed-count
:branch (settings/remote-sync-branch)}})
;; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —
- ;; write-files! updates the local ref before returning). Fail loudly rather than silently advancing
+ ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing
;; the version while skipping the reconcile, which would leave local missing the folded-in remote
;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.
(throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "
@@ -651,159 +685,220 @@
{:include-field-values false :include-database-secrets false
:continue-on-error false :skip-archived true})
+(defn- merge-incremental-export-plans-reducer [a b]
+ (if (or (= :remote-sync/incremental-not-possible a)
+ (= :remote-sync/incremental-not-possible b))
+ (reduced :remote-sync/incremental-not-possible)
+ (merge-with into a b)))
+
+(defn- merge-incremental-export-plans [a b]
+ (unreduced (merge-incremental-export-plans-reducer a b)))
+
(defn- export-closure
- "All `[model-type id]` entities a full export would pull for the entity `[model-type model-id]`
+ "All `{:model_type :model_id}` entities a full export would pull for the entity `[model-type model-id]`
(its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."
[model-type model-id]
- (keys (merge-with into
- (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))
- (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %))))))
+ ;; serdes works in [model-type id] tuples; map the closure keys to {:model_type :model_id} on the way out
+ (map (fn [[mt id]] {:model_type mt :model_id id})
+ (keys (merge-with into
+ (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))
+ (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))))
(defn- untracked-content-deps
- "The `[model-type id]` entities in the change's export closure that are remote-sync content but have
- no RemoteSyncObject row — dependencies a full export would pull (via `serdes/descendants`/`required`)
- yet that aren't independently tracked (e.g. a card in a non-synced collection referenced by a synced
- card, and that card's collection). These get written alongside the change so the incremental export
- matches what a full export would emit. Excludes the entity itself and non-content deps (e.g.
- databases, which are resolved by reference at import)."
+ "The `{:model_type :model_id}` entities in `[model-type model-id]`'s export closure that are remote-sync
+ content but have no RemoteSyncObject row — untracked deps an incremental export must write to match a full
+ export. Excludes the entity itself and non-content deps (e.g. databases)."
[model-type model-id]
(into #{}
- (filter (fn [[mt id]]
- (and (not (and (= mt model-type) (= id model-id)))
- (spec/spec-for-model-type mt)
- (not (t2/exists? :model/RemoteSyncObject :model_type mt :model_id id)))))
+ (filter (fn [{:keys [model_type model_id]}]
+ (and (not (and (= model_type model-type) (= model_id model-id)))
+ (spec/spec-for-model-type model_type)
+ (not (t2/exists? :model/RemoteSyncObject :model_type model_type :model_id model_id)))))
(export-closure model-type model-id)))
-(defn- dep-upserts
- "Serializes the untracked dependency entities `dep-ids` (a set of `[model-type id]`) into file-specs
- to upsert, reusing the storage context `opts` so paths dedupe consistently with the changed entities.
- Returns `[]` when there are no deps, a vector of file-specs on success, or nil if any dependency's
- path is already occupied by a different entity (a collision that requires a full export)."
+(defn- ->sized-chunks
+ "Builds chunks of maximum size based on model type."
+ [rows]
+ (for [[model-type rows] (group-by :model_type rows)
+ chunk-rows (partition-all app-db-batch-size rows)]
+ {:model_type model-type :rows chunk-rows}))
+
+(defn- extract-chunk
+ "Extract one chunk's entities in a single query.
+
+ Argument:
+ - {:model_type .. :rows [{:model_type .. :model_id ..}]}
+
+ Return:
+ - [[row entity]] (if no entity, then omit)"
+ [{:keys [model_type rows]}]
+ (let [id->row (u/index-by :model_id rows)
+ opts {:where [:in :id (mapv :model_id rows)] :skip-archived true}]
+ ;; extract-one must run inside the extract-query reduction, while its ResultSet is open
+ (into [] (keep (fn [instance]
+ (when-let [row (id->row (:id instance))]
+ [row (serdes/extract-one model_type opts instance)])))
+ (serdes/extract-query model_type opts))))
+
+(defn- path-free?
+ "Is this path free to write the entity with eid to?
+
+ Argument:
+ - {:eid .. :file-exists? .. :file-eid}
+
+ Return:
+ - boolean"
+ [{:keys [eid file-exists? file-eid]}]
+ (or (not file-exists?) (= eid file-eid)))
+
+(defn- entity-path-info
+ "Info for deciding where an entity goes: `{:eid :new-path :file-exists? :file-eid}` — the entity's id, its
+ target `path`, and whether a file is already there (`:file-eid` is that file's entity_id, nil if absent or
+ unreadable)."
+ [snapshot path eid]
+ (let [content (source.p/read-file snapshot path)]
+ {:eid eid
+ :new-path path
+ :file-exists? (boolean content)
+ :file-eid (when content (try (:entity_id (yaml/parse-string content))
+ (catch Exception _ nil)))}))
+
+(defn- dependency->incremental-export-plan [snapshot opts [row entity]]
+ (let [path (source/entity->path opts entity)]
+ (if (path-free? (entity-path-info snapshot path (:entity_id entity)))
+ {:writes [(assoc row :file_path path)]}
+ :remote-sync/incremental-not-possible)))
+
+(defn- dependency-chunk->incremental-export-plan
+ "Plan fragment ({:writes [{:model_type :model_id :file_path}]}) for one chunk of dependency rows, or
+ `:remote-sync/incremental-not-possible` if any target path collides with a different entity."
+ [snapshot opts chunk]
+ (->> chunk
+ (extract-chunk)
+ (map #(dependency->incremental-export-plan snapshot opts %))
+ (reduce merge-incremental-export-plans-reducer {})))
+
+(defn- dependencies->incremental-export-plan
+ "Plan fragment ({:writes [...]}) for the untracked dependency entities `dep-ids` ({:model_type :model_id}),
+ one chunk at a time, or `:remote-sync/incremental-not-possible` if any target path collides."
[snapshot opts dep-ids]
- (if (empty? dep-ids)
- []
- (let [specs (->> dep-ids
- (map (fn [[mt id]] {:model_type mt :model_id id}))
- spec/extract-entities-for-rows
- (map (fn [e] [(source/entity->file-spec opts e) (:entity_id e)])))]
- ;; keep going only if every dependency's target path is free: absent, or already holding that
- ;; same entity. A present file whose entity_id can't be read counts as occupied — don't clobber it.
- (when (every? (fn [[spec eid]]
- (if-let [content (source.p/read-file snapshot (:path spec))]
- (= eid (try (:entity_id (yaml/parse-string content))
- (catch Exception _ nil)))
- true))
- specs)
- (mapv first specs)))))
-
-(defn- incremental-updates-for-row
- "The updates a single dirty `row` contributes to an incremental plan: a map of
- {:upserts :delete-paths :synced :removed-ids :pull} (any subset), or `:remote-sync/unsyncable-record`
- if the row can't be synced incrementally and the whole batch must fall back to a full export."
- [opts snapshot {:keys [status file_path] :as row}]
+ (->> dep-ids
+ (->sized-chunks)
+ (map #(dependency-chunk->incremental-export-plan snapshot opts %))
+ (reduce merge-incremental-export-plans-reducer {})))
+
+(defn- row->incremental-export-plan
+ "Decide a `row`'s contribution to an incremental plan.
+
+ Return:
+ - `:remote-sync/incremental-not-possible` OR
+ - {:writes :delete-paths :removed-ids :pull}"
+ [{:keys [id model_type model_id status file_path]} file-info]
(try
- (let [info (delay (when-let [entity (first (spec/extract-entities-for-rows [row]))]
- (let [spec (source/entity->file-spec opts entity)
- content (source.p/read-file snapshot (:path spec))
- yaml (when content
- ;; errors parsing should throw an invalidate the row
- (yaml/parse-string content))]
- {:entity entity
- :spec spec
- :eid (:entity_id entity)
- :new-path (:path spec)
- :file-exists? (boolean content)
- :file-eid (:entity_id yaml)})))]
- (cond
- (not= :entity-id ;; only entity-id models can be synced incrementally
- (:identity (spec/spec-for-model-type (:model_type row))))
- :remote-sync/unsyncable-record
-
- (not (#{"create" "update" "removed" "delete"} status))
- :remote-sync/unsyncable-record
-
- (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export
- (str/blank? file_path))
- :remote-sync/unsyncable-record
-
- (#{"removed" "delete"} status)
- {:delete-paths [file_path]
- :removed-ids [(:id row)]}
-
- ;; past here, we're seeing create and update statuses
- (not @info) ;; entity no longer exists
- :remote-sync/unsyncable-record
-
- ;; create: brand-new file, no old path to delete.
- ;; Target must be free or same entity id
- (and (= "create" status)
- (or (not (:file-exists? @info))
- (= (:eid @info) (:file-eid @info))))
- {:pull (untracked-content-deps (:model_type row) (:model_id row))
- :upserts [(:spec @info)]
- :synced [{:id (:id row) :file_path (:new-path @info)}]}
-
- ;; in-place update: at its stored path, or (no stored path) the repo file at
- ;; new-path is already this entity — overwrite.
- (and (= "update" status)
- (or (= file_path (:new-path @info))
- (and (str/blank? file_path)
- (= (:eid @info) (:file-eid @info)))))
- {:pull (untracked-content-deps (:model_type row) (:model_id row))
- :upserts [(:spec @info)]
- :synced [{:id (:id row) :file_path (:new-path @info)}]}
-
- ;; rename: update whose stored path differs from new path. Write the
- ;; new file and delete the old one.
- (and (= "update" status)
- (not (str/blank? file_path))
- (not= file_path (:new-path @info))
- (or (not (:file-exists? @info))
- (= (:eid @info) (:file-eid @info))))
- {:pull (untracked-content-deps (:model_type row) (:model_id row))
- :upserts [(:spec @info)]
- :synced [{:id (:id row) :file_path (:new-path @info)}]
- :delete-paths [file_path]}
-
- ;; any other create/update case (e.g. the target path is occupied by a different entity) needs
- ;; a full export
- :else
- :remote-sync/unsyncable-record))
+ (cond
+ (not= :entity-id ;; only entity-id models can be synced incrementally
+ (:identity (spec/spec-for-model-type model_type)))
+ :remote-sync/incremental-not-possible
+
+ (not (#{"create" "update" "removed" "delete"} status))
+ :remote-sync/incremental-not-possible
+
+ (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export
+ (str/blank? file_path))
+ :remote-sync/incremental-not-possible
+
+ (#{"removed" "delete"} status)
+ {:delete-paths [file_path]
+ :removed-ids [id]}
+
+ ;; past here, we're seeing create and update statuses
+ (not file-info) ;; entity no longer exists
+ :remote-sync/incremental-not-possible
+
+ ;; create: brand-new file, no old path to delete.
+ ;; Target must be free or same entity id
+ (and (= "create" status)
+ (path-free? file-info))
+ {:pull (untracked-content-deps model_type model_id)
+ :writes [{:id id
+ :model_type model_type
+ :model_id model_id
+ :file_path (:new-path file-info)}]}
+
+ ;; in-place update: at its stored path, or (no stored path) the repo file at
+ ;; new-path is already this entity — overwrite.
+ (and (= "update" status)
+ (or (= file_path (:new-path file-info))
+ (and (str/blank? file_path)
+ (= (:eid file-info) (:file-eid file-info)))))
+ {:pull (untracked-content-deps model_type model_id)
+ :writes [{:id id
+ :model_type model_type
+ :model_id model_id
+ :file_path (:new-path file-info)}]}
+
+ ;; rename: update whose stored path differs from new path. Write the
+ ;; new file and delete the old one.
+ (and (= "update" status)
+ (not (str/blank? file_path))
+ (not= file_path (:new-path file-info))
+ (path-free? file-info))
+ {:pull (untracked-content-deps model_type model_id)
+ :writes [{:id id
+ :model_type model_type
+ :model_id model_id
+ :file_path (:new-path file-info)}]
+ :delete-paths [file_path]}
+
+ ;; any other create/update case (e.g. the target path is occupied by a different entity) needs
+ ;; a full export
+ :else
+ :remote-sync/incremental-not-possible)
(catch Exception _
- :remote-sync/unsyncable-record)))
-
-(defn- incremental-plan
- "Builds a plan for a safe incremental export of the current `dirty-rows`, or `:remote-sync/unsyncable-batch` if any
- row can't be handled incrementally.
-
- Every row must be on an entity-id model and be one of:
- - `create`/`update` — re-serialize the entity and upsert its file. A create, or an update whose path
- is unchanged, is an overwrite-in-place; an update whose stored `file_path` differs is a rename, so
- the old file is also deleted. The target path must be free or already hold this same entity, and
- the change must not reference remote-sync content that isn't already in the repo (else a full
- export, which pulls that dependency via `serdes/descendants`, is required).
- - `delete`/`removed` — delete the entity's stored `file_path`. Requires a stored `file_path`
- (after a fresh import none is recorded yet, so we fall back to a full export, which self-heals).
-
- Returns one of:
- - {:upserts [file-spec] :delete-paths [path] :synced [{:id :file_path}] :removed-ids [id]} — a safe
- incremental plan;
- - `:remote-sync/unsyncable-batch` — a row (or a dependency path collision) forces a full export."
- [snapshot dirty-rows]
- (let [opts (serdes/storage-base-context)
- plan (->> dirty-rows
- (map #(incremental-updates-for-row opts snapshot %))
- (reduce (fn [plan updates]
- (if (= updates :remote-sync/unsyncable-record)
- (reduced :remote-sync/unsyncable-batch)
- (merge-with into plan updates)))
- {:upserts [] :delete-paths [] :synced [] :removed-ids [] :pull #{}}))]
- (if (= plan :remote-sync/unsyncable-batch)
- :remote-sync/unsyncable-batch
- (if-let [deps (dep-upserts snapshot opts (:pull plan))]
- (-> plan (update :upserts into deps) (dissoc :pull))
- ;; a dependency's path is occupied by a different entity — needs a full export
- :remote-sync/unsyncable-batch))))
+ :remote-sync/incremental-not-possible)))
+
+(defn- chunk->incremental-export-plan
+ "Plan fragment for one chunk of create/update rows.
+
+ Returns:
+ - :remote-sync/incremental-not-possible when the chunk can't be synced
+ - {:writes :delete-paths :removed-ids :pull}"
+ [snapshot opts chunk]
+ (let [found (extract-chunk chunk)]
+ (if (< (count found) (count (:rows chunk)))
+ :remote-sync/incremental-not-possible ; extract-chunk omits gone entities; some row is unsyncable
+ (->> found
+ (map (fn [[row entity]]
+ (row->incremental-export-plan row (entity-path-info snapshot (source/entity->path opts entity) (:entity_id entity)))))
+ (reduce merge-incremental-export-plans-reducer {})))))
+
+(defn- incremental-export-plan
+ "Build an incremental export plan for `rows` (`RemoteSyncObject`s) against `snapshot`, one chunk at a time.
+
+ Returns:
+ - {:writes [{:id :model_type :model_id :file_path}] :delete-paths [path] :removed-ids [id]}, or
+ - :remote-sync/incremental-not-possible when any row can't go incrementally"
+ [snapshot rows]
+ (let [opts (serdes/storage-base-context)
+ ;; create/update rows on entity-id models need an entity (extracted per chunk); everything else
+ ;; (removed/delete, non-entity-id, bad status) is decided with no entity
+ {cu-rows true other-rows false} (group-by #(boolean (and (#{"create" "update"} (:status %))
+ (= :entity-id (:identity (spec/spec-for-model-type (:model_type %))))))
+ rows)
+ ;; the no-extraction rows first (cheap)
+ plan (->> other-rows
+ (map #(row->incremental-export-plan % nil))
+ (reduce merge-incremental-export-plans-reducer {:writes [] :delete-paths [] :removed-ids [] :pull #{}}))
+ plan (->> cu-rows
+ (->sized-chunks)
+ (map #(chunk->incremental-export-plan snapshot opts %))
+ (reduce merge-incremental-export-plans-reducer plan))
+ plan (->> (:pull plan) ;; nil when plan is already :incremental-not-possible
+ (dependencies->incremental-export-plan snapshot opts)
+ (merge-incremental-export-plans plan))]
+ (if (= plan :remote-sync/incremental-not-possible)
+ :remote-sync/incremental-not-possible
+ (dissoc plan :pull))))
(defn- path-top-level-dir [^String path]
(let [i (str/index-of path "/")]
@@ -816,64 +911,148 @@
(not (settings/remote-sync-transforms)) (into ["transforms" "python-libraries" "python_libraries"])
(not (settings/library-is-remote-synced?)) (conj "snippets")))
+(defn- stage-write [commit opts [row entity]]
+ (let [path (or (:file_path row) (source/entity->path opts entity))
+ content (source/entity->content entity)]
+ (source.p/stage-upsert! commit {:path path :content content})
+ (when (:id row)
+ [{:id (:id row) :file_path path :content_hash (source/content-hash content)}])))
+
+(defn- chunk-stage-writes
+ [commit opts chunk]
+ (->> chunk
+ (extract-chunk)
+ (mapcat #(stage-write commit opts %))
+ (doall)))
+
+(defn- stage-writes
+ "Stage WriteRows ({:model_type :model_id, optional :id/:file_path}) to `commit`, chunking internally per
+ model type so each chunk's entities load in one query with bounded memory. Callers pass a flat seq.
+
+ Returns:
+ - [{:id :file_path :content_hash}]"
+ [commit opts rows]
+ (->> rows
+ (->sized-chunks)
+ (mapcat #(chunk-stage-writes commit opts %))
+ (doall)))
+
+(defn- stage-deletes [commit delete-paths]
+ (doseq [delete-path delete-paths]
+ (source.p/stage-delete! commit delete-path)))
+
+(defn- exportable-write-rows
+ "WriteRows for a full export — every exportable id tagged with its RemoteSyncObject id (untracked deps get :id nil)."
+ []
+ (let [rso-id (u/index-by (juxt :model_type :model_id) :id (t2/select [:model/RemoteSyncObject :id :model_type :model_id]))]
+ (for [[model ids] (spec/exportable-entities)
+ id ids]
+ {:model_type model :model_id id :id (rso-id [model id])})))
+
+(defn- find-departed-entities
+ "Find RSO rows for entity-id entities that left the synced set (removed/delete status and not in
+ `targets`), matching the incremental path; path/hybrid and still-exported rows are kept."
+ [exported-rows]
+ (let [exported (into #{} (map #(select-keys % [:model_type :model_id])) exported-rows)]
+ (->> (t2/select [:model/RemoteSyncObject :id :model_type :model_id]
+ :status [:in ["removed" "delete"]])
+ (filter #(= :entity-id (:identity (spec/spec-for-model-type (:model_type %)))))
+ (remove #(exported (select-keys % [:model_type :model_id])))
+ (map :id))))
+
+(defn- mark-rows-synced!
+ "Mark `ids` synced via chunked CASE updates. Rows that appear in `synced` ([{:id :file_path :content_hash}])
+ get their file_path/content_hash written; any other id in `ids` keeps its existing path/hash. Callers pick the
+ scope: the incremental export passes only its write set, a full export passes every RemoteSyncObject id."
+ [ids synced sync-timestamp]
+ (let [by-id (u/index-by :id synced)]
+ (doseq [id-chunk (partition-all app-db-batch-size ids)
+ :let [hits (filter by-id id-chunk)]]
+ (t2/update! :model/RemoteSyncObject
+ {:id [:in (vec id-chunk)]}
+ (cond-> {:status "synced" :status_changed_at sync-timestamp}
+ (seq hits)
+ (assoc :file_path (into [:case]
+ (concat
+ (mapcat (fn [id]
+ [[:= :id id] (:file_path (by-id id))])
+ hits)
+ [:else :file_path]))
+ :content_hash (into [:case]
+ (concat
+ (mapcat (fn [id]
+ [[:= :id id] (:content_hash (by-id id))])
+ hits)
+ [:else :content_hash]))))))))
+
(defn- full-export!
- "Re-serializes the entire remote-synced set into `snapshot` and commits it, then marks every
- RemoteSyncObject row synced (as of `sync-timestamp`) and records each entity's file_path. Used when
- the pending changes can't be applied incrementally (name collisions, disabled content needing a
- reconcile, etc.). Throws if there is no remote-syncable content. Returns {:status :success}."
+ "Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.
+
+ Returns:
+ - {:status :success}
+ or throws"
[snapshot task-id message sync-timestamp]
- (let [models (spec/extract-entities-for-export)]
- (when (not models)
+ (let [export-rows (exportable-write-rows)]
+ (when (empty? export-rows)
(throw (ex-info "No remote-syncable content available." {})))
(remote-sync.task/update-progress! task-id 0.3)
- (let [{:keys [version entries]} (source/store! models snapshot task-id message)]
- (remote-sync.task/set-version! task-id version)
- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})
- (record-exported-paths! entries)
+ (let [opts (serdes/storage-base-context)
+ commit (source.p/open-commit snapshot)
+ [synced version] (try
+ (source.p/replace-all! commit) ; replace the managed dirs wholesale
+ [(stage-writes commit opts export-rows)
+ (source.p/finish-commit! commit message)]
+ (catch Throwable e
+ (source.p/abort-commit! commit)
+ (throw e)))]
+ (t2/with-transaction [_]
+ (remote-sync.task/set-version! task-id version)
+ (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]
+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
+ (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp))
{:status :success
- :outcome {:kind "pushed"
- :count (count entries)
- :branch (settings/remote-sync-branch)}})))
+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))
+
+(defn- incremental-export!
+ [plan disabled-files task-id snapshot message sync-timestamp]
+ (let [{:keys [writes delete-paths removed-ids]} plan
+ delete-paths (into (vec delete-paths) disabled-files)]
+ (remote-sync.task/update-progress! task-id 0.3)
+ (let [opts (serdes/storage-base-context)
+ commit (source.p/open-commit snapshot)
+ [synced version] (try
+ (let [synced (stage-writes commit opts writes)]
+ (stage-deletes commit delete-paths)
+ [synced (source.p/finish-commit! commit message)])
+ (catch Throwable e
+ (source.p/abort-commit! commit)
+ (throw e)))]
+ (t2/with-transaction [_]
+ (remote-sync.task/set-version! task-id version)
+ ;; delete departed rows first, then update RSO metadata — same order as full-export!
+ (doseq [removed-ids (partition-all 500 removed-ids)]
+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
+ (mark-rows-synced! (map :id synced) synced sync-timestamp)))
+ (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))
+ {:status :success
+ :outcome {:kind "pushed"
+ :count (+ (count writes) (count delete-paths))
+ :branch (settings/remote-sync-branch)}}))
(defn export!
"Exports remote-synced collections to a remote source repository.
Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, a commit message string, and
optional keyword arguments:
- - :force? - when true, overwrite the remote branch wholesale even if it has advanced (no merge)
- - :merge? - when true and the remote has advanced, perform a 3-way merge (rather than refusing)
+ - :force? - when true, overwrite the remote branch wholesale unconditionally (destructive)
+ - :merge? - when true and the remote has advanced, perform a 3-way merge if no conflicts
- :source - the Source the snapshot came from, used to resolve the merge base and the merged result
- :base-snapshot - a snapshot of the last synced version (the merge base), supplied when the remote
branch has advanced and a 3-way merge is required
- - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs
- from the current setting at task start, the export aborts with `:error` (defense-in-depth
- against any path that mutates the setting between scheduling and the work running).
-
- Takes the incremental fast-path when every pending change can be applied incrementally (see
- `incremental-plan` — creates, in-place updates, renames, and deletes of entity-id models). Otherwise does a
- full export: re-serializes the entire synced set, writes it, marks all RemoteSyncObject rows synced,
- and records each entity's `file_path` so future renames/deletes can go incremental.
-
- Behavior when the remote branch has advanced beyond the last sync:
- - `force?` -> overwrite the remote wholesale (full re-serialize).
- - `merge?` -> 3-way merge; non-conflicting remote changes are merged in and reconciled into the
- local app DB; genuine same-entity conflicts return a `:conflict`.
- - neither (default) -> refuse with `:conflict` (the UI's export preflight decides force/branch/merge).
-
- When the remote has NOT advanced, takes the incremental fast-path (see `incremental-plan`) when
- possible, otherwise a full re-serialize.
Returns a map with :status (`:success`, `:conflict`, or `:error`), and optionally :message, :version,
and :merge-summary keys. Various exceptions are caught and converted to error status maps."
- [^SourceSnapshot snapshot task-id message & {:keys [force? merge? base-snapshot pre-task-branch] src :source}]
- (when (branch-changed-since-scheduling? pre-task-branch)
- (log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"
- pre-task-branch (settings/remote-sync-branch))
- (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
- {:pre-task-branch pre-task-branch
- :current-branch (settings/remote-sync-branch)})))
- (when (not snapshot)
- (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable." {})))
+ [^SourceSnapshot snapshot task-id message & {:keys [force? merge? base-snapshot] src :source}]
(let [sync-timestamp (t/instant)]
(try
(analytics/inc! :metabase-remote-sync/exports)
@@ -882,14 +1061,11 @@
remote-version (source.p/version snapshot)
diverged? (and (some? base-version)
(not= base-version remote-version))
- ;; Pending-change inputs for the undiverged path; only computed there (incremental-plan can
- ;; be expensive, and they are unused on the force/merge branches).
- normal? (and (not force?) (not diverged?))
- disabled-files (when normal?
- (filterv (comp (disabled-content-dirs) path-top-level-dir)
- (source.p/list-files snapshot)))
- dirty-rows (when normal? (seq (remote-sync.object/dirty-rows)))
- plan (when dirty-rows (incremental-plan snapshot dirty-rows))]
+ disabled-files (delay (filterv (comp (disabled-content-dirs) path-top-level-dir)
+ (source.p/list-files snapshot)))
+ dirty-rows (delay (remote-sync.object/dirty-rows))
+ plan (delay (when (seq @dirty-rows)
+ (incremental-export-plan snapshot @dirty-rows)))]
(cond
;; Forced overwrite — full re-serialize, replacing managed dirs (discards remote divergence).
force?
@@ -897,59 +1073,38 @@
(log/info "Remote sync full export: forced overwrite")
(full-export! snapshot task-id message sync-timestamp))
- ;; Remote hasn't advanced and there's nothing to export: no dirty rows and no stale files in
- ;; now-disabled content dirs.
- (and (not diverged?) (empty? dirty-rows) (empty? disabled-files))
- (do
- (log/info "Remote sync export: no changes to export")
- {:status :success
- :outcome {:kind "push-skipped"}})
-
- ;; Remote hasn't advanced but a dirty row can't be applied incrementally → full re-serialize.
- (and (not diverged?) (= plan :remote-sync/unsyncable-batch))
- (do
- (log/info "Remote sync full export: a pending change can't be applied incrementally")
- (full-export! snapshot task-id message sync-timestamp))
+ (and diverged? merge?)
+ (cond
+ (nil? base-snapshot)
+ {:status :conflict
+ :version remote-version
+ :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
+ :message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}
- ;; Remote hasn't advanced — incremental fast-path: write only the changed entities, deleting
- ;; their old paths plus files left behind in now-disabled content dirs, preserving every other.
- (not diverged?)
- (let [{:keys [upserts delete-paths synced removed-ids]} plan
- delete-paths (into (vec delete-paths) disabled-files)]
- (remote-sync.task/update-progress! task-id 0.3)
- (let [written-version (source.p/apply-changes! snapshot message upserts delete-paths)]
- (remote-sync.task/set-version! task-id written-version))
- (doseq [{:keys [id file_path]} synced]
- (t2/update! :model/RemoteSyncObject :id id
- {:status "synced" :file_path file_path :status_changed_at sync-timestamp}))
- (when (seq removed-ids)
- (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))
- (log/infof "Remote sync incremental export: wrote %d, deleted %d"
- (count upserts) (count delete-paths))
- {:status :success
- :outcome {:kind "pushed"
- :count (+ (count upserts) (count delete-paths))
- :branch (settings/remote-sync-branch)}})
+ :else
+ (export-merged! src snapshot base-snapshot task-id message sync-timestamp
+ (spec/extract-entities-for-export)))
- ;; Remote advanced and the caller did not ask to merge — refuse. The UI's export preflight
- ;; drives the choice between force, new branch, and merge.
- (not merge?)
+ diverged? ;; and not merge? option
{:status :conflict
:version remote-version
:conflicts []
:message "The remote branch has changed since your last sync. Choose how to proceed."}
- ;; Merge requested but the merge base is gone (force-push/rebase) — no safe 3-way merge.
- (nil? base-snapshot)
- {:status :conflict
- :version remote-version
- :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]
- :message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}
+ ;; There's nothing to export: no dirty rows and no stale files.
+ (and (empty? @dirty-rows) (empty? @disabled-files))
+ (do
+ (log/info "Remote sync export: no changes to export")
+ {:status :success
+ :outcome {:kind "push-skipped"}})
- ;; Merge: 3-way merge, push the result, and reconcile the local app DB.
- :else
- (export-merged! src snapshot base-snapshot task-id message sync-timestamp
- (spec/extract-entities-for-export)))))
+ (not= @plan :remote-sync/incremental-not-possible)
+ (incremental-export! @plan @disabled-files task-id snapshot message sync-timestamp)
+
+ :else ;; fall back to full
+ (do
+ (log/info "Remote sync full export: a pending change can't be applied incrementally")
+ (full-export! snapshot task-id message sync-timestamp)))))
(catch Exception e
;; handle-task-result! records the failure on this result, and skips entirely when the task
;; was already cancelled (ended_at set) — so cancellation needs no special case here.
@@ -1200,13 +1355,18 @@
:conflicts true})))
(run-async! "import" branch
(fn [task-id]
+ (when (branch-changed-since-scheduling? pre-task-branch)
+ (log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"
+ pre-task-branch (settings/remote-sync-branch))
+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
+ {:pre-task-branch pre-task-branch
+ :current-branch (settings/remote-sync-branch)})))
(import! snapshot task-id
(assoc import-args
:force? force?
:force-deletion? force-deletion?
:merge? merge?
- :base-snapshot base-snapshot
- :pre-task-branch pre-task-branch)))
+ :base-snapshot base-snapshot)))
:on-success on-success)))
(defn async-export!
@@ -1227,6 +1387,9 @@
Returns a RemoteSyncTask."
[branch force? message & {:keys [on-success merge?]}]
(guards/ensure-no-active-task!)
+ (when-not (settings/remote-sync-enabled)
+ (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."
+ {:status-code 400})))
(let [pre-task-branch (settings/remote-sync-branch)
source (source/source-from-settings branch)
last-task-version (remote-sync.task/last-version)
@@ -1239,12 +1402,17 @@
(source.p/snapshot-at source last-task-version))]
(run-async! "export" branch
(fn [task-id]
+ (when (branch-changed-since-scheduling? pre-task-branch)
+ (log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"
+ pre-task-branch (settings/remote-sync-branch))
+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"
+ {:pre-task-branch pre-task-branch
+ :current-branch (settings/remote-sync-branch)})))
(export! snapshot task-id message
:force? force?
:merge? merge?
:source source
- :base-snapshot base-snapshot
- :pre-task-branch pre-task-branch))
+ :base-snapshot base-snapshot))
:on-success on-success)))
(defn preview-export-merge
@@ -1270,7 +1438,7 @@
no-changes
(if-let [base-snapshot (source.p/snapshot-at source base-version)]
(serdes/with-cache
- (if-let [models (spec/extract-entities-for-export)]
+ (if-let [models (seq (spec/extract-entities-for-export))]
(assoc (source/preview-merge models snapshot base-snapshot nil) :diverged? true)
(assoc no-changes :diverged? true)))
;; No merge base — the remote history was rewritten. A merge is impossible, but a force push is
@@ -1278,7 +1446,7 @@
{:diverged? true :clean? false :reason :history-rewritten
:conflicts [] :summary {:added 0 :updated 0 :removed 0}
:force-push-casualties (serdes/with-cache
- (if-let [models (spec/extract-entities-for-export)]
+ (if-let [models (seq (spec/extract-entities-for-export))]
(source/force-push-casualties-no-base models snapshot)
{:deleted [] :overwritten []}))}))))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
index 2a03d2d3a9c..7e1be88f6b5 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj
@@ -1,11 +1,14 @@
(ns metabase-enterprise.remote-sync.source
(:require
+ [buddy.core.codecs :as codecs]
+ [buddy.core.hash :as buddy-hash]
[clojure.string :as str]
[metabase-enterprise.remote-sync.merge :as remote-sync.merge]
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.source.git :as git]
[metabase-enterprise.remote-sync.source.ingestable :as ingestable]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.serialization.core :as serialization]
[metabase.models.serialization :as serdes]
[metabase.settings.core :as setting]
@@ -31,10 +34,7 @@
(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)
(source.p/read-file original-snapshot path)))
- (write-files! [_ _message _files]
- (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))
-
- (apply-changes! [_ _message _upserts _delete-paths]
+ (open-commit [_]
(throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))
(version [_]
@@ -47,20 +47,38 @@
(atom nil) (atom []))
(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))
-(defn- remote-sync-path
+(defn entity->path
+ "The repo-relative path an extracted `entity` serializes to, using storage context `opts`."
[opts entity]
(let [resolved (serialization/resolve-storage-path opts entity)
dirnames (drop-last resolved)
basename (str (last resolved) ".yaml")]
(str/join File/separator (concat dirnames [basename]))))
+(defn entity->content
+ "The serialized YAML string for an extracted `entity`."
+ [entity]
+ (yaml/generate-string (serialization/serialization-deep-sort entity)
+ {:dumper-options {:flow-style :block :split-lines false}}))
+
(defn entity->file-spec
"Serializes a single extracted entity into a `{:path :content}` file spec, using storage context
`opts` (from `serdes/storage-base-context`)."
[opts entity]
- {:path (remote-sync-path opts entity)
- :content (yaml/generate-string (serialization/serialization-deep-sort entity)
- {:dumper-options {:flow-style :block :split-lines false}})})
+ {:path (entity->path opts entity)
+ :content (entity->content entity)})
+
+(defn content-hash
+ "SHA-256 (hex) of a serialized YAML `content` string."
+ [^String content]
+ (codecs/bytes->hex (buddy-hash/sha256 content)))
+
+(defn row->content-hash
+ "SHA-256 (hex) of the serialized YAML for the entity named by `row` ({:model_type :model_id}), or nil if it
+ can't be extracted. Hashes the live DB serialization (never on-disk bytes), so it's stable across sync points."
+ [row]
+ (when-let [entity (first (spec/extract-entities-for-rows [row]))]
+ (content-hash (:content (entity->file-spec (serdes/storage-base-context) entity)))))
(defn serialize-specs
"Serializes a stream of entities into an eager vector of `{:path :content}` file specs. Reports progress
@@ -82,31 +100,6 @@
spec)))
stream)))
-(defn store!
- "Stores serialized entities from a stream to a remote source and commits the changes.
-
- Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source
- implementing the SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask
- identifier used to track progress updates), and a message (the commit message to use when writing
- files to the source).
-
- Returns `{:version <written-version> :entries [{:model_type :entity_id :path} ...]}`."
- [stream snapshot task-id message]
- (let [opts (serdes/storage-base-context)
- stream-count (bounded-count 10000 stream)
- entries (volatile! [])
- version (->> stream
- (map-indexed (fn [idx entity]
- (let [spec (entity->file-spec opts entity)]
- (vswap! entries conj {:model_type (-> entity :serdes/meta last :model)
- :entity_id (:entity_id entity)
- :path (:path spec)})
- (remote-sync.task/update-progress!
- task-id (-> (min (inc idx) stream-count) (/ stream-count) (* 0.65) (+ 0.3)))
- spec)))
- (source.p/write-files! snapshot message))]
- {:version version :entries @entries}))
-
(defn- snapshot->specs
"Reads a snapshot's managed-directory files into a sequence of `{:path :content}` specs, matching the
shape produced by [[serialize-specs]]. Used to read the merge base and remote-tip trees for merging."
@@ -140,14 +133,13 @@
(defn specs->snapshot
"Builds an in-memory read-only SourceSnapshot backed by `specs` (a seq of `{:path :content}`), so merged
- content can be loaded into the app DB without writing it to git. `write-files!` is unsupported."
+ content can be loaded into the app DB without writing it to git. Writing is unsupported."
[specs]
(let [by-path (into {} (map (juxt :path :content)) specs)]
(reify source.p/SourceSnapshot
(list-files [_] (vec (keys by-path)))
(read-file [_ path] (get by-path path))
- (write-files! [_ _ _] (throw (ex-info "in-memory merge snapshot is read-only" {})))
- (apply-changes! [_ _ _ _] (throw (ex-info "in-memory merge snapshot is read-only" {})))
+ (open-commit [_] (throw (ex-info "in-memory merge snapshot is read-only" {})))
(version [_] nil))))
(defn preview-merge
@@ -174,6 +166,20 @@
[stream snapshot]
(remote-sync.merge/force-push-casualties [] (serialize-specs stream nil) (snapshot->specs snapshot)))
+(defn- replace-all-files!
+ "Wholesale-commit `file-specs` ({:path :content}) to `snapshot`: clear the managed dirs, stage every spec,
+ and push. Returns the new version; aborts the commit on any error."
+ [snapshot message file-specs]
+ (let [commit (source.p/open-commit snapshot)]
+ (try
+ (source.p/replace-all! commit)
+ (doseq [spec file-specs]
+ (source.p/stage-upsert! commit spec))
+ (source.p/finish-commit! commit message)
+ (catch Throwable e
+ (source.p/abort-commit! commit)
+ (throw e)))))
+
(defn merge-and-store!
"Like [[store!]], but reconciles the freshly serialized local state against a remote branch that has
advanced beyond the last sync. Performs an entity-identity 3-way merge of:
@@ -189,7 +195,7 @@
(if (seq conflicts)
{:status :conflict :conflicts conflicts :summary summary}
{:status :success
- :version (source.p/write-files! snapshot message merged)
+ :version (replace-all-files! snapshot message merged)
:summary summary})))
(defn source-from-settings
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
index e453ef46cba..01a8103e9fd 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj
@@ -14,9 +14,10 @@
(java.net URI)
(org.apache.commons.io FileUtils)
(org.eclipse.jgit.api Git GitCommand TransportCommand)
- (org.eclipse.jgit.dircache DirCache DirCacheEditor$DeletePath DirCacheEditor$DeleteTree
- DirCacheEditor$PathEdit DirCacheEntry)
+ (org.eclipse.jgit.dircache DirCache DirCacheBuilder DirCacheEditor DirCacheEditor$DeletePath
+ DirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)
(org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)
+ (org.eclipse.jgit.lib ObjectInserter ObjectReader)
(org.eclipse.jgit.revwalk RevCommit RevWalk)
(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdate
RemoteRefUpdate$Status UsernamePasswordCredentialsProvider)
@@ -289,68 +290,76 @@
(str/replace-first (.getName ^Ref target) "refs/heads/" ""))))
(throw (ex-info "Failed to get a default branch for git repository." {:head-ref head-ref})))))
-(defn- commit-edits!
- "Builds a commit by editing the branch tip's tree in place and pushes it. Seeds an in-core index from
- the parent commit's tree — so every unchanged entry and subtree is carried forward by object id — then
- applies only the edits: whole subtrees named in `delete-dirs` are removed (used to fully replace a
- managed directory), paths in `delete-paths` are removed, and `upserts` (file specs with :path/:content)
- are written as new blobs. `writeTree` only rewrites the subtrees on the path to an edit, so the work is
- proportional to the number of changes rather than the size of the repo.
-
- Returns the new commit sha. Throws ExceptionInfo if the write or push fails."
- [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message upserts delete-paths delete-dirs]
- (let [repo (.getRepository git)
- branch-ref (qualify-branch branch)
- parent-id (.resolve repo version)]
- (with-open [inserter (.newObjectInserter repo)
- reader (.newObjectReader repo)
- rev-walk (RevWalk. repo)]
- (let [index (DirCache/newInCore)]
- ;; Seed the index from the parent tree; unchanged entries and subtrees carry forward by id.
- (let [builder (.builder index)]
- (when parent-id
- (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader
- (.getTree (.parseCommit rev-walk parent-id))))
- (.finish builder))
- ;; Apply only the edits.
- (let [editor (.editor index)]
- (doseq [^String dir delete-dirs]
- (.add editor (DirCacheEditor$DeleteTree. dir)))
- (doseq [^String path delete-paths]
- (.add editor (DirCacheEditor$DeletePath. path)))
- (doseq [{:keys [^String path content]} upserts]
- (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]
- (.add editor (proxy [DirCacheEditor$PathEdit] [path]
- (apply [^DirCacheEntry entry]
- (.setFileMode entry FileMode/REGULAR_FILE)
- (.setObjectId entry blob-id))))))
- (.finish editor))
- (let [tree-id (.writeTree index inserter)
- commit-builder (doto (CommitBuilder.)
- (.setTreeId tree-id)
- (.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))
- (.setCommitter (PersonIdent. "Metabase Library" "[email protected]"))
- (.setMessage message))]
- (when parent-id
- (.setParentId commit-builder parent-id))
- (let [commit-id (.insert inserter commit-builder)]
- (.flush inserter)
- (doto (.updateRef repo branch-ref)
- (.setNewObjectId commit-id)
- (.update))
- (push-branch! snapshot)
- (.name commit-id)))))))
-
-(defn- write-files!
- "Full export: replace every managed dir wholesale with `files` (managed-dir files not in `files` are
- removed), preserving everything outside the managed dirs."
- [{:keys [managed-dirs] :as snapshot} ^String message files]
- (commit-edits! snapshot message files nil managed-dirs))
-
-(defn- apply-changes!
- "Incremental patch: write `upserts`, remove `delete-paths`, and preserve every other file."
- [snapshot ^String message upserts delete-paths]
- (commit-edits! snapshot message upserts delete-paths nil))
+(defn- close-commit-resources! [inserter reader rev-walk]
+ (.close ^ObjectInserter inserter)
+ (.close ^ObjectReader reader)
+ (.close ^RevWalk rev-walk))
+
+;; A commit being built incrementally against a GitSnapshot. Holds the open JGit resources (inserter, reader,
+;; rev-walk) and the in-core index/editor; blobs are inserted as files are staged and the tree is written and
+;; pushed at finish. Edits the branch tip's tree in place — unchanged entries/subtrees carry forward by object
+;; id — so writeTree's work is proportional to the number of changes, not the repo size.
+(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id]
+ source.p/CommitBuilder
+ (stage-upsert! [_ {:keys [^String path content]}]
+ (let [blob-id (.insert ^ObjectInserter inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]
+ (.add ^DirCacheEditor editor
+ (proxy [DirCacheEditor$PathEdit] [path]
+ (apply [^DirCacheEntry entry]
+ (.setFileMode entry FileMode/REGULAR_FILE)
+ (.setObjectId entry blob-id)))))
+ nil)
+
+ (stage-delete! [_ path]
+ (.add ^DirCacheEditor editor (DirCacheEditor$DeletePath. ^String path))
+ nil)
+
+ (replace-all! [_]
+ (doseq [^String dir (:managed-dirs snapshot)]
+ (.add ^DirCacheEditor editor (DirCacheEditor$DeleteTree. dir)))
+ nil)
+
+ (finish-commit! [_ message]
+ (.finish ^DirCacheEditor editor)
+ (let [^Git git (:git snapshot)
+ repo (.getRepository git)
+ branch-ref (qualify-branch (:branch snapshot))
+ tree-id (.writeTree ^DirCache index ^ObjectInserter inserter)
+ commit-builder (doto (CommitBuilder.)
+ (.setTreeId tree-id)
+ (.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))
+ (.setCommitter (PersonIdent. "Metabase Library" "[email protected]"))
+ (.setMessage ^String message))]
+ (when parent-id
+ (.setParentId commit-builder parent-id))
+ (let [commit-id (.insert ^ObjectInserter inserter commit-builder)]
+ (.flush ^ObjectInserter inserter)
+ (doto (.updateRef repo branch-ref)
+ (.setNewObjectId commit-id)
+ (.update))
+ (push-branch! snapshot)
+ (close-commit-resources! inserter reader rev-walk) ; close only after a successful push
+ (.name commit-id))))
+
+ (abort-commit! [_]
+ (close-commit-resources! inserter reader rev-walk)
+ nil))
+
+(defn- open-commit*
+ "Begin a GitCommit against `snapshot`, seeding the in-core index from the parent tree."
+ [{:keys [^Git git ^String version] :as snapshot}]
+ (let [repo (.getRepository git)
+ parent-id (.resolve repo version)
+ inserter (.newObjectInserter repo)
+ reader (.newObjectReader repo)
+ rev-walk (RevWalk. repo)
+ index (DirCache/newInCore)]
+ (let [^DirCacheBuilder builder (.builder index)]
+ (when parent-id
+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader
+ (.getTree (.parseCommit rev-walk parent-id))))
+ (.finish builder))
+ (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id)))
(defn branches
"Retrieves all branch names from the remote repository.
@@ -429,11 +438,8 @@
(read-file [this path]
(read-file this path))
- (write-files! [this message files]
- (write-files! this message files))
-
- (apply-changes! [this message upserts delete-paths]
- (apply-changes! this message upserts delete-paths))
+ (open-commit [this]
+ (open-commit* this))
(version [this]
(:version this))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
index f035a58d8a5..dac3be98fb1 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj
@@ -56,26 +56,8 @@
Returns the file contents as a string, or nil if the file doesn't exist.")
- (write-files! [snapshot message files]
- "Writes multiple files to the source with a commit message.
-
- Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),
- and files (a sequence of file specs). Each file spec is a map with :path and :content keys.
-
- All existing files within managed directories that are not in the write set are removed.
- Files outside managed directories are always preserved.
-
- Returns the version of the written files.")
-
- (apply-changes! [snapshot message upserts delete-paths]
- "Incrementally updates the source: writes/overwrites `upserts`, removes `delete-paths`,
- and PRESERVES every other existing file (unlike `write-files!`, which deletes managed-dir
- files absent from the write set).
-
- Takes a SourceSnapshot instance, a commit message, `upserts` (a sequence of file specs, each
- a map with :path and :content), and `delete-paths` (a sequence of path strings to remove).
-
- Returns the version of the written files.")
+ (open-commit [snapshot]
+ "Begin building one commit incrementally; returns a CommitBuilder.")
(version [snapshot]
"Gets a version identifier for the current state of the snapshot.
@@ -84,6 +66,26 @@
Returns a version identifier string (e.g., a git SHA)."))
+(defprotocol CommitBuilder
+ "A single commit being built incrementally — stage files one at a time, then finish (or abort). Lets the
+ caller stream files into a commit without holding them all at once.
+
+ Lifecycle / resources: a builder holds native resources open (for git, a JGit `ObjectInserter`,
+ `ObjectReader`, and `RevWalk`) from `open-commit` until they're released. The caller MUST eventually call
+ `finish-commit!` (on success) or `abort-commit!` (on failure) to release them — including when staging
+ throws, so callers should wrap staging in `try` and `abort-commit!` in the `catch`/`finally`. A builder
+ left open leaks those resources; the lifecycle is the caller's responsibility, not the builder's."
+ (stage-upsert! [commit file-spec]
+ "Stage one file (a `{:path :content}` map) into the commit. Returns nil.")
+ (stage-delete! [commit path]
+ "Stage removal of `path` from the commit. Returns nil.")
+ (replace-all! [commit]
+ "Clear the snapshot's managed dirs, so the staged files replace them wholesale. Returns nil.")
+ (finish-commit! [commit message]
+ "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")
+ (abort-commit! [commit]
+ "Release the commit's resources without committing or pushing. Returns nil."))
+
(defprotocol Diffable
"Optional capability for snapshots that can cheaply report which files changed since a prior version.
Implemented by sources backed by real history (git, in-memory versioned test sources); snapshots that
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 744db12c134..9f196d55965 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -1148,58 +1148,44 @@
(defmethod query-export-roots :default [_] nil)
-(defn- resolve-targets
- "Expands collection IDs to include all descendant collections.
- Takes a set of collection IDs and returns a set including the original IDs
- plus all IDs of nested collections."
- [targets opts]
- (when (seq targets)
- (merge-with into
- (u/traverse targets #(serdes/descendants (first %) (second %) opts))
- (u/traverse targets #(serdes/required (first %) (second %))))))
+(defn exportable-entities
+ "What a full export would serialize: a map of {model-name [id ...]} — the export roots plus their transitive
+ `serdes/descendants`/`required` closure — or `{}` when there is no remote-syncable content."
+ []
+ (let [opts {:include-field-values false
+ :include-database-secrets false
+ :continue-on-error false
+ :skip-archived true}
+ root-targets (into []
+ (mapcat query-export-roots)
+ (vals (enabled-specs)))
+ targets (-> #{}
+ (into (keys (u/traverse root-targets #(serdes/descendants (first %) (second %) opts))))
+ (into (keys (u/traverse root-targets #(serdes/required (first %) (second %))))))]
+ (u/group-by first second targets)))
(defn extract-entities-for-export
"Extracts all entities for remote-sync export based on enabled specs.
Returns a lazy sequence of serialized entities ready for storage.
- Iterates over enabled specs and uses `query-export-roots` to find root targets
- for each model type. Models with `:export-scope :derived` or no export-scope
- are expanded from other targets via serdes/descendants.
-
Only extracts models that:
1. Have a spec in remote-sync-specs
2. Are currently enabled (based on :enabled? field)
3. Are in one of the provided collections (or descendants)"
[]
- (let [specs (enabled-specs)
- ;; Collect all root targets by iterating over enabled specs
- root-targets (into []
- (comp (map val)
- (mapcat query-export-roots)
- (filter identity))
- specs)]
- (when-let [targets (resolve-targets
- root-targets
- {:include-field-values false
- :include-database-secrets false
- :continue-on-error false
- :skip-archived true})]
- (eduction (map (fn [[model ids]]
- (serdes/extract-all model {:where [:in :id ids]
- :skip-archived true})))
- cat
- (u/group-by first second (keys targets))))))
+ (eduction (map (fn [[model ids]]
+ (serdes/extract-all model {:where [:in :id ids]
+ :skip-archived true})))
+ cat
+ (exportable-entities)))
(defn extract-entities-for-rows
"Serializes the entities named by `rows`, grouped by model type. Each row is a map with a
`:model_type` (the serdes model name) and a `:model_id` (the local DB primary key). Returns a lazy
sequence of serialized entities."
[rows]
- (let [by-model (reduce (fn [m {:keys [model_type model_id]}]
- (update m model_type (fnil conj #{}) model_id))
- {}
- rows)]
+ (let [by-model (u/group-by :model_type :model_id conj #{} rows)]
(eduction (map (fn [[model ids]]
(serdes/extract-all model {:where [:in :id ids]
:skip-archived true})))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj
new file mode 100644
index 00000000000..09c23cea9ed
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj
@@ -0,0 +1,449 @@
+(ns metabase-enterprise.remote-sync.content-hash-test
+ "Tests for content-hash-based dirty detection in remote sync (GHY-3933).
+
+ The yellow dot / dirty status must track whether an entity's *serialized form* changed since the last
+ sync — not merely whether an update event fired. After a sync, each RemoteSyncObject row records a
+ `content_hash` of the entity's serialized YAML. When an update event arrives, the consumer re-serializes
+ the entity and compares: an unchanged serialization (a no-op save, or an edit to a non-serialized field
+ like a transform's schedule) must leave the row `synced`, so a subsequent push produces no empty commit.
+
+ These tests cover every tracked model type (entity-id, hybrid, and path identities) and the
+ null-baseline / real-change / revert edge cases."
+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.remote-sync.content-hash-test]}}}}}}
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.models.remote-sync-object :as sync-object]
+ [metabase-enterprise.remote-sync.source :as source]
+ [metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers]
+ [metabase.collections.models.collection :as collection]
+ [metabase.collections.test-utils :refer [with-library-synced]]
+ [metabase.events.core :as events]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(defn- noop-update-status!
+ "Seeds a synced RemoteSyncObject for [model-type model-id] whose `content_hash` matches the entity's
+ current serialization, publishes a no-op `topic` carrying the unchanged `object`, and returns the row's
+ status afterward. Asserts a baseline hash is computable for the model type (i.e. serialization works).
+ `extra-payload` is merged into the event (e.g. {:previous-object …} for events whose schema requires it)."
+ ([model-type model-id topic object]
+ (noop-update-status! model-type model-id topic object {}))
+ ([model-type model-id topic object extra-payload]
+ (t2/delete! :model/RemoteSyncObject)
+ (let [row {:model_type model-type :model_id model-id}
+ hash (source/row->content-hash row)]
+ (is (some? hash) (str "row->content-hash should be computable for " model-type))
+ (t2/insert! :model/RemoteSyncObject
+ (merge row {:model_name "Baseline"
+ :status "synced"
+ :content_hash hash
+ :status_changed_at (t/offset-date-time)}))
+ (events/publish-event! topic (merge {:object object :user-id (mt/user->id :rasta)} extra-payload))
+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))))
+
+;;; ------------------------------------------- entity-id models -------------------------------------------
+
+(deftest card-noop-update-stays-synced-test
+ (testing "A no-op Card update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (is (= "synced" (noop-update-status! "Card" (:id card) :event/card-update card {:previous-object card}))))))
+
+(deftest dashboard-noop-update-stays-synced-test
+ (testing "A no-op Dashboard update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Dashboard dash {:name "Dash" :collection_id (:id coll)}]
+ (is (= "synced" (noop-update-status! "Dashboard" (:id dash) :event/dashboard-update dash))))))
+
+(deftest document-noop-update-stays-synced-test
+ (testing "A no-op Document update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Document doc {:collection_id (u/the-id coll)}]
+ (is (= "synced" (noop-update-status! "Document" (:id doc) :event/document-update doc))))))
+
+(deftest timeline-noop-update-stays-synced-test
+ (testing "A no-op Timeline update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Timeline tl {:name "TL" :collection_id (:id coll)}]
+ (is (= "synced" (noop-update-status! "Timeline" (:id tl) :event/timeline-update tl))))))
+
+(deftest snippet-noop-update-stays-synced-test
+ (testing "A no-op NativeQuerySnippet update keeps it synced when the Library is synced (GHY-3933)"
+ (with-library-synced
+ (mt/with-temp [:model/Collection sc {:name "Snippets" :namespace "snippets"}
+ :model/NativeQuerySnippet snip {:name "Snip" :content "SELECT 1" :collection_id (:id sc)}]
+ (is (= "synced" (noop-update-status! "NativeQuerySnippet" (:id snip) :event/snippet-update snip)))))))
+
+(deftest collection-noop-update-stays-synced-test
+ (testing "A no-op Collection update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}]
+ (is (= "synced" (noop-update-status! "Collection" (:id coll) :event/collection-update coll))))))
+
+;;; ------------------------------------------- path & hybrid models -------------------------------------------
+
+(deftest table-noop-update-stays-synced-test
+ (testing "A no-op Table update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Database db {:name "DB"}
+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}
+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)
+ :is_published true :collection_id (:id coll)}]
+ (is (= "synced" (noop-update-status! "Table" (:id table) :event/table-update table))))))
+
+(deftest field-noop-update-stays-synced-test
+ (testing "A no-op Field update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Database db {:name "DB"}
+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}
+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)
+ :is_published true :collection_id (:id coll)}
+ :model/Field field {:name "F" :table_id (:id table) :base_type :type/Integer}]
+ (is (= "synced" (noop-update-status! "Field" (:id field) :event/field-update field))))))
+
+(deftest segment-noop-update-stays-synced-test
+ (testing "A no-op Segment update keeps it synced (GHY-3933)"
+ (mt/with-temp [:model/Database db {:name "DB"}
+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}
+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)
+ :is_published true :collection_id (:id coll)}
+ :model/Segment seg {:name "S" :table_id (:id table)
+ :definition {:source-table (:id table) :filter [:> [:field 1 nil] 0]}}]
+ (is (= "synced" (noop-update-status! "Segment" (:id seg) :event/segment-update seg))))))
+
+(deftest measure-noop-update-stays-synced-test
+ (testing "A no-op Measure update keeps it synced (GHY-3933)"
+ (mt/with-model-cleanup [:model/Measure]
+ (mt/with-temp [:model/Database db {:name "DB"}
+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}
+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)
+ :is_published true :collection_id (:id coll)}
+ :model/Field _ {:name "F" :table_id (:id table) :base_type :type/Integer}
+ :model/Measure measure {:name "M" :table_id (:id table)}]
+ (is (= "synced" (noop-update-status! "Measure" (:id measure) :event/measure-update measure)))))))
+
+;;; ------------------------------------------- transforms models (setting-gated) -------------------------------------------
+
+(deftest transform-noop-update-stays-synced-test
+ (testing "A no-op Transform update keeps it synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:namespace collection/transforms-ns :location "/"}
+ :model/Transform tr {:name "T" :collection_id (:id coll)}]
+ (is (= "synced" (noop-update-status! "Transform" (:id tr) :event/transform-update tr))))))))
+
+(deftest transform-tag-noop-update-stays-synced-test
+ (testing "A no-op TransformTag update keeps it synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]
+ (mt/with-temp [:model/TransformTag tag {:name "Tag"}]
+ (is (= "synced" (noop-update-status! "TransformTag" (:id tag) :event/transform-tag-update tag))))))))
+
+(deftest python-library-noop-update-stays-synced-test
+ (testing "A no-op PythonLibrary update keeps it synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/PythonLibrary]
+ (let [lib (t2/insert-returning-instance! :model/PythonLibrary {:path "lib.py" :source "# x"})]
+ (is (= "synced" (noop-update-status! "PythonLibrary" (:id lib) :event/python-library-update lib)))))))))
+
+;;; ------------------------------------------- edge cases -------------------------------------------
+
+(deftest null-baseline-update-marks-dirty-test
+ (testing "An update with no recorded baseline hash still marks dirty — exports the first time (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card card {:name "C" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "C"
+ :status "synced" :status_changed_at (t/offset-date-time)})
+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})
+ (is (= "update" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))
+ "A null content_hash baseline must be treated as dirty"))))
+
+(deftest changed-content-marks-dirty-test
+ (testing "A real change to serialized content marks dirty (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card card {:name "Original" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "Original"
+ :status "synced"
+ :content_hash (source/row->content-hash {:model_type "Card" :model_id (:id card)})
+ :status_changed_at (t/offset-date-time)})
+ (t2/update! :model/Card (:id card) {:name "Renamed"})
+ (events/publish-event! :event/card-update {:object (t2/select-one :model/Card :id (:id card))
+ :previous-object card
+ :user-id (mt/user->id :rasta)})
+ (is (= "update" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))
+ "A changed serialization must mark the row dirty"))))
+
+(deftest revert-to-baseline-resyncs-test
+ (testing "An update whose content matches the synced baseline clears a stale dirty flag (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card card {:name "C" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "C"
+ :status "update"
+ :content_hash (source/row->content-hash {:model_type "Card" :model_id (:id card)})
+ :status_changed_at (t/offset-date-time)})
+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))
+ "Content matching the baseline must clear a stale dirty flag")
+ (is (not (sync-object/dirty?))))))
+
+;;; ------------------------------------------- mark-rows-synced! -------------------------------------------
+
+(deftest content-metadata-matches-row-hash-test
+ (testing "a full export writes a content_hash matching per-row row->content-hash for every identity flavor —
+ including the extract-query overrides (Collection, NativeQuerySnippet), hybrids (Measure), and path
+ models (Field) that have no entity_id (GHY-3933)"
+ (with-library-synced
+ (mt/with-temporary-setting-values [remote-sync-type :read-write]
+ (mt/with-temp [:model/Database db {:name "DB"}
+ :model/Collection rs {:is_remote_synced true :name "RS" :location "/"}
+ :model/Collection data {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}
+ :model/Collection snips {:name "Snippets" :namespace "snippets"}
+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id rs)}
+ :model/NativeQuerySnippet snip {:name "Snip" :content "SELECT 1" :collection_id (:id snips)}
+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db) :is_published true :collection_id (:id data)}
+ :model/Field field {:name "F" :table_id (:id table) :base_type :type/Integer}
+ :model/Measure measure {:name "M" :table_id (:id table)}]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})
+ rows (mapv (fn [[mt id]] {:model_type mt :model_id id})
+ [["Card" (:id card)] ["Collection" (:id rs)] ["NativeQuerySnippet" (:id snip)]
+ ["Field" (:id field)] ["Measure" (:id measure)]])]
+ (t2/delete! :model/RemoteSyncObject)
+ (doseq [row rows]
+ (t2/insert! :model/RemoteSyncObject (merge row {:model_name "x" :status "synced"
+ :status_changed_at (t/offset-date-time)})))
+ ;; force? routes through full-export!, which computes + records the content_hash for every flavor
+ (impl/export! (source.p/snapshot (test-helpers/create-mock-source)) task-id "Full export" :force? true)
+ (doseq [{:keys [model_type model_id] :as row} rows]
+ (is (= (source/row->content-hash row)
+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type model_type :model_id model_id))
+ (str "written hash should match per-row hash for " model_type))))))))))
+
+(deftest mark-rows-synced!-chunks-rows-test
+ (testing "mark-rows-synced! writes every synced row even when they span multiple batches (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card c1 {:name "C1" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}
+ :model/Card c2 {:name "C2" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}
+ :model/Card c3 {:name "C3" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (doseq [card [c1 c2 c3]]
+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name (:name card)
+ :status "create" :status_changed_at (t/offset-date-time)}))
+ ;; synced entries keyed by RSO id, carrying each card's real content hash
+ (let [synced (mapv (fn [r] {:id (:id r) :file_path "p"
+ :content_hash (source/row->content-hash (select-keys r [:model_type :model_id]))})
+ (t2/select [:model/RemoteSyncObject :id :model_type :model_id]))]
+ ;; force more than one chunk for the 3 rows
+ (with-redefs [impl/app-db-batch-size 2]
+ (#'impl/mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced (t/offset-date-time))))
+ (doseq [card [c1 c2 c3]]
+ (is (= (source/row->content-hash {:model_type "Card" :model_id (:id card)})
+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type "Card" :model_id (:id card)))
+ (str "every chunked row should get its hash: " (:name card)))))))
+
+(deftest insert-with-metadata!-chunks-rows-test
+ (testing "insert-with-metadata! inserts every row with its content_hash even across multiple chunks (GHY-3933)"
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card c1 {:name "C1" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}
+ :model/Card c2 {:name "C2" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}
+ :model/Card c3 {:name "C3" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (t2/delete! :model/RemoteSyncObject)
+ (let [rows (mapv (fn [c] {:model_type "Card" :model_id (:id c) :model_name (:name c)
+ :status "synced" :status_changed_at (t/offset-date-time)})
+ [c1 c2 c3])]
+ ;; force more than one chunk for the 3 cards
+ (with-redefs [impl/app-db-batch-size 2]
+ (#'impl/insert-with-metadata! rows [])))
+ (doseq [card [c1 c2 c3]]
+ (is (= (source/row->content-hash {:model_type "Card" :model_id (:id card)})
+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type "Card" :model_id (:id card)))
+ (str "every chunked row should be inserted with its hash: " (:name card)))))))
+
+;;; ------------------------------------------- export integration -------------------------------------------
+;;; These run a real export! to a mock source — which records the content_hash baseline through the actual
+;;; export path (full/incremental) — then fire a no-op update and assert it stays synced. Unlike the
+;;; consumer tests above (which seed the hash directly), these exercise the export-side baseline recording.
+
+(defn- export-baseline-then-noop-status!
+ "Seeds `create`-status RemoteSyncObject `seeds` (so export has something to write), runs a real export! to
+ a mock source (recording content_hash baselines), then publishes a no-op event and returns the resulting
+ status of [model-type model-id]. `target` is {:model-type :model-id :topic :object :payload}."
+ [seeds {:keys [model-type model-id topic object payload]}]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (t2/delete! :model/RemoteSyncObject)
+ (doseq [seed seeds]
+ (t2/insert! :model/RemoteSyncObject (merge {:status "create" :status_changed_at (t/offset-date-time)} seed)))
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Initial export")]
+ (is (= :success (:status result)) (str "export should succeed: " result))
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))
+ (str model-type " should be synced after export")))
+ (events/publish-event! topic (merge {:object object :user-id (mt/user->id :rasta)} payload))
+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))))
+
+(deftest card-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op Card update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}
+ {:model_type "Card" :model_id (:id card) :model_name "Card" :model_collection_id (:id coll)}]
+ {:model-type "Card" :model-id (:id card) :topic :event/card-update :object card
+ :payload {:previous-object card}})))))))
+
+(deftest dashboard-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op Dashboard update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Dashboard dash {:name "Dash" :collection_id (:id coll)}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}
+ {:model_type "Dashboard" :model_id (:id dash) :model_name "Dash" :model_collection_id (:id coll)}]
+ {:model-type "Dashboard" :model-id (:id dash) :topic :event/dashboard-update :object dash})))))))
+
+(deftest document-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op Document update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Document doc {:collection_id (u/the-id coll)}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}
+ {:model_type "Document" :model_id (:id doc) :model_name "Doc" :model_collection_id (:id coll)}]
+ {:model-type "Document" :model-id (:id doc) :topic :event/document-update :object doc})))))))
+
+(deftest timeline-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op Timeline update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}
+ :model/Timeline tl {:name "TL" :collection_id (:id coll)}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}
+ {:model_type "Timeline" :model_id (:id tl) :model_name "TL" :model_collection_id (:id coll)}]
+ {:model-type "Timeline" :model-id (:id tl) :topic :event/timeline-update :object tl})))))))
+
+(deftest collection-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op Collection update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS" :location "/"}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}]
+ {:model-type "Collection" :model-id (:id coll) :topic :event/collection-update :object coll})))))))
+
+(deftest transform-tag-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op TransformTag update stays synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true remote-sync-enabled true]
+ (mt/with-temp [:model/TransformTag tag {:name "Tag"}]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "TransformTag" :model_id (:id tag) :model_name "Tag"}]
+ {:model-type "TransformTag" :model-id (:id tag) :topic :event/transform-tag-update :object tag}))))))))
+
+(deftest python-library-export-then-noop-stays-synced-test
+ (testing "After a real export, a no-op PythonLibrary update stays synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/PythonLibrary]
+ (let [lib (t2/insert-returning-instance! :model/PythonLibrary {:path "lib.py" :source "# x"})]
+ (is (= "synced"
+ (export-baseline-then-noop-status!
+ [{:model_type "PythonLibrary" :model_id (:id lib) :model_name "lib.py"}]
+ {:model-type "PythonLibrary" :model-id (:id lib) :topic :event/python-library-update :object lib})))))))))
+
+(deftest full-export-records-metadata-and-stays-synced-test
+ (testing "A forced full export (real store-and-record! + mark-rows-synced!) writes file_path and
+ content_hash, and a subsequent no-op update stays synced (GHY-3933)"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS" :location "/"}
+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}
+ :model/RemoteSyncObject _ {:model_type "Collection" :model_id (:id coll) :model_name "RS" :status "create" :status_changed_at (t/offset-date-time)}
+ :model/RemoteSyncObject _ {:model_type "Card" :model_id (:id card) :model_name "Card" :model_collection_id (:id coll) :status "create" :status_changed_at (t/offset-date-time)}
+ ;; a removed/delete row whose entity left the synced set — full export must drop it
+ :model/RemoteSyncObject leftover {:model_type "Card" :model_id 999999 :model_name "Gone" :status "delete" :status_changed_at (t/offset-date-time)}]
+ ;; force? routes through full-export! -> store-and-record! (no incremental fast-path)
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Full export" :force? true)]
+ (is (= :success (:status result)) (str "forced full export should succeed: " result))
+ (let [row (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))]
+ (is (= "synced" (:status row)))
+ (is (some? (:content_hash row)) "content_hash recorded from store-and-record!'s serialization")
+ (is (some? (:file_path row)) "file_path recorded from store-and-record!'s serialization"))
+ (is (nil? (t2/select-one :model/RemoteSyncObject :id (:id leftover)))
+ "removed/delete row dropped by full export (as the incremental path does)"))
+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))
+ "a no-op update after a full export stays synced")))))))
+
+;;; ------------------------------------------- import integration -------------------------------------------
+;;; These import a snapshot from a mock source — which records the content_hash baseline through the actual
+;;; import path (`insert-with-metadata!`) — then fire a no-op update and assert it stays synced. This is the
+;;; "right after a pull" scenario from GHY-3933.
+
+(defn- import-then-noop-status!
+ "Imports `files` from a mock source (recording content_hash baselines via the import path), looks up the
+ imported entity with `lookup` (a 0-arg thunk returning the instance), publishes a no-op `topic`, and
+ returns the resulting RemoteSyncObject status. `payload-fn` (optional) returns extra event payload."
+ [files model-type lookup topic & {:keys [payload-fn]}]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ ms (test-helpers/create-mock-source :initial-files files)
+ result (impl/import! (source.p/snapshot ms) task-id :force? true :force-deletion? true)]
+ (is (= :success (:status result)) (str "import should succeed: " result))
+ (let [entity (lookup)
+ model-id (:id entity)]
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))
+ (str model-type " should be synced after import"))
+ (events/publish-event! topic (merge {:object entity :user-id (mt/user->id :rasta)}
+ (when payload-fn (payload-fn entity))))
+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id))))))
+
+(deftest transform-import-then-noop-stays-synced-test
+ (testing "After a real import, a no-op Transform update stays synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/Transform :model/Collection]
+ ;; the transform YAML references the "test-data (h2)" database by name; force it to load so the
+ ;; import resolves the reference regardless of test order.
+ (mt/db)
+ (let [coll-eid "transforms-coll-xxxxx"
+ tr-eid "test-transform-xxxxxx"
+ files {"main" {"collections/transforms/transforms/transforms.yaml"
+ (test-helpers/generate-collection-yaml coll-eid "Transforms" :namespace "transforms")
+ "collections/transforms/transforms/test_transform.yaml"
+ (test-helpers/generate-transform-yaml tr-eid "Test Transform" :collection-id coll-eid)}}]
+ (is (= "synced"
+ (import-then-noop-status!
+ files "Transform"
+ #(t2/select-one :model/Transform :entity_id tr-eid)
+ :event/transform-update)))))))))
+
+(deftest transform-tag-import-then-noop-stays-synced-test
+ (testing "After a real import, a no-op TransformTag update stays synced (GHY-3933)"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/TransformTag]
+ (let [tag-eid "test-transform-tag-xx"
+ files {"main" {"transforms/transform_tags/test_tag.yaml"
+ (test-helpers/generate-transform-tag-yaml tag-eid "Test Tag")}}]
+ (is (= "synced"
+ (import-then-noop-status!
+ files "TransformTag"
+ #(t2/select-one :model/TransformTag :entity_id tag-eid)
+ :event/transform-tag-update)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
index 608404e29c9..8b3fb93d230 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj
@@ -7,7 +7,7 @@
[clojure.test :refer :all]
[java-time.api :as t]
[metabase-enterprise.remote-sync.events :as remote-sync.events]
- [metabase-enterprise.remote-sync.impl :as impl]
+ [metabase-enterprise.remote-sync.spec :as spec]
[metabase.collections.models.collection :as collection]
[metabase.collections.test-utils :refer [with-library-synced]]
[metabase.events.core :as events]
@@ -18,6 +18,14 @@
(use-fixtures :once (fixtures/initialize :db))
+(defn- mark-synced!
+ "Populate the RemoteSyncObject table with synced rows for `imported-data` (a `:by-entity-id` map), as a
+ successful import does — used to set up a synced baseline before firing an event."
+ [imported-data]
+ (t2/delete! :model/RemoteSyncObject)
+ (when-let [rows (seq (spec/sync-all-entities! (t/instant) imported-data))]
+ (t2/insert! :model/RemoteSyncObject rows)))
+
;;; Model Change Event Tests
(deftest card-create-event-creates-entry-test
@@ -514,7 +522,7 @@
:model/Card card {:name "Test Card"
:dataset_query (mt/mbql-query venues)
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Card" #{(:entity_id card)}}})
+ (mark-synced! {:by-entity-id {"Card" #{(:entity_id card)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/card-update
@@ -550,7 +558,7 @@
:model/Collection normal-collection {:name "Normal"}
:model/Dashboard dashboard {:name "Test Dashboard"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})
+ (mark-synced! {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id dashboard))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/dashboard-update
@@ -582,7 +590,7 @@
(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}
:model/Collection normal-collection {:name "Normal"}
:model/Document document {:collection_id (u/the-id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Document" #{(:entity_id document)}}})
+ (mark-synced! {:by-entity-id {"Document" #{(:entity_id document)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id document))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/document-update
@@ -611,7 +619,7 @@
(deftest existing-collection-type-changed-from-remote-synced-test
(testing "existing collection type changed from remote-synced is marked as removed"
(mt/with-temp [:model/Collection collection {:is_remote_synced true :name "Remote-Sync"}]
- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Collection" #{(:entity_id collection)}}})
+ (mark-synced! {:by-entity-id {"Collection" #{(:entity_id collection)}}})
(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id collection))]
(is (= "synced" (:status initial-entry)))
(events/publish-event! :event/collection-update
@@ -788,7 +796,7 @@
:model/Collection normal-collection {:name "Normal Collection"}
:model/Timeline timeline {:name "Test Timeline"
:collection_id (:id remote-sync-collection)}]
- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})
+ (mark-synced! {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})
(is (= 1 (count (t2/select :model/RemoteSyncObject))))
(events/publish-event! :event/timeline-update
{:object (assoc timeline :collection_id (:id normal-collection))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index af379af4586..330fa079b52 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -189,13 +189,13 @@
;; export! tests
-(deftest export!-with-no-source-configured-test
- (testing "export! with no source configured throws"
- (mt/with-temporary-setting-values [remote-sync-type :read-write]
- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
- (is (thrown-with-msg? clojure.lang.ExceptionInfo
- #"Remote sync source is not enabled"
- (impl/export! nil task-id "Test commit")))))))
+(deftest async-export!-with-no-source-configured-test
+ (testing "async-export! throws at the API call (not inside the async task) when remote sync isn't configured"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-url nil]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Remote sync source is not enabled"
+ (impl/async-export! "main" false "Test commit"))))))
(deftest export!-with-no-remote-synced-collections-test
(testing "export! is a no-op success when nothing is dirty (no remote-synced content to export)"
@@ -1672,8 +1672,13 @@ serdes/meta:
(version [_] version)
(list-files [_] [])
(read-file [_ _] nil)
- (write-files! [_ _ _] version)
- (apply-changes! [_ _ _ _] version)))
+ (open-commit [_]
+ (reify source.p/CommitBuilder
+ (stage-upsert! [_ _] nil)
+ (stage-delete! [_ _] nil)
+ (replace-all! [_] nil)
+ (finish-commit! [_ _] "written-version")
+ (abort-commit! [_] nil)))))
(defn- export-test-source
"A minimal Source whose snapshot-at returns a snapshot at the requested version."
@@ -1770,18 +1775,16 @@ serdes/meta:
(deftest export!-refuses-when-diverged-without-merge-flag-test
(testing "when the remote advanced and neither force? nor merge? is set, export! refuses without writing or merging"
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
- (let [merged? (atom false)
- stored? (atom false)]
+ (let [merged? (atom false)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})
- source/store! (fn [_ _ _ _] (reset! stored? true) "v")]
+ source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:source (export-test-source)
:base-snapshot (export-test-snapshot "base-B"))]
(is (= :conflict (:status result)))
(is (false? @merged?) "no merge without the merge flag")
- (is (false? @stored?) "nothing written")
+ ;; :conflict short-circuits before any write — the version is never advanced
(is (nil? (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))
(deftest export!-force-overwrites-without-merging-test
@@ -1789,17 +1792,16 @@ serdes/meta:
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
(let [merged? (atom false)]
(with-redefs [remote-sync.task/last-version (constantly "base-B")
- spec/extract-entities-for-export (constantly [{:dummy true}])
- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})
- ;; force? routes through full-export!, which uses store!'s {:version :entries} result.
- source/store! (fn [& _] {:version "forced-version" :entries []})]
+ spec/exportable-entities (constantly {"Card" [999999999]}) ; absent id -> empty extraction; routing is what's under test
+ source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})]
(let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg"
:force? true
:source (export-test-source)
:base-snapshot (export-test-snapshot "base-B"))]
(is (= :success (:status result)))
(is (false? @merged?) "force? skips the merge path")
- (is (= "forced-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))
+ ;; force? routes through full-export!, which commits via the snapshot and advances the version
+ (is (= "written-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))
(deftest export!-no-merge-when-not-diverged-test
(testing "when the remote has not advanced, export! takes the normal (non-merge) export path"
@@ -1885,7 +1887,7 @@ serdes/meta:
{:merged [{:path "collections/x.yaml" :content "y"}]
:conflicts []
:summary {:added 1 :updated 0 :removed 0}})
- ;; simulate the load marking everything synced (what sync-objects! does), then running
+ ;; simulate the load marking everything synced (what the import insert does), then running
;; the in-transaction finalize (restore-dirty + set-version)
impl/load-snapshot! (fn [_ _ _ & {:keys [finalize!]}]
(t2/update! :model/RemoteSyncObject
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
index 8ade5adf547..edab7ea9fcb 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj
@@ -96,7 +96,7 @@
path (atom :fallback)]
(with-redefs [impl/incremental-load-snapshot!
(fn [& args] (let [r (apply real args)]
- (reset! path (if (= r @#'impl/incremental-not-possible) :fallback :incremental))
+ (reset! path (if (= r :remote-sync/incremental-not-possible) :fallback :incremental))
r))]
[(import-at! src "v1" :force? false) @path])))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
index 8c7c0c09b8d..c523d51b4d8 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj
@@ -134,8 +134,7 @@
:model/NativeQuerySnippet snippet {:name "Existing Snippet" :content "SELECT 1" :collection_id coll-id}]
(is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))
"Should have no snippet tracking entries initially")
- ;; Enable snippet sync
- (rs-events/sync-snippet-tracking! true)
+ (rs-events/enable-snippet-tracking!)
;; Verify tracking entries created
(is (t2/exists? :model/RemoteSyncObject
:model_type "Collection"
@@ -168,8 +167,7 @@
(is (= 2 (t2/count :model/RemoteSyncObject :model_type [:in ["Collection" "NativeQuerySnippet"]]
:model_id [:in [coll-id (:id snippet)]]))
"Should have 2 tracking entries")
- ;; Disable snippet sync
- (rs-events/sync-snippet-tracking! false)
+ (rs-events/disable-snippet-tracking!)
;; Verify tracking entries removed
(is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))
"Snippet tracking entries should be removed")
@@ -417,6 +415,6 @@ is_sample: false
(let [files-after-export (get @(:files-atom mock-source) "main")]
(is (not (some #(str/includes? % "archived_snippet") (keys files-after-export)))
"Archived snippet file should be deleted after export")))
- (testing "RemoteSyncObject entry is updated to synced after export"
- (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id)))
- "RemoteSyncObject entry for archived snippet should have synced status")))))))))
+ (testing "the archived snippet's tracking row is dropped after export (it left the synced set)"
+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id))
+ "RemoteSyncObject entry for the archived snippet should be deleted")))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 69cc8774109..156c801ee34 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -16,6 +16,24 @@
(set! *warn-on-reflection* true)
+(defn- write-files!
+ "Test seeding helper: wholesale-write `files` ({:path :content}) to `snapshot` via the commit builder
+ (clear the managed dirs, stage every file, push). Returns the new version."
+ [snapshot message files]
+ (let [c (source.p/open-commit snapshot)]
+ (source.p/replace-all! c)
+ (doseq [f files] (source.p/stage-upsert! c f))
+ (source.p/finish-commit! c message)))
+
+(defn- apply-changes!
+ "Test helper: incremental patch via the commit builder — stage `upserts`, remove `delete-paths`, push.
+ Returns the new version."
+ [snapshot message upserts delete-paths]
+ (let [c (source.p/open-commit snapshot)]
+ (doseq [u upserts] (source.p/stage-upsert! c u))
+ (doseq [p delete-paths] (source.p/stage-delete! c p))
+ (source.p/finish-commit! c message)))
+
(defn- git-working-branch
"The working branch in the given repo"
[{:keys [^Git git]}]
@@ -210,11 +228,11 @@
(str thirddir-path "path2.txt") "File 2 in third dir"}
:branches ["branch-1" "branch-2"])]
(testing "All files in managed dirs not in write set are removed; root files outside managed dirs are preserved"
- (source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}
- {:path (str subdir-path "path.txt") :content "Updated subdir content"}
- {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
- {:path (str thirddir-path "path.txt") :content "Updated third dir content"}
- {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
+ (write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}
+ {:path (str subdir-path "path.txt") :content "Updated subdir content"}
+ {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}
+ {:path (str thirddir-path "path.txt") :content "Updated third dir content"}
+ {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])
(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))
(let [master-snap (source.p/snapshot master)]
;; otherdir files are removed because collections/ is a managed dir and those files weren't in the write set
@@ -240,7 +258,7 @@
(git/list-files (assoc remote :version "master"))))
(is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))
(testing "Writing only to collections/ removes all other collection files"
- (source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
+ (write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])
(is (= [(str thirddir-path "path.txt")
"master.txt"
"master2.txt"]
@@ -255,10 +273,10 @@
(str subdir "edit.yaml") "old content"
(str subdir "remove.yaml") "delete me"})]
(testing "apply-changes! overwrites/adds upserts, removes delete-paths, and PRESERVES every other file"
- (source.p/apply-changes! (source.p/snapshot master) "Incremental"
- [{:path (str subdir "edit.yaml") :content "new content"}
- {:path (str subdir "new.yaml") :content "brand new"}]
- [(str subdir "remove.yaml")])
+ (apply-changes! (source.p/snapshot master) "Incremental"
+ [{:path (str subdir "edit.yaml") :content "new content"}
+ {:path (str subdir "new.yaml") :content "brand new"}]
+ [(str subdir "remove.yaml")])
(is (= ["Incremental" "Initial commit"] (map :message (git/log master))))
(let [snap (source.p/snapshot master)]
(is (= [(str subdir "edit.yaml")
@@ -285,9 +303,9 @@
"collections/a/deep/sibling.yaml" "deep sibling"
"collections/b/edit.yaml" "old"
"notes.txt" "root note"})]
- (source.p/apply-changes! (source.p/snapshot master) "Incremental deep"
- [{:path "collections/b/edit.yaml" :content "new"}]
- [])
+ (apply-changes! (source.p/snapshot master) "Incremental deep"
+ [{:path "collections/b/edit.yaml" :content "new"}]
+ [])
(let [snap (source.p/snapshot master)]
(is (= "new" (source.p/read-file snap "collections/b/edit.yaml")))
(is (= "deep keep" (source.p/read-file snap "collections/a/deep/nested/keep.yaml"))
@@ -303,8 +321,8 @@
"transforms/t1/t.yaml" "a transform"
"transforms/t2/t.yaml" "another transform"
"notes.txt" "root note"})]
- (source.p/write-files! (source.p/snapshot master) "Full"
- [{:path "collections/new/new.yaml" :content "new collection"}])
+ (write-files! (source.p/snapshot master) "Full"
+ [{:path "collections/new/new.yaml" :content "new collection"}])
(let [snap (source.p/snapshot master)]
(is (= ["collections/new/new.yaml" "notes.txt"] (source.p/list-files snap))
"transforms/ (managed, no upserts) fully removed; collections/ reconciled to the write set; non-managed notes.txt preserved")
@@ -318,9 +336,9 @@
(mt/with-temp-dir [remote-dir nil]
(let [[master _remote] (init-source! "master" remote-dir
:files {"collections/a/keep.yaml" "keep"})]
- (source.p/apply-changes! (source.p/snapshot master) "Delete missing + add"
- [{:path "collections/a/new.yaml" :content "new"}]
- ["collections/a/gone.yaml"])
+ (apply-changes! (source.p/snapshot master) "Delete missing + add"
+ [{:path "collections/a/new.yaml" :content "new"}]
+ ["collections/a/gone.yaml"])
(let [snap (source.p/snapshot master)]
(is (= ["collections/a/keep.yaml" "collections/a/new.yaml"] (source.p/list-files snap)))
(is (= "new" (source.p/read-file snap "collections/a/new.yaml")))
@@ -335,10 +353,10 @@
"remove.txt" "delete me"
"deep/nested/keep.txt" "deep unchanged"})
from-version (:version (source.p/snapshot master))]
- (source.p/apply-changes! (source.p/snapshot master) "Change set"
- [{:path "edit.txt" :content "new content"}
- {:path "add.txt" :content "brand new"}]
- ["remove.txt"])
+ (apply-changes! (source.p/snapshot master) "Change set"
+ [{:path "edit.txt" :content "new content"}
+ {:path "add.txt" :content "brand new"}]
+ ["remove.txt"])
(let [snap (source.p/snapshot master)]
(testing "added / modified / deleted are reported in their own buckets"
(is (= {:added #{"add.txt"}
@@ -386,7 +404,7 @@
;; Make source be behind again
(git-working-add! remote "only-on-remote.txt" "Initially on remote")
(git-working-commit! remote "Only on remote")
- (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
+ (write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])
(testing "Remote has the new commit with just the files committed, but only version is in history"
(is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))
(is (= ["additional-file.txt" "initially-source.txt" "master.txt" "only-on-remote.txt" "subdir/path.txt"] (git/list-files (assoc remote :version "master"))))
@@ -400,8 +418,8 @@
(git-working-commit! remote "New-branch on remote")
(is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))
(is (nil? (git/log new-branch)))
- (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
- {:path "new-branch-file.txt" :content "Updated on source"}])
+ (write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}
+ {:path "new-branch-file.txt" :content "Updated on source"}])
(is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch"))))))))))
(deftest git-source-using-commit-ref
@@ -410,8 +428,8 @@
:files {"master.txt" "File in master"
"subdir/path.txt" "File in subdir"})
old-master (source.p/snapshot master)]
- (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}
- {:path "new-file.txt" :content "New file in master"}])
+ (write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}
+ {:path "new-file.txt" :content "New file in master"}])
(is (= "File in master" (source.p/read-file old-master "master.txt")))
(is (= "Updated file in master" (source.p/read-file (source.p/snapshot master) "master.txt")))
(is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master))))))
@@ -427,7 +445,7 @@
(is (= (git/commit-sha master "master") initial-version)
"version should match the commit id for the branch")
(testing "version changes after writing files"
- (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])
+ (write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])
(let [new-version (source.p/version (source.p/snapshot master))]
(is (not= initial-version new-version) "version should change after commit")
(is (= 40 (count new-version)) "new version should also be a full SHA-1 hash")
@@ -469,13 +487,13 @@
(str old-col-path "cards/card2.yaml") "Card 2 in old col"
(str kept-col-path "dashboards/dash1.yaml") "Dashboard in kept col"})]
(testing "Writing to a managed dir removes all stale files in ALL managed dirs"
- (source.p/write-files! (source.p/snapshot master) "Rename database"
- [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}
- {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}
- {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}
- {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}
- {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}
- {:path "snippets/old_snippet.yaml" :content "Old snippet"}])
+ (write-files! (source.p/snapshot master) "Rename database"
+ [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}
+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}
+ {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}
+ {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}
+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}
+ {:path "snippets/old_snippet.yaml" :content "Old snippet"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files "databases/new_db/new_db.yaml") "New database file should exist")
(is (contains? files "databases/new_db/schemas/public.yaml") "New schema file should exist")
@@ -485,11 +503,11 @@
(is (contains? files "snippets/old_snippet.yaml") "Written snippet file should remain")
(is (contains? files "unmanaged/keep_me.txt") "Unmanaged files should be untouched")))
(testing "Entity moved between collections removes files from old collection"
- (source.p/write-files! (source.p/snapshot master) "Move card to new collection"
- [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}
- {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}
- {:path "databases/new_db/new_db.yaml" :content "Renamed database"}
- {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])
+ (write-files! (source.p/snapshot master) "Move card to new collection"
+ [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}
+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}
+ {:path "databases/new_db/new_db.yaml" :content "Renamed database"}
+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files (str new-col-path "cards/card1.yaml")) "Moved card should exist in new collection")
(is (contains? files (str kept-col-path "dashboards/dash1.yaml")) "Kept collection files should remain")
@@ -505,9 +523,9 @@
:files {(str col-path "cards/eid123_OldCardName.yaml") "Card with old name"
(str col-path "cards/eid456_OtherCard.yaml") "Other card"})]
(testing "Entity renamed within a collection removes the old-named file"
- (source.p/write-files! (source.p/snapshot master) "Rename card"
- [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}
- {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])
+ (write-files! (source.p/snapshot master) "Rename card"
+ [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}
+ {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])
(let [files (set (source.p/list-files (source.p/snapshot master)))]
(is (contains? files (str col-path "cards/eid123_NewCardName.yaml")) "Renamed card should exist")
(is (contains? files (str col-path "cards/eid456_OtherCard.yaml")) "Other card should still exist")
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
index 281b6fb3a6a..31895f28fe7 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj
@@ -12,12 +12,8 @@
(read-file [_ path]
(get files path))
- (write-files! [_ _message new-files]
- (into {} (map (juxt :path :content) new-files))
- "written-files-version")
-
- (apply-changes! [_ _message _upserts _delete-paths]
- "written-files-version")
+ (open-commit [_]
+ (throw (UnsupportedOperationException. "open-commit not implemented in this test mock")))
(version [_]
"mock-version"))
@@ -102,9 +98,6 @@
(deftest wrapping-source-is-read-only-test
(testing "WrappingSnapshot is a read-only ingestion view: write operations throw rather than write"
(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"})
- wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])
- file-spec {:path "collections/foo.yaml" :content "foo-content"}]
- (is (thrown? UnsupportedOperationException
- (source.p/write-files! wrapped-snap "test commit" [file-spec])))
+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])]
(is (thrown? UnsupportedOperationException
- (source.p/apply-changes! wrapped-snap "test commit" [file-spec] []))))))
+ (source.p/open-commit wrapped-snap))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
index 20a0eb4955c..e3d19543dcf 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj
@@ -1,55 +1,16 @@
(ns metabase-enterprise.remote-sync.source-test
(:require
- [clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as th]
[metabase.test :as mt]
- [metabase.test.fixtures :as fixtures]
- [toucan2.core :as t2]))
+ [metabase.test.fixtures :as fixtures]))
(use-fixtures :once (fixtures/initialize :db))
(use-fixtures :each th/clean-remote-sync-state)
-(defrecord MockSourceSnapshot [written-files]
- source.p/SourceSnapshot
- (list-files [_]
- [])
-
- (read-file [_ _path]
- nil)
-
- (write-files! [_ message files]
- (let [realized-files (vec files)]
- (reset! written-files {:message message :files realized-files}))
- "mock-written-version")
-
- (apply-changes! [_ message upserts _delete-paths]
- (reset! written-files {:message message :files (vec upserts)})
- "mock-written-version")
-
- (version [_]
- "mock-version"))
-
-(defrecord MockSource [written-files]
- source.p/Source
- (create-branch [_ _branch _base]
- nil)
-
- (branches [_]
- ["main"])
-
- (default-branch [_]
- "main")
-
- (snapshot [_]
- (->MockSourceSnapshot written-files))
-
- (snapshot-at [_ _version]
- (->MockSourceSnapshot written-files)))
-
(defn- create-test-entity
"Creates a test entity with serdes/meta for testing"
[id label model-name]
@@ -71,135 +32,18 @@
(reify source.p/SourceSnapshot
(list-files [_] (vec (keys by-path)))
(read-file [_ p] (get by-path p))
- (write-files! [_ message files]
- (reset! written {:message message :files (vec files)})
- "merged-version")
- (apply-changes! [_ message upserts _delete-paths]
- (reset! written {:message message :files (vec upserts)})
- "merged-version")
+ (open-commit [_]
+ (let [staged (atom [])]
+ (reify source.p/CommitBuilder
+ (stage-upsert! [_ spec] (swap! staged conj spec) nil)
+ (stage-delete! [_ _path] nil)
+ (replace-all! [_] nil)
+ (finish-commit! [_ message]
+ (reset! written {:message message :files @staged})
+ "merged-version")
+ (abort-commit! [_] nil))))
(version [_] "remote-tip"))))
-(deftest store!-basic-test
- (testing "store! writes files to source with correct structure"
- (mt/with-temp [:model/User user {:first_name "Test"
- :last_name "User"
- :password "password123"}
- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
- :initiated_by (:id user)}]
- (let [written-files (atom nil)
- mock-source (->MockSource written-files)
- test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")
- (create-test-entity "test-id-2" "entity-two" "Card")]]
- (is (= "mock-written-version" (:version (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message"))))
- (testing "write-files! was called with correct message"
- (is (= "Test commit message" (:message @written-files))))
- (testing "write-files! was called with correct number of files"
- (is (= 2 (count (:files @written-files)))))
- (testing "each file has path and content"
- (doseq [file (:files @written-files)]
- (is (contains? file :path) "File should have :path")
- (is (contains? file :content) "File should have :content")
- (is (string? (:path file)) "Path should be a string")
- (is (string? (:content file)) "Content should be a string")))
- (testing "file paths end with .yaml"
- (doseq [file (:files @written-files)]
- (is (str/ends-with? (:path file) ".yaml")
- "File paths should end with .yaml")))
- (testing "file content is valid YAML containing entity data"
- (doseq [file (:files @written-files)]
- (is (str/includes? (:content file) "serdes/meta")
- "Content should include serdes/meta")
- (is (or (str/includes? (:content file) "test-id-1")
- (str/includes? (:content file) "test-id-2"))
- "Content should include entity ID")))))))
-
-(deftest store!-progress-tracking-test
- (testing "store! updates task progress as files are written"
- (mt/with-temp [:model/User user {:first_name "Test"
- :last_name "User"
- :password "password123"}
- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
- :initiated_by (:id user)}]
- (let [written-files (atom nil)
- mock-source (->MockSource written-files)
- test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")
- (create-test-entity "test-id-2" "entity-two" "Card")
- (create-test-entity "test-id-3" "entity-three" "Dashboard")]]
- (let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (nil? (:progress initial-task)) "Progress should be nil initially"))
- (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")
- (let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]
- (is (some? (:progress final-task)) "Progress should be updated after store!")
- (is (> (:progress final-task) 0.3) "Progress should be greater than 0.3")
- (is (<= (:progress final-task) 0.95) "Progress should be at most 0.95")
- (is (some? (:last_progress_report_at final-task))
- "last_progress_report_at should be set"))))))
-
-(deftest store!-empty-stream-test
- (testing "store! handles empty stream correctly"
- (mt/with-temp [:model/User user {:first_name "Test"
- :last_name "User"
- :password "password123"}
- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
- :initiated_by (:id user)}]
- (let [written-files (atom nil)
- mock-source (->MockSource written-files)]
- (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")
- (testing "write-files! was called even with empty stream"
- (is (some? @written-files)))
- (testing "files list is empty"
- (is (empty? (:files @written-files))))))))
-
-(deftest store!-deterministic-yaml-test
- (testing "store! produces deterministic YAML output with consistent key ordering"
- (mt/with-temp [:model/User user {:first_name "Test"
- :last_name "User"
- :password "password123"}
- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"
- :initiated_by (:id user)}]
- (let [written-files-1 (atom nil)
- written-files-2 (atom nil)
- mock-source-1 (->MockSource written-files-1)
- mock-source-2 (->MockSource written-files-2)
- ;; Create entity with keys in non-alphabetical order to test sorting
- test-entity {:serdes/meta [{:id "test-id" :label "test-entity" :model "Card"}]
- :z_last_field "should be last (unknown)"
- :a_first_field "should be after known fields (unknown)"
- :name "Test Card"
- :description "A test card"
- :entity_id "test-id"
- :display "table"
- :dataset_query {:z_field 1 :a_field 2 :database 3}
- :visualization_settings {:zebra true :apple false}}]
- ;; Store the same entity twice
- (source/store! [test-entity] (source.p/snapshot mock-source-1) task-id "First commit")
- (source/store! [test-entity] (source.p/snapshot mock-source-2) task-id "Second commit")
- (testing "YAML content is identical between runs"
- (let [content-1 (-> @written-files-1 :files first :content)
- content-2 (-> @written-files-2 :files first :content)]
- (is (= content-1 content-2)
- "YAML content should be identical for the same entity")))
- (testing "known keys appear in serialization-order.edn order"
- (let [content (-> @written-files-1 :files first :content)
- name-pos (str/index-of content "name:")
- description-pos (str/index-of content "description:")
- display-pos (str/index-of content "display:")]
- ;; Per serialization-order.edn, Card order is: name, description, entity_id, ... display
- (is (< name-pos description-pos)
- "name should appear before description")
- (is (< description-pos display-pos)
- "description should appear before display")))
- (testing "unknown keys are sorted alphabetically after known keys"
- (let [content (-> @written-files-1 :files first :content)
- a-first-pos (str/index-of content "a_first_field:")
- z-last-pos (str/index-of content "z_last_field:")]
- (is (< a-first-pos z-last-pos)
- "a_first_field should appear before z_last_field (alphabetical)")))))))
-
(deftest merge-and-store!-clean-merge-test
(testing "when local and remote changed different entities, merge-and-store! writes the union with no conflict"
(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 3afbf5b947e..57358755ae0 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -194,39 +194,32 @@ width: fixed
;; git read-file contract).
(get-in @files-atom [branch path])))
- (write-files! [_this _message files]
- (case fail-mode
- :write-files-error (throw (Exception. "Failed to write files"))
- :store-error (throw (Exception. "Store failed"))
- :network-error (throw (java.net.UnknownHostException. "Remote host not found"))
- ;; Default success case - remove all files in managed dirs, then add new files
- (let [write-entries (remove #(str/blank? (:path %)) files)
- write-paths (into #{} (map :path) write-entries)
- current-files (get @files-atom branch {})
- ;; Keep files outside managed dirs or in the write set
- kept-files (into {}
- (filter (fn [[path _]]
- (or (not (contains? managed-dirs (top-level-dir path)))
- (contains? write-paths path))))
- current-files)
- ;; Add new files
- final-files (into kept-files (map (juxt :path :content) write-entries))]
- (swap! files-atom assoc branch final-files)))
- "write-files-version")
-
- (apply-changes! [_this _message upserts delete-paths]
- (case fail-mode
- :apply-changes-error (throw (Exception. "Failed to apply changes"))
- :network-error (throw (java.net.UnknownHostException. "Remote host not found"))
- ;; Default: overwrite/add upserts, remove delete-paths, preserve every other file.
- (let [write-entries (remove #(str/blank? (:path %)) upserts)
- delete-set (into #{} (remove str/blank?) delete-paths)]
- (swap! files-atom update branch
- (fn [current]
- (as-> (or current {}) files
- (apply dissoc files delete-set)
- (into files (map (juxt :path :content)) write-entries))))))
- "apply-changes-version")
+ (open-commit [_this]
+ (let [upserts (atom [])
+ deletes (atom #{})
+ replace-all? (atom false)]
+ (reify source.p/CommitBuilder
+ (stage-upsert! [_ file-spec] (swap! upserts conj file-spec) nil)
+ (stage-delete! [_ path] (swap! deletes conj path) nil)
+ (replace-all! [_] (reset! replace-all? true) nil)
+ (finish-commit! [_ _message]
+ (case fail-mode
+ :write-files-error (throw (Exception. "Failed to write files"))
+ :store-error (throw (Exception. "Store failed"))
+ :apply-changes-error (throw (Exception. "Failed to apply changes"))
+ :network-error (throw (java.net.UnknownHostException. "Remote host not found"))
+ (let [managed (if @replace-all? (set managed-dirs) #{})]
+ (swap! files-atom update branch
+ (fn [current]
+ (as-> (or current {}) files
+ (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))
+ (contains? @deletes p))))
+ files)
+ (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))
+ @upserts))))))
+ ;; version string discriminates a wholesale replace (replace-all!) from an incremental patch
+ (if @replace-all? "write-files-version" "apply-changes-version"))
+ (abort-commit! [_] nil))))
(version [_this]
"mock-version"))
@@ -288,8 +281,8 @@ width: fixed
Unlike [[create-mock-source]] (which always reports \"mock-version\"), this lets a test control
versions: `:current` is the version `snapshot` reports, and `:trees` maps version -> {path content}.
`snapshot-at` returns a snapshot for a version present in `trees`, or nil for an unknown version (to
- model a base orphaned by a force-push/rebase). `write-files!` records the written set under a fresh
- version, advances `:current`, and returns the new version so an export can fast-forward onto it."
+ model a base orphaned by a force-push/rebase). Committing (via `open-commit`) records the written set
+ under a fresh version, advances `:current`, and returns the new version so an export can fast-forward onto it."
[& {:keys [current trees branch managed-dirs]
:or {current "v-remote" branch "main" managed-dirs ingest/legal-top-level-paths}}]
(let [managed (set managed-dirs)
@@ -312,30 +305,27 @@ width: fixed
source.p/SourceSnapshot
(list-files [_] (vec (keys (get-in @state [:trees version] {}))))
(read-file [_ path] (get-in @state [:trees version path]))
- (write-files! [_ _message files]
- (let [n (:counter (swap! state update :counter inc))
- new-version (str "written-" n)
- kept (into {}
- (remove (fn [[p _]] (contains? managed (top-level-dir p))))
- (get-in @state [:trees version] {}))
- tree (into kept
- (comp (remove #(str/blank? (:path %)))
- (map (juxt :path :content)))
- files)]
- (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))
- new-version))
- (apply-changes! [_ _message upserts delete-paths]
- (let [n (:counter (swap! state update :counter inc))
- new-version (str "written-" n)
- delete-set (into #{} (remove str/blank?) delete-paths)
- tree (as-> (get-in @state [:trees version] {}) t
- (apply dissoc t delete-set)
- (into t
- (comp (remove #(str/blank? (:path %)))
- (map (juxt :path :content)))
- upserts))]
- (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))
- new-version))
+ (open-commit [_]
+ (let [staged-upserts (atom [])
+ staged-deletes (atom #{})
+ replace-all? (atom false)]
+ (reify source.p/CommitBuilder
+ (stage-upsert! [_ file-spec] (swap! staged-upserts conj file-spec) nil)
+ (stage-delete! [_ path] (swap! staged-deletes conj path) nil)
+ (replace-all! [_] (reset! replace-all? true) nil)
+ (finish-commit! [_ _message]
+ (let [replace-dirs (if @replace-all? managed #{}) ; `managed` = (set managed-dirs)
+ n (:counter (swap! state update :counter inc))
+ new-version (str "written-" n)
+ tree (as-> (get-in @state [:trees version] {}) t
+ (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))
+ (contains? @staged-deletes p)))) t)
+ (into t (comp (remove #(str/blank? (:path %)))
+ (map (juxt :path :content)))
+ @staged-upserts))]
+ (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))
+ new-version))
+ (abort-commit! [_] nil))))
(version [_] version)))]
(reify source.p/Source
(branches [_] [branch])
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
index 6a279468330..08f4be77ec2 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj
@@ -5,6 +5,15 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.test-helpers :as th]))
+(defn- write-files!
+ "Wholesale-write `files` ({:path :content}) to `snapshot` via the commit builder (clear managed dirs,
+ stage every file, push)."
+ [snapshot message files]
+ (let [c (source.p/open-commit snapshot)]
+ (source.p/replace-all! c)
+ (doseq [f files] (source.p/stage-upsert! c f))
+ (source.p/finish-commit! c message)))
+
(deftest mock-source-write-files-managed-dir-cleanup-test
(testing "MockSource removes files in managed dirs not in write set"
(let [source (th/create-mock-source
@@ -14,8 +23,8 @@
"other/file4.yaml" "content4"}}
:managed-dirs #{"collections"})
snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Write only abc"
- [{:path "collections/abc/file1.yaml" :content "new-content1"}])
+ (write-files! snapshot "Write only abc"
+ [{:path "collections/abc/file1.yaml" :content "new-content1"}])
(is (= #{"collections/abc/file1.yaml" "other/file4.yaml"}
(set (source.p/list-files snapshot)))
"Only written files in managed dirs should remain; unmanaged dirs untouched"))))
@@ -27,8 +36,8 @@
"unmanaged/file2.yaml" "content2"}}
:managed-dirs #{"collections"})
snapshot (source.p/snapshot source)]
- (source.p/write-files! snapshot "Write collections"
- [{:path "collections/abc/file1.yaml" :content "new-content"}])
+ (write-files! snapshot "Write collections"
+ [{:path "collections/abc/file1.yaml" :content "new-content"}])
(is (= #{"collections/abc/file1.yaml" "unmanaged/file2.yaml"}
(set (source.p/list-files snapshot)))
"Unmanaged directory files should be preserved"))))
@@ -41,8 +50,8 @@
:managed-dirs #{"collections" "snippets"})
snapshot (source.p/snapshot source)]
;; Write only to collections, nothing to snippets
- (source.p/write-files! snapshot "Write only collections"
- [{:path "collections/abc/file1.yaml" :content "new-content"}])
+ (write-files! snapshot "Write only collections"
+ [{:path "collections/abc/file1.yaml" :content "new-content"}])
(is (= #{"collections/abc/file1.yaml"}
(set (source.p/list-files snapshot)))
"Snippets dir should be cleaned even though no snippet files were written"))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index 09ac4c2ef28..422c3393696 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -73,6 +73,38 @@
(is (= "update" (:status entry))
"Transform should have 'update' status after modification")))))))
+(deftest transform-noop-update-does-not-redirty-test
+ (testing "A transform-update event whose serialized form is unchanged must not re-dirty a synced transform (GHY-3933)"
+ ;; Repro for the \"yellow dot with no changes\" report: editing a non-serialized aspect of a transform
+ ;; (e.g. its schedule/run_trigger) — or any no-op re-save — fires :event/transform-update and flips the
+ ;; RemoteSyncObject to \"update\", lighting the dirty indicator. But the serialized YAML is identical, so a
+ ;; subsequent push produces a commit with no files. The dirty state should track the serialized form, not
+ ;; the mere occurrence of an update event.
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms true
+ remote-sync-enabled true]
+ (mt/with-model-cleanup [:model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]
+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxxxx" :location "/"}
+ :model/Transform transform {:name "Synced Transform" :collection_id coll-id}
+ :model/RemoteSyncObject _rso1 {:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}
+ :model/RemoteSyncObject _rso2 {:model_type "Transform" :model_id (:id transform) :model_name "Synced Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}]
+ ;; Establish a synced baseline by exporting to the mock remote.
+ (let [mock-source (test-helpers/create-mock-source)
+ result (impl/export! (source.p/snapshot mock-source) task-id "Initial export")]
+ (is (= :success (:status result))
+ (str "Export should succeed. Result: " result))
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Transform" :model_id (:id transform))))
+ "Transform should be synced after export"))
+ ;; Re-publish an update event for the unchanged transform — the serialized form is identical to
+ ;; what was just exported, so this must NOT re-dirty it.
+ (events/publish-event! :event/transform-update {:object (t2/select-one :model/Transform :id (:id transform))})
+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Transform" :model_id (:id transform))))
+ "A no-op update must leave the transform synced (GHY-3933)")
+ (is (not (sync-object/dirty?))
+ "Instance must not report dirty state after a no-op transform update (GHY-3933)"))))))))
+
(deftest transform-tag-event-creates-sync-object-test
(testing "Creating a transform tag creates a RemoteSyncObject entry when transform sync is enabled"
(mt/with-premium-features #{:transforms-basic}
diff --git a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
index 686182239a6..1433c072446 100644
--- a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
+++ b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx
@@ -40,7 +40,6 @@ export const SearchInputContainer = styled.div<{
}
return css`
background-color: var(--mb-color-background_page-primary);
-
&:hover {
background-color: var(--mb-color-background_page-secondary);
}
diff --git a/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yaml b/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yaml
new file mode 100644
index 00000000000..ea79396193e
--- /dev/null
+++ b/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yaml
@@ -0,0 +1,22 @@
+## Add content_hash column to remote_sync_object for content-based dirty detection (GHY-3933)
+
+databaseChangeLog:
+
+ - changeSet:
+ id: v63.2026-06-18T00:00:00
+ author: ericnormand
+ comment: Add remote_sync_object.content_hash to record a hash of each entity's serialized form as of the last sync.
+ changes:
+ - addColumn:
+ tableName: remote_sync_object
+ columns:
+ - column:
+ name: content_hash
+ type: ${text.type}
+ remarks: SHA-256 (hex) of this entity's serialized YAML as of the last sync or null for no last sync.
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: remote_sync_object
+ columnName: content_hash