The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Track remote-sync dirty state by serialized content hash (GHY-3933) (#76098)
Eric Normand585f612d7492
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.cljindex df5ec237ba7..1f5511b7f07 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj@@ -17,6 +17,7 @@- NativeQuerySnippet, snippets-namespace Collections (when Library is remote-synced)"(:require[java-time.api :as t]+ [metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.spec :as spec][metabase.collections.core :as collections][metabase.events.core :as events]@@ -24,42 +25,57 @@[methodical.core :as methodical][toucan2.core :as t2]))-(defn sync-snippet-tracking!- "Called when the Library collection's remote sync status changes.- When enabled: mark all existing snippets and snippets-namespace collections- as 'create' for initial sync.- When disabled: remove all snippet-related tracking entries."- [enabled?]- (let [timestamp (t/offset-date-time)]- (if enabled?- (do- ;; Mark all snippets-namespace collections for initial sync- (doseq [coll (t2/select [:model/Collection :id :name] :namespace "snippets")]- (t2/insert! :model/RemoteSyncObject- {:model_type "Collection"- :model_id (:id coll)- :model_name (:name coll)- :status "create"- :status_changed_at timestamp}))- ;; Mark all existing snippets for initial sync- (doseq [snippet (t2/select [:model/NativeQuerySnippet :id :name :collection_id])]- (t2/insert! :model/RemoteSyncObject- {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :model_name (:name snippet)- :model_collection_id (:collection_id snippet)- :status "create"- :status_changed_at timestamp})))- (let [snippet-coll-ids (t2/select-pks-set :model/Collection :namespace "snippets")]- (t2/delete! :model/RemoteSyncObject- :model_type "NativeQuerySnippet")- (when (seq snippet-coll-ids)- (t2/delete! :model/RemoteSyncObject- :model_type "Collection"- :model_id [:in snippet-coll-ids]))))))+(defn enable-snippet-tracking!+ "Mark all existing snippets and snippets-namespace collections as 'create' for initial sync."+ []+ (let [timestamp (t/offset-date-time)+ rows (concat+ (for [coll (t2/select [:model/Collection :id :name] :namespace "snippets")]+ {:model_type "Collection"+ :model_id (:id coll)+ :model_name (:name coll)+ :status "create"+ :status_changed_at timestamp})+ (for [snippet (t2/select [:model/NativeQuerySnippet :id :name :collection_id])]+ {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name (:name snippet)+ :model_collection_id (:collection_id snippet)+ :status "create"+ :status_changed_at timestamp}))]+ (when (seq rows)+ (t2/insert! :model/RemoteSyncObject rows))))++(defn disable-snippet-tracking!+ "Remove all snippet-related tracking entries."+ []+ (let [snippet-coll-ids (t2/select-pks-set :model/Collection :namespace "snippets")]+ (t2/delete! :model/RemoteSyncObject+ :model_type "NativeQuerySnippet")+ (when (seq snippet-coll-ids)+ (t2/delete! :model/RemoteSyncObject+ :model_type "Collection"+ :model_id [:in snippet-coll-ids]))));;; ----------------------------------------- Helper Functions ---------------------------------------------------------+(defn- resolve-status+ "Suppresses a no-op 'update' based on status and content_hash, otherwise keep status unchanged."+ [model-type model-id status existing]+ (cond+ (not= "update" status)+ status++ (nil? (:content_hash existing))+ status++ (not= (:content_hash existing)+ (source/row->content-hash {:model_type model-type :model_id model-id}))+ status++ :else ;; hash has not changed+ "synced"))+(defn- create-or-update-remote-sync-object-entry!"Creates or updates a remote sync object entry for a model change.@@ -93,7 +109,7 @@(not (= "create" (:status existing)))(let [model-details (hydrate-details-fn model-id)](t2/update! :model/RemoteSyncObject (:id existing)- {:status status+ {:status (resolve-status model-type model-id status existing):status_changed_at (t/offset-date-time):model_name (:name model-details):model_collection_id (:collection_id model-details)@@ -129,7 +145,7 @@(let [model-details (spec/hydrate-model-details model-spec model-id)fields (spec/build-sync-object-fields model-spec model-details)](t2/update! :model/RemoteSyncObject (:id existing)- (merge {:status status+ (merge {:status (resolve-status model-type model-id status existing):status_changed_at (t/offset-date-time)}fields))))))@@ -227,11 +243,11 @@(and is-now-synced? (not snippets-already-tracked?))(do(log/info "Library collection became remote-synced, enabling snippet sync tracking")- (sync-snippet-tracking! true))+ (enable-snippet-tracking!))(and (not is-now-synced?) snippets-already-tracked?)(do(log/info "Library collection is no longer remote-synced, disabling snippet sync tracking")- (sync-snippet-tracking! false)))))+ (disable-snippet-tracking!)))))(methodical/defmethod events/publish-event! ::collection-change-event[topic event]diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex a6218510f9d..07b27426cc7 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -55,19 +55,6 @@(spec/transforms-namespace-collection? entity))))serdes-paths))))-(defn- sync-objects!- "Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records and- inserts new ones for each imported entity, marking them as 'synced' with the given timestamp.-- Takes a timestamp instant and imported-data map from spec/extract-imported-entities.-- Uses the spec system to determine how to query and build sync objects for each model type."- [timestamp imported-data]- (t2/delete! :model/RemoteSyncObject)- (let [all-inserts (spec/sync-all-entities! timestamp imported-data)]- (when (seq all-inserts)- (t2/insert! :model/RemoteSyncObject all-inserts))))-(defn- build-entity-id-where-clause"Builds a HoneySQL WHERE clause for entity_id filtering.Combines the imported entity-ids exclusion with any spec-level entity_id condition."@@ -219,21 +206,65 @@{:first-import-conflicts (vec first-import-conflicts):deletion-conflicts (spec/check-deletion-conflicts imported-data)}))-(defn- record-exported-paths!- "Records each exported entity's repo file path on its RemoteSyncObject row, so later renames and- deletes can resolve the old path. `entries` is a seq of {:model_type :entity_id :path}; only- entity-id models are recorded. Correlates entity_id -> model_id per model type."- [entries]- (doseq [[model-type es] (group-by :model_type entries)- :let [spec (spec/spec-for-model-type model-type)]- :when (and spec (= :entity-id (:identity spec)))]- (let [eid->path (into {} (map (juxt :entity_id :path)) es)- id->eid (t2/select-pk->fn :entity_id (:model-key spec) :entity_id [:in (vec (keys eid->path))])]- (doseq [[id eid] id->eid- :let [path (eid->path eid)]- :when path]- (t2/update! :model/RemoteSyncObject :model_type model-type :model_id id- {:file_path path})))))+(def app-db-batch-size+ "Max rows per select/update batch, to keep IN-lists and CASE expressions bounded."+ 500)++(defn- import-content-metadata+ "Content-hash entries for imported `rows` ({:model_type :model_id}), re-serializing each entity once to hash+ it. `repo-paths` supplies the real repo path for entity-id models (else the freshly computed one).++ Returns:+ - [{:model_type :model_id :path :content_hash}] (entities that fail to serialize are omitted)"+ [rows repo-paths]+ (let [storage-opts (serdes/storage-base-context)+ repo-by-eid (into {} (map (fn [{:keys [model_type entity_id path]}] [[model_type entity_id] path])) repo-paths)+ serialize (fn [model-type opts id->eid instance]+ (try+ (let [fspec (source/entity->file-spec storage-opts (serdes/extract-one model-type opts instance))+ repo-path (some->> (some-> id->eid (get (:id instance)))+ (vector model-type)+ repo-by-eid)]+ {:model_type model-type+ :model_id (:id instance)+ :path (or repo-path (:path fspec))+ :content_hash (source/content-hash (:content fspec))})+ (catch Exception e+ (log/warnf e "Skipping %s %s: failed to serialize for content hash" model-type (:id instance))+ nil)))]+ ;; One transduction over the model groups: stream each model's extract-query through `serialize` via an+ ;; eduction — extract-one runs while the ResultSet is open, with no intermediate per-model sequence.+ (into []+ (mapcat (fn [[model-type model-rows]]+ (let [spec (spec/spec-for-model-type model-type)+ model-key (:model-key spec)+ opts {:where [:in :id (mapv :model_id model-rows)] :skip-archived true}+ ;; entity-id models: map local id -> entity_id so we can look up the repo path+ id->eid (when (and model-key (= :entity-id (:identity spec)))+ (t2/select-pk->fn :entity_id model-key :id [:in (mapv :model_id model-rows)]))]+ (eduction (keep #(serialize model-type opts id->eid %))+ (serdes/extract-query model-type opts)))))+ (group-by :model_type rows))))++(defn- merge-content-metadata+ "Folds `metadata` ({:model_type :model_id :path :content_hash}) into matching `rows` as :file_path ++ :content_hash; rows with no metadata are unchanged."+ [rows metadata]+ (let [by-key (u/index-by (juxt :model_type :model_id) metadata)]+ (mapv (fn [row]+ (if-let [m (by-key [(:model_type row) (:model_id row)])]+ (assoc row :file_path (:path m) :content_hash (:content_hash m))+ row))+ rows)))++(defn- insert-with-metadata!+ "Inserts RemoteSyncObject `rows` after an import, one `content-hash-batch-size` chunk at a time, folding+ each chunk's file_path + content_hash (`repo-paths` gives entity-id models their real path) into its insert."+ [rows repo-paths]+ (serdes/with-cache+ (doseq [chunk (partition-all app-db-batch-size rows)]+ (t2/insert! :model/RemoteSyncObject+ (merge-content-metadata chunk (import-content-metadata chunk repo-paths))))))(defn- branch-changed-since-scheduling?"Returns true if `pre-task-branch` was captured by the async-* function and the@@ -273,10 +304,12 @@(settings/remote-sync-transforms! true))(t2/with-transaction [_conn](remove-unsynced! (spec/all-syncable-collection-ids) imported-data)- (sync-objects! sync-timestamp imported-data)- ;; Record the actual repo path each entity was read from, so later renames/deletes resolve the- ;; real file and stay on the incremental export fast-path.- (record-exported-paths! (source.ingestable/cached-file-paths base-ingestable))+ ;; Replace the RemoteSyncObject table, folding each entity's repo file_path (so later renames/deletes+ ;; resolve the real file) and serialized-content hash (so a post-pull no-op edit stays synced) into the+ ;; insert. Chunked so insert/IN params and memory stay bounded.+ (t2/delete! :model/RemoteSyncObject)+ (insert-with-metadata! (spec/sync-all-entities! sync-timestamp imported-data)+ (source.ingestable/cached-file-paths base-ingestable))(when finalize! (finalize!)))(when (and (not has-transforms?)(settings/remote-sync-transforms))@@ -287,11 +320,6 @@;;; ------------------------------------------- Incremental Import Fast-Path --------------------------------------------(def ^:private incremental-not-possible- "Sentinel returned by [[incremental-load-snapshot!]] when a change can't be applied incrementally and the- caller must run the full [[load-snapshot!]] instead."- :remote-sync/incremental-import-not-possible)-(def ^:private full-import-models"Models whose change forces a full import on the incremental fast-path: Collection (a rename moves everydescendant's file, a delete cascades to its contents) and the feature models (their presence drives the@@ -339,16 +367,16 @@all-models (into add-models (map :model_type) deleted-rsos)](cond(nil? changed) ;; first import, or diffing not possible (force-push/rebase, non-diffable source)- incremental-not-possible+ :remote-sync/incremental-not-possible(some nil? deleted-rsos) ;; some deleted file not present in appdb- incremental-not-possible+ :remote-sync/incremental-not-possible(some full-import-models all-models) ;; anything not incrementally importable by model type?- incremental-not-possible+ :remote-sync/incremental-not-possible(some #(not= :entity-id (:identity (spec/spec-for-model-type %))) all-models) ;; anything not entity-id model?- incremental-not-possible+ :remote-sync/incremental-not-possible:else{:ingestable ingestable :deleted-rsos deleted-rsos})))@@ -390,10 +418,9 @@(when (seq deletes)(t2/delete! :model/RemoteSyncObject {:where (rso-where-for deletes)}))(when (seq sync-rows)+ ;; fold file_path + content_hash into the insert so the touched rows are written once (chunked)(t2/delete! :model/RemoteSyncObject {:where (rso-where-for sync-rows)})- (t2/insert! :model/RemoteSyncObject sync-rows))- (when ingestable- (record-exported-paths! (source.ingestable/cached-file-paths ingestable)))+ (insert-with-metadata! sync-rows (when ingestable (source.ingestable/cached-file-paths ingestable))))(when finalize! (finalize!)));; We skip the whole-appdb reindex the full load runs. Added/modified entities are already;; re-indexed by the load itself — serdes' t2 insert!/update! fire the :hook/search-index@@ -427,51 +454,27 @@(-> row (dissoc :id) (assoc :status_changed_at timestamp))))))(defn- import-merged!- "Local-only merge for the pull path. Performs an entity-identity 3-way merge of local state against the- remote tip and, on a clean merge, applies the merged result to the LOCAL app DB only — it does not push.- Remote-originated changes end up 'synced'; the un-pushed local changes are preserved as dirty so they- can be pushed later. Sets the last version to the remote tip (so the branch is no longer 'behind').-- On a genuine same-entity conflict, returns `:conflict` without touching local state.-- `base-snapshot` is nil only when no safe 3-way merge is possible (no prior sync, or the base commit was- orphaned by a force-push/rebase) — that surfaces as a conflict. When the remote has not advanced past- the base, there is nothing to fold in, so it is a no-op success that keeps local changes dirty."+ "Import in merge mode. Should only be called when you have a base-snapshot and its version differs from snaphot's version.++ Entity-identity 3-way merge of local state against the remote tip:+ - on same-entity conflict, returns `:conflict` without touching local state;+ - on a clean merge, applies the merged result to the LOCAL app DB only (no push).+ - Marks remote changes synced+ - Local changes stay dirty+ - Sets version to remote tip"[snapshot base-snapshot task-id sync-timestamp]- (let [merge-result (delay (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id))]- (cond- (nil? base-snapshot)- {:status :conflict- :version (source.p/version snapshot)- :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]- :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}-- ;; The remote hasn't advanced past the merge base — nothing to fold in. Keep local changes as-is (still- ;; dirty, to be pushed later); a no-op success, not a spurious "history was rewritten" conflict.- (= (source.p/version base-snapshot) (source.p/version snapshot))- (do- (log/info "Pull merge: remote has not advanced; keeping local changes unchanged")- {:status :success- :version (source.p/version snapshot)- :merge-summary {:added 0 :updated 0 :removed 0}- :outcome {:kind "pull-skipped"}})-- (seq (:conflicts @merge-result))- (let [conflicts (mapv remote-sync.merge/conflict-label (:conflicts @merge-result))]- (log/infof "Pull merge conflict on %d entit(ies): %s"- (count conflicts) (str/join ", " conflicts))+ (let [{:keys [conflicts merged summary]} (source/compute-merge (spec/extract-entities-for-export) snapshot base-snapshot task-id)]+ (if (seq conflicts)+ (let [labels (mapv remote-sync.merge/conflict-label conflicts)]+ (log/infof "Pull merge conflict on %d entit(ies): %s" (count labels) (str/join ", " labels)){:status :conflict:version (source.p/version snapshot)- :conflicts conflicts+ :conflicts labels:message "Import blocked: the same content was changed both locally and on the remote branch."})-- :else- ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are- ;; disjoint from the remote changes, so restoring them reproduces exactly the local diff vs remote.- ;; Restore + set-version run inside the load's transaction so that a crash can't leave the dirty- ;; markers overwritten by the load (which would lose them — a retry captures nothing).- (let [{:keys [merged summary]} @merge-result- dirty-objects (capture-dirty-objects)]+ ;; Capture the local (un-pushed) changes before loading; the clean merge guarantees they are disjoint+ ;; from the remote changes, so restoring them reproduces exactly the local diff vs remote. Restore ++ ;; set-version run inside the load's transaction so a crash can't leave the dirty markers overwritten.+ (let [dirty-objects (capture-dirty-objects)](load-snapshot! (source/specs->snapshot merged) task-id sync-timestamp:finalize! (fn [](restore-dirty-objects! dirty-objects sync-timestamp)@@ -479,39 +482,29 @@(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)(count dirty-objects))- {:status :success- :version (source.p/version snapshot)+ {:status :success+ :version (source.p/version snapshot):merge-summary summary- :outcome {:kind "pulled"- :count (apply + (vals summary))- :branch (settings/remote-sync-branch)}}))))+ :outcome {:kind "pulled"+ :count (apply + (vals summary))+ :branch (settings/remote-sync-branch)}}))))(defn import!"Imports and reloads Metabase entities from a remote snapshot.Takes a non-nil SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, and optional keyword arguments:+ - :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local+ - :base-snapshot - the merge base (last synced version), required when :merge? is set- :force? - forces import even when the snapshot version matches the last imported version or has first-import conflicts- :force-deletion? - forces past deletion conflicts (unsynced local transforms the import would delete); defaults- to :force?. Kept separate so the setup/enable flow can force past version/dirty guards (force? true) while still- surfacing transform-deletion as a conflict the admin must confirm (force-deletion? false) — see GHY-3900.- - :merge? - perform a local-only 3-way merge (keep un-pushed local changes) instead of overwriting local- - :base-snapshot - the merge base (last synced version), required when :merge? is set- - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs- from the current setting at task start, the import aborts with `:error` to protect data- integrity (the load mutates the app DB, so we refuse to proceed when state has drifted)+ to :force?.Loads serialized entities, removes entities not in the import, syncs the remote-sync-object table, andoptionally creates a remote-synced collection.Returns a map with :status (either :success or :error), :version, and :message keys. Various exceptions may be- thrown during import and are caught and converted to error status maps."- [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot pre-task-branch]}]- (when (branch-changed-since-scheduling? pre-task-branch)- (log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"- pre-task-branch (settings/remote-sync-branch))- (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"- {:pre-task-branch pre-task-branch- :current-branch (settings/remote-sync-branch)})))+ thrown during import."+ [^SourceSnapshot snapshot task-id & {:keys [force? force-deletion? merge? base-snapshot]}](log/info "Reloading remote entities from the remote source")(analytics/inc! :metabase-remote-sync/imports)(let [sync-timestamp (t/instant)]@@ -539,21 +532,64 @@incremental-plan (delay (incremental-import-plan snapshot last-imported-version))dirty? (delay (remote-sync.object/dirty?))](cond+ ;; --- Merge mode: fold remote changes into local, keeping un-pushed local changes. ---merge?- (import-merged! snapshot base-snapshot task-id sync-timestamp)+ (cond+ ;; No safe merge base (no prior sync, or base orphaned by a force-push/rebase) — can't 3-way merge.+ (nil? base-snapshot)+ {:status :conflict+ :version snapshot-version+ :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]+ :message "Cannot merge: the remote branch history was rewritten. Discard local changes and pull, or push to a new branch."}++ ;; Remote hasn't advanced past the merge base — nothing to fold in; keep local changes dirty.+ (= (source.p/version base-snapshot) snapshot-version)+ (do+ (log/info "Pull merge: remote has not advanced; keeping local changes unchanged")+ {:status :success+ :version snapshot-version+ :merge-summary {:added 0 :updated 0 :removed 0}+ :outcome {:kind "pull-skipped"}})++ :else+ (import-merged! snapshot base-snapshot task-id sync-timestamp))- ;; Cheap no-op pull: nothing changed remotely, so nothing is loaded or deleted- (and (not force?) (= last-imported-version snapshot-version))+ ;; --- Forced reload: bypasses the no-op/incremental guards. Deletion conflicts (when+ ;; force-deletion? is false) still block; otherwise a full reload. ---+ force?+ (cond+ (seq @blocking-conflicts)+ (let [message (format "Skipping import: snapshot version %s contains conflicts use force to override" snapshot-version)]+ (log/info message)+ {:status :conflict+ :version snapshot-version+ :conflicts (into #{} (map :category) @blocking-conflicts)+ :conflict-details @blocking-conflicts+ :message message})++ :else+ (let [_ (log/info "Remote sync full import: forced")+ imported-data (load-snapshot! snapshot task-id sync-timestamp :finalize! finalize!)]+ (log/info "Successfully reloaded entities from git repository")+ {:status :success+ :version snapshot-version+ :outcome {:kind "pulled"+ :count (pulled-change-count imported-data)+ :branch (settings/remote-sync-branch)}}))++ ;; --- Normal pull ---+ ;; Cheap no-op pull: nothing changed remotely, so nothing is loaded or deleted.+ (= last-imported-version snapshot-version)(do(log/infof "Skipping import: snapshot version %s matches last imported version" snapshot-version){:status :success:version snapshot-version:outcome {:kind "pull-skipped"}})- ;; Incremental fast-path: not forced, no local drift, and the change is incrementally loadable.- ;; Touches only changed files, so it can't wholesale-delete unsynced transforms — no conflict scan,- ;; and only the cheap O(changes) diff is computed (never the full-tree get-conflicts read).- (and (not force?) (not @dirty?) (not= incremental-not-possible @incremental-plan))+ ;; Incremental fast-path: no local drift and the change is incrementally loadable. Tested before the+ ;; conflict. It touches only changed files, so it can't wholesale-delete unsynced transforms.+ (and (not @dirty?)+ (not= :remote-sync/incremental-not-possible @incremental-plan))(incremental-load-snapshot! @incremental-plan snapshot-version task-id sync-timestamp :finalize! finalize!)(seq @blocking-conflicts)@@ -565,10 +601,8 @@:conflict-details @blocking-conflicts:message message})- ;; No blocking conflicts: do the full reload.- :else+ :else ;; fall back to full import(let [reason (cond- force? "forced"@dirty? "local changes pending"first-import? "first import":else "changes not incrementally loadable")@@ -637,7 +671,7 @@:pushed pushed-count:branch (settings/remote-sync-branch)}});; The merge was pushed to `version`, but its commit can't be resolved locally (should not happen —- ;; write-files! updates the local ref before returning). Fail loudly rather than silently advancing+ ;; finish-commit! updates the local ref before returning). Fail loudly rather than silently advancing;; the version while skipping the reconcile, which would leave local missing the folded-in remote;; changes. The push already landed, so a retry sees the divergence and re-merges + reconciles.(throw (ex-info (format (str "Merge pushed to %s but its commit could not be resolved locally to "@@ -651,159 +685,220 @@{:include-field-values false :include-database-secrets false:continue-on-error false :skip-archived true})+(defn- merge-incremental-export-plans-reducer [a b]+ (if (or (= :remote-sync/incremental-not-possible a)+ (= :remote-sync/incremental-not-possible b))+ (reduced :remote-sync/incremental-not-possible)+ (merge-with into a b)))++(defn- merge-incremental-export-plans [a b]+ (unreduced (merge-incremental-export-plans-reducer a b)))+(defn- export-closure- "All `[model-type id]` entities a full export would pull for the entity `[model-type model-id]`+ "All `{:model_type :model_id}` entities a full export would pull for the entity `[model-type model-id]`(its transitive `serdes/descendants` + `serdes/required`, including the entity itself)."[model-type model-id]- (keys (merge-with into- (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))- (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %))))))+ ;; serdes works in [model-type id] tuples; map the closure keys to {:model_type :model_id} on the way out+ (map (fn [[mt id]] {:model_type mt :model_id id})+ (keys (merge-with into+ (u/traverse #{[model-type model-id]} #(serdes/descendants (first %) (second %) closure-opts))+ (u/traverse #{[model-type model-id]} #(serdes/required (first %) (second %)))))))(defn- untracked-content-deps- "The `[model-type id]` entities in the change's export closure that are remote-sync content but have- no RemoteSyncObject row — dependencies a full export would pull (via `serdes/descendants`/`required`)- yet that aren't independently tracked (e.g. a card in a non-synced collection referenced by a synced- card, and that card's collection). These get written alongside the change so the incremental export- matches what a full export would emit. Excludes the entity itself and non-content deps (e.g.- databases, which are resolved by reference at import)."+ "The `{:model_type :model_id}` entities in `[model-type model-id]`'s export closure that are remote-sync+ content but have no RemoteSyncObject row — untracked deps an incremental export must write to match a full+ export. Excludes the entity itself and non-content deps (e.g. databases)."[model-type model-id](into #{}- (filter (fn [[mt id]]- (and (not (and (= mt model-type) (= id model-id)))- (spec/spec-for-model-type mt)- (not (t2/exists? :model/RemoteSyncObject :model_type mt :model_id id)))))+ (filter (fn [{:keys [model_type model_id]}]+ (and (not (and (= model_type model-type) (= model_id model-id)))+ (spec/spec-for-model-type model_type)+ (not (t2/exists? :model/RemoteSyncObject :model_type model_type :model_id model_id)))))(export-closure model-type model-id)))-(defn- dep-upserts- "Serializes the untracked dependency entities `dep-ids` (a set of `[model-type id]`) into file-specs- to upsert, reusing the storage context `opts` so paths dedupe consistently with the changed entities.- Returns `[]` when there are no deps, a vector of file-specs on success, or nil if any dependency's- path is already occupied by a different entity (a collision that requires a full export)."+(defn- ->sized-chunks+ "Builds chunks of maximum size based on model type."+ [rows]+ (for [[model-type rows] (group-by :model_type rows)+ chunk-rows (partition-all app-db-batch-size rows)]+ {:model_type model-type :rows chunk-rows}))++(defn- extract-chunk+ "Extract one chunk's entities in a single query.++ Argument:+ - {:model_type .. :rows [{:model_type .. :model_id ..}]}++ Return:+ - [[row entity]] (if no entity, then omit)"+ [{:keys [model_type rows]}]+ (let [id->row (u/index-by :model_id rows)+ opts {:where [:in :id (mapv :model_id rows)] :skip-archived true}]+ ;; extract-one must run inside the extract-query reduction, while its ResultSet is open+ (into [] (keep (fn [instance]+ (when-let [row (id->row (:id instance))]+ [row (serdes/extract-one model_type opts instance)])))+ (serdes/extract-query model_type opts))))++(defn- path-free?+ "Is this path free to write the entity with eid to?++ Argument:+ - {:eid .. :file-exists? .. :file-eid}++ Return:+ - boolean"+ [{:keys [eid file-exists? file-eid]}]+ (or (not file-exists?) (= eid file-eid)))++(defn- entity-path-info+ "Info for deciding where an entity goes: `{:eid :new-path :file-exists? :file-eid}` — the entity's id, its+ target `path`, and whether a file is already there (`:file-eid` is that file's entity_id, nil if absent or+ unreadable)."+ [snapshot path eid]+ (let [content (source.p/read-file snapshot path)]+ {:eid eid+ :new-path path+ :file-exists? (boolean content)+ :file-eid (when content (try (:entity_id (yaml/parse-string content))+ (catch Exception _ nil)))}))++(defn- dependency->incremental-export-plan [snapshot opts [row entity]]+ (let [path (source/entity->path opts entity)]+ (if (path-free? (entity-path-info snapshot path (:entity_id entity)))+ {:writes [(assoc row :file_path path)]}+ :remote-sync/incremental-not-possible)))++(defn- dependency-chunk->incremental-export-plan+ "Plan fragment ({:writes [{:model_type :model_id :file_path}]}) for one chunk of dependency rows, or+ `:remote-sync/incremental-not-possible` if any target path collides with a different entity."+ [snapshot opts chunk]+ (->> chunk+ (extract-chunk)+ (map #(dependency->incremental-export-plan snapshot opts %))+ (reduce merge-incremental-export-plans-reducer {})))++(defn- dependencies->incremental-export-plan+ "Plan fragment ({:writes [...]}) for the untracked dependency entities `dep-ids` ({:model_type :model_id}),+ one chunk at a time, or `:remote-sync/incremental-not-possible` if any target path collides."[snapshot opts dep-ids]- (if (empty? dep-ids)- []- (let [specs (->> dep-ids- (map (fn [[mt id]] {:model_type mt :model_id id}))- spec/extract-entities-for-rows- (map (fn [e] [(source/entity->file-spec opts e) (:entity_id e)])))]- ;; keep going only if every dependency's target path is free: absent, or already holding that- ;; same entity. A present file whose entity_id can't be read counts as occupied — don't clobber it.- (when (every? (fn [[spec eid]]- (if-let [content (source.p/read-file snapshot (:path spec))]- (= eid (try (:entity_id (yaml/parse-string content))- (catch Exception _ nil)))- true))- specs)- (mapv first specs)))))--(defn- incremental-updates-for-row- "The updates a single dirty `row` contributes to an incremental plan: a map of- {:upserts :delete-paths :synced :removed-ids :pull} (any subset), or `:remote-sync/unsyncable-record`- if the row can't be synced incrementally and the whole batch must fall back to a full export."- [opts snapshot {:keys [status file_path] :as row}]+ (->> dep-ids+ (->sized-chunks)+ (map #(dependency-chunk->incremental-export-plan snapshot opts %))+ (reduce merge-incremental-export-plans-reducer {})))++(defn- row->incremental-export-plan+ "Decide a `row`'s contribution to an incremental plan.++ Return:+ - `:remote-sync/incremental-not-possible` OR+ - {:writes :delete-paths :removed-ids :pull}"+ [{:keys [id model_type model_id status file_path]} file-info](try- (let [info (delay (when-let [entity (first (spec/extract-entities-for-rows [row]))]- (let [spec (source/entity->file-spec opts entity)- content (source.p/read-file snapshot (:path spec))- yaml (when content- ;; errors parsing should throw an invalidate the row- (yaml/parse-string content))]- {:entity entity- :spec spec- :eid (:entity_id entity)- :new-path (:path spec)- :file-exists? (boolean content)- :file-eid (:entity_id yaml)})))]- (cond- (not= :entity-id ;; only entity-id models can be synced incrementally- (:identity (spec/spec-for-model-type (:model_type row))))- :remote-sync/unsyncable-record-- (not (#{"create" "update" "removed" "delete"} status))- :remote-sync/unsyncable-record-- (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export- (str/blank? file_path))- :remote-sync/unsyncable-record-- (#{"removed" "delete"} status)- {:delete-paths [file_path]- :removed-ids [(:id row)]}-- ;; past here, we're seeing create and update statuses- (not @info) ;; entity no longer exists- :remote-sync/unsyncable-record-- ;; create: brand-new file, no old path to delete.- ;; Target must be free or same entity id- (and (= "create" status)- (or (not (:file-exists? @info))- (= (:eid @info) (:file-eid @info))))- {:pull (untracked-content-deps (:model_type row) (:model_id row))- :upserts [(:spec @info)]- :synced [{:id (:id row) :file_path (:new-path @info)}]}-- ;; in-place update: at its stored path, or (no stored path) the repo file at- ;; new-path is already this entity — overwrite.- (and (= "update" status)- (or (= file_path (:new-path @info))- (and (str/blank? file_path)- (= (:eid @info) (:file-eid @info)))))- {:pull (untracked-content-deps (:model_type row) (:model_id row))- :upserts [(:spec @info)]- :synced [{:id (:id row) :file_path (:new-path @info)}]}-- ;; rename: update whose stored path differs from new path. Write the- ;; new file and delete the old one.- (and (= "update" status)- (not (str/blank? file_path))- (not= file_path (:new-path @info))- (or (not (:file-exists? @info))- (= (:eid @info) (:file-eid @info))))- {:pull (untracked-content-deps (:model_type row) (:model_id row))- :upserts [(:spec @info)]- :synced [{:id (:id row) :file_path (:new-path @info)}]- :delete-paths [file_path]}-- ;; any other create/update case (e.g. the target path is occupied by a different entity) needs- ;; a full export- :else- :remote-sync/unsyncable-record))+ (cond+ (not= :entity-id ;; only entity-id models can be synced incrementally+ (:identity (spec/spec-for-model-type model_type)))+ :remote-sync/incremental-not-possible++ (not (#{"create" "update" "removed" "delete"} status))+ :remote-sync/incremental-not-possible++ (and (#{"removed" "delete"} status) ;; removed/delete with no stored path needs a full export+ (str/blank? file_path))+ :remote-sync/incremental-not-possible++ (#{"removed" "delete"} status)+ {:delete-paths [file_path]+ :removed-ids [id]}++ ;; past here, we're seeing create and update statuses+ (not file-info) ;; entity no longer exists+ :remote-sync/incremental-not-possible++ ;; create: brand-new file, no old path to delete.+ ;; Target must be free or same entity id+ (and (= "create" status)+ (path-free? file-info))+ {:pull (untracked-content-deps model_type model_id)+ :writes [{:id id+ :model_type model_type+ :model_id model_id+ :file_path (:new-path file-info)}]}++ ;; in-place update: at its stored path, or (no stored path) the repo file at+ ;; new-path is already this entity — overwrite.+ (and (= "update" status)+ (or (= file_path (:new-path file-info))+ (and (str/blank? file_path)+ (= (:eid file-info) (:file-eid file-info)))))+ {:pull (untracked-content-deps model_type model_id)+ :writes [{:id id+ :model_type model_type+ :model_id model_id+ :file_path (:new-path file-info)}]}++ ;; rename: update whose stored path differs from new path. Write the+ ;; new file and delete the old one.+ (and (= "update" status)+ (not (str/blank? file_path))+ (not= file_path (:new-path file-info))+ (path-free? file-info))+ {:pull (untracked-content-deps model_type model_id)+ :writes [{:id id+ :model_type model_type+ :model_id model_id+ :file_path (:new-path file-info)}]+ :delete-paths [file_path]}++ ;; any other create/update case (e.g. the target path is occupied by a different entity) needs+ ;; a full export+ :else+ :remote-sync/incremental-not-possible)(catch Exception _- :remote-sync/unsyncable-record)))--(defn- incremental-plan- "Builds a plan for a safe incremental export of the current `dirty-rows`, or `:remote-sync/unsyncable-batch` if any- row can't be handled incrementally.-- Every row must be on an entity-id model and be one of:- - `create`/`update` — re-serialize the entity and upsert its file. A create, or an update whose path- is unchanged, is an overwrite-in-place; an update whose stored `file_path` differs is a rename, so- the old file is also deleted. The target path must be free or already hold this same entity, and- the change must not reference remote-sync content that isn't already in the repo (else a full- export, which pulls that dependency via `serdes/descendants`, is required).- - `delete`/`removed` — delete the entity's stored `file_path`. Requires a stored `file_path`- (after a fresh import none is recorded yet, so we fall back to a full export, which self-heals).-- Returns one of:- - {:upserts [file-spec] :delete-paths [path] :synced [{:id :file_path}] :removed-ids [id]} — a safe- incremental plan;- - `:remote-sync/unsyncable-batch` — a row (or a dependency path collision) forces a full export."- [snapshot dirty-rows]- (let [opts (serdes/storage-base-context)- plan (->> dirty-rows- (map #(incremental-updates-for-row opts snapshot %))- (reduce (fn [plan updates]- (if (= updates :remote-sync/unsyncable-record)- (reduced :remote-sync/unsyncable-batch)- (merge-with into plan updates)))- {:upserts [] :delete-paths [] :synced [] :removed-ids [] :pull #{}}))]- (if (= plan :remote-sync/unsyncable-batch)- :remote-sync/unsyncable-batch- (if-let [deps (dep-upserts snapshot opts (:pull plan))]- (-> plan (update :upserts into deps) (dissoc :pull))- ;; a dependency's path is occupied by a different entity — needs a full export- :remote-sync/unsyncable-batch))))+ :remote-sync/incremental-not-possible)))++(defn- chunk->incremental-export-plan+ "Plan fragment for one chunk of create/update rows.++ Returns:+ - :remote-sync/incremental-not-possible when the chunk can't be synced+ - {:writes :delete-paths :removed-ids :pull}"+ [snapshot opts chunk]+ (let [found (extract-chunk chunk)]+ (if (< (count found) (count (:rows chunk)))+ :remote-sync/incremental-not-possible ; extract-chunk omits gone entities; some row is unsyncable+ (->> found+ (map (fn [[row entity]]+ (row->incremental-export-plan row (entity-path-info snapshot (source/entity->path opts entity) (:entity_id entity)))))+ (reduce merge-incremental-export-plans-reducer {})))))++(defn- incremental-export-plan+ "Build an incremental export plan for `rows` (`RemoteSyncObject`s) against `snapshot`, one chunk at a time.++ Returns:+ - {:writes [{:id :model_type :model_id :file_path}] :delete-paths [path] :removed-ids [id]}, or+ - :remote-sync/incremental-not-possible when any row can't go incrementally"+ [snapshot rows]+ (let [opts (serdes/storage-base-context)+ ;; create/update rows on entity-id models need an entity (extracted per chunk); everything else+ ;; (removed/delete, non-entity-id, bad status) is decided with no entity+ {cu-rows true other-rows false} (group-by #(boolean (and (#{"create" "update"} (:status %))+ (= :entity-id (:identity (spec/spec-for-model-type (:model_type %))))))+ rows)+ ;; the no-extraction rows first (cheap)+ plan (->> other-rows+ (map #(row->incremental-export-plan % nil))+ (reduce merge-incremental-export-plans-reducer {:writes [] :delete-paths [] :removed-ids [] :pull #{}}))+ plan (->> cu-rows+ (->sized-chunks)+ (map #(chunk->incremental-export-plan snapshot opts %))+ (reduce merge-incremental-export-plans-reducer plan))+ plan (->> (:pull plan) ;; nil when plan is already :incremental-not-possible+ (dependencies->incremental-export-plan snapshot opts)+ (merge-incremental-export-plans plan))]+ (if (= plan :remote-sync/incremental-not-possible)+ :remote-sync/incremental-not-possible+ (dissoc plan :pull))))(defn- path-top-level-dir [^String path](let [i (str/index-of path "/")]@@ -816,64 +911,148 @@(not (settings/remote-sync-transforms)) (into ["transforms" "python-libraries" "python_libraries"])(not (settings/library-is-remote-synced?)) (conj "snippets")))+(defn- stage-write [commit opts [row entity]]+ (let [path (or (:file_path row) (source/entity->path opts entity))+ content (source/entity->content entity)]+ (source.p/stage-upsert! commit {:path path :content content})+ (when (:id row)+ [{:id (:id row) :file_path path :content_hash (source/content-hash content)}])))++(defn- chunk-stage-writes+ [commit opts chunk]+ (->> chunk+ (extract-chunk)+ (mapcat #(stage-write commit opts %))+ (doall)))++(defn- stage-writes+ "Stage WriteRows ({:model_type :model_id, optional :id/:file_path}) to `commit`, chunking internally per+ model type so each chunk's entities load in one query with bounded memory. Callers pass a flat seq.++ Returns:+ - [{:id :file_path :content_hash}]"+ [commit opts rows]+ (->> rows+ (->sized-chunks)+ (mapcat #(chunk-stage-writes commit opts %))+ (doall)))++(defn- stage-deletes [commit delete-paths]+ (doseq [delete-path delete-paths]+ (source.p/stage-delete! commit delete-path)))++(defn- exportable-write-rows+ "WriteRows for a full export — every exportable id tagged with its RemoteSyncObject id (untracked deps get :id nil)."+ []+ (let [rso-id (u/index-by (juxt :model_type :model_id) :id (t2/select [:model/RemoteSyncObject :id :model_type :model_id]))]+ (for [[model ids] (spec/exportable-entities)+ id ids]+ {:model_type model :model_id id :id (rso-id [model id])})))++(defn- find-departed-entities+ "Find RSO rows for entity-id entities that left the synced set (removed/delete status and not in+ `targets`), matching the incremental path; path/hybrid and still-exported rows are kept."+ [exported-rows]+ (let [exported (into #{} (map #(select-keys % [:model_type :model_id])) exported-rows)]+ (->> (t2/select [:model/RemoteSyncObject :id :model_type :model_id]+ :status [:in ["removed" "delete"]])+ (filter #(= :entity-id (:identity (spec/spec-for-model-type (:model_type %)))))+ (remove #(exported (select-keys % [:model_type :model_id])))+ (map :id))))++(defn- mark-rows-synced!+ "Mark `ids` synced via chunked CASE updates. Rows that appear in `synced` ([{:id :file_path :content_hash}])+ get their file_path/content_hash written; any other id in `ids` keeps its existing path/hash. Callers pick the+ scope: the incremental export passes only its write set, a full export passes every RemoteSyncObject id."+ [ids synced sync-timestamp]+ (let [by-id (u/index-by :id synced)]+ (doseq [id-chunk (partition-all app-db-batch-size ids)+ :let [hits (filter by-id id-chunk)]]+ (t2/update! :model/RemoteSyncObject+ {:id [:in (vec id-chunk)]}+ (cond-> {:status "synced" :status_changed_at sync-timestamp}+ (seq hits)+ (assoc :file_path (into [:case]+ (concat+ (mapcat (fn [id]+ [[:= :id id] (:file_path (by-id id))])+ hits)+ [:else :file_path]))+ :content_hash (into [:case]+ (concat+ (mapcat (fn [id]+ [[:= :id id] (:content_hash (by-id id))])+ hits)+ [:else :content_hash]))))))))+(defn- full-export!- "Re-serializes the entire remote-synced set into `snapshot` and commits it, then marks every- RemoteSyncObject row synced (as of `sync-timestamp`) and records each entity's file_path. Used when- the pending changes can't be applied incrementally (name collisions, disabled content needing a- reconcile, etc.). Throws if there is no remote-syncable content. Returns {:status :success}."+ "Re-serialize and commit the entire remote-synced set, then reconcile every RemoteSyncObject.++ Returns:+ - {:status :success}+ or throws"[snapshot task-id message sync-timestamp]- (let [models (spec/extract-entities-for-export)]- (when (not models)+ (let [export-rows (exportable-write-rows)]+ (when (empty? export-rows)(throw (ex-info "No remote-syncable content available." {})))(remote-sync.task/update-progress! task-id 0.3)- (let [{:keys [version entries]} (source/store! models snapshot task-id message)]- (remote-sync.task/set-version! task-id version)- (t2/update! :model/RemoteSyncObject {:status "synced" :status_changed_at sync-timestamp})- (record-exported-paths! entries)+ (let [opts (serdes/storage-base-context)+ commit (source.p/open-commit snapshot)+ [synced version] (try+ (source.p/replace-all! commit) ; replace the managed dirs wholesale+ [(stage-writes commit opts export-rows)+ (source.p/finish-commit! commit message)]+ (catch Throwable e+ (source.p/abort-commit! commit)+ (throw e)))]+ (t2/with-transaction [_]+ (remote-sync.task/set-version! task-id version)+ (doseq [removed-ids (partition-all 500 (find-departed-entities export-rows))]+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))+ (mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced sync-timestamp)){:status :success- :outcome {:kind "pushed"- :count (count entries)- :branch (settings/remote-sync-branch)}})))+ :outcome {:kind "pushed" :count (count synced) :branch (settings/remote-sync-branch)}})))++(defn- incremental-export!+ [plan disabled-files task-id snapshot message sync-timestamp]+ (let [{:keys [writes delete-paths removed-ids]} plan+ delete-paths (into (vec delete-paths) disabled-files)]+ (remote-sync.task/update-progress! task-id 0.3)+ (let [opts (serdes/storage-base-context)+ commit (source.p/open-commit snapshot)+ [synced version] (try+ (let [synced (stage-writes commit opts writes)]+ (stage-deletes commit delete-paths)+ [synced (source.p/finish-commit! commit message)])+ (catch Throwable e+ (source.p/abort-commit! commit)+ (throw e)))]+ (t2/with-transaction [_]+ (remote-sync.task/set-version! task-id version)+ ;; delete departed rows first, then update RSO metadata — same order as full-export!+ (doseq [removed-ids (partition-all 500 removed-ids)]+ (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))+ (mark-rows-synced! (map :id synced) synced sync-timestamp)))+ (log/infof "Remote sync incremental export: wrote %d, deleted %d" (count writes) (count delete-paths))+ {:status :success+ :outcome {:kind "pushed"+ :count (+ (count writes) (count delete-paths))+ :branch (settings/remote-sync-branch)}}))(defn export!"Exports remote-synced collections to a remote source repository.Takes a SourceSnapshot instance, a RemoteSyncTask ID for progress tracking, a commit message string, andoptional keyword arguments:- - :force? - when true, overwrite the remote branch wholesale even if it has advanced (no merge)- - :merge? - when true and the remote has advanced, perform a 3-way merge (rather than refusing)+ - :force? - when true, overwrite the remote branch wholesale unconditionally (destructive)+ - :merge? - when true and the remote has advanced, perform a 3-way merge if no conflicts- :source - the Source the snapshot came from, used to resolve the merge base and the merged result- :base-snapshot - a snapshot of the last synced version (the merge base), supplied when the remotebranch has advanced and a 3-way merge is required- - :pre-task-branch - the value of `remote-sync-branch` at scheduling time; if it differs- from the current setting at task start, the export aborts with `:error` (defense-in-depth- against any path that mutates the setting between scheduling and the work running).-- Takes the incremental fast-path when every pending change can be applied incrementally (see- `incremental-plan` — creates, in-place updates, renames, and deletes of entity-id models). Otherwise does a- full export: re-serializes the entire synced set, writes it, marks all RemoteSyncObject rows synced,- and records each entity's `file_path` so future renames/deletes can go incremental.-- Behavior when the remote branch has advanced beyond the last sync:- - `force?` -> overwrite the remote wholesale (full re-serialize).- - `merge?` -> 3-way merge; non-conflicting remote changes are merged in and reconciled into the- local app DB; genuine same-entity conflicts return a `:conflict`.- - neither (default) -> refuse with `:conflict` (the UI's export preflight decides force/branch/merge).-- When the remote has NOT advanced, takes the incremental fast-path (see `incremental-plan`) when- possible, otherwise a full re-serialize.Returns a map with :status (`:success`, `:conflict`, or `:error`), and optionally :message, :version,and :merge-summary keys. Various exceptions are caught and converted to error status maps."- [^SourceSnapshot snapshot task-id message & {:keys [force? merge? base-snapshot pre-task-branch] src :source}]- (when (branch-changed-since-scheduling? pre-task-branch)- (log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"- pre-task-branch (settings/remote-sync-branch))- (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"- {:pre-task-branch pre-task-branch- :current-branch (settings/remote-sync-branch)})))- (when (not snapshot)- (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable." {})))+ [^SourceSnapshot snapshot task-id message & {:keys [force? merge? base-snapshot] src :source}](let [sync-timestamp (t/instant)](try(analytics/inc! :metabase-remote-sync/exports)@@ -882,14 +1061,11 @@remote-version (source.p/version snapshot)diverged? (and (some? base-version)(not= base-version remote-version))- ;; Pending-change inputs for the undiverged path; only computed there (incremental-plan can- ;; be expensive, and they are unused on the force/merge branches).- normal? (and (not force?) (not diverged?))- disabled-files (when normal?- (filterv (comp (disabled-content-dirs) path-top-level-dir)- (source.p/list-files snapshot)))- dirty-rows (when normal? (seq (remote-sync.object/dirty-rows)))- plan (when dirty-rows (incremental-plan snapshot dirty-rows))]+ disabled-files (delay (filterv (comp (disabled-content-dirs) path-top-level-dir)+ (source.p/list-files snapshot)))+ dirty-rows (delay (remote-sync.object/dirty-rows))+ plan (delay (when (seq @dirty-rows)+ (incremental-export-plan snapshot @dirty-rows)))](cond;; Forced overwrite — full re-serialize, replacing managed dirs (discards remote divergence).force?@@ -897,59 +1073,38 @@(log/info "Remote sync full export: forced overwrite")(full-export! snapshot task-id message sync-timestamp))- ;; Remote hasn't advanced and there's nothing to export: no dirty rows and no stale files in- ;; now-disabled content dirs.- (and (not diverged?) (empty? dirty-rows) (empty? disabled-files))- (do- (log/info "Remote sync export: no changes to export")- {:status :success- :outcome {:kind "push-skipped"}})-- ;; Remote hasn't advanced but a dirty row can't be applied incrementally → full re-serialize.- (and (not diverged?) (= plan :remote-sync/unsyncable-batch))- (do- (log/info "Remote sync full export: a pending change can't be applied incrementally")- (full-export! snapshot task-id message sync-timestamp))+ (and diverged? merge?)+ (cond+ (nil? base-snapshot)+ {:status :conflict+ :version remote-version+ :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]+ :message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}- ;; Remote hasn't advanced — incremental fast-path: write only the changed entities, deleting- ;; their old paths plus files left behind in now-disabled content dirs, preserving every other.- (not diverged?)- (let [{:keys [upserts delete-paths synced removed-ids]} plan- delete-paths (into (vec delete-paths) disabled-files)]- (remote-sync.task/update-progress! task-id 0.3)- (let [written-version (source.p/apply-changes! snapshot message upserts delete-paths)]- (remote-sync.task/set-version! task-id written-version))- (doseq [{:keys [id file_path]} synced]- (t2/update! :model/RemoteSyncObject :id id- {:status "synced" :file_path file_path :status_changed_at sync-timestamp}))- (when (seq removed-ids)- (t2/delete! :model/RemoteSyncObject :id [:in removed-ids]))- (log/infof "Remote sync incremental export: wrote %d, deleted %d"- (count upserts) (count delete-paths))- {:status :success- :outcome {:kind "pushed"- :count (+ (count upserts) (count delete-paths))- :branch (settings/remote-sync-branch)}})+ :else+ (export-merged! src snapshot base-snapshot task-id message sync-timestamp+ (spec/extract-entities-for-export)))- ;; Remote advanced and the caller did not ask to merge — refuse. The UI's export preflight- ;; drives the choice between force, new branch, and merge.- (not merge?)+ diverged? ;; and not merge? option{:status :conflict:version remote-version:conflicts []:message "The remote branch has changed since your last sync. Choose how to proceed."}- ;; Merge requested but the merge base is gone (force-push/rebase) — no safe 3-way merge.- (nil? base-snapshot)- {:status :conflict- :version remote-version- :conflicts ["Remote history was rewritten (force-push or rebase); cannot merge automatically."]- :message "Cannot merge: the remote branch history was rewritten. Re-import then export, or force the export to overwrite."}+ ;; There's nothing to export: no dirty rows and no stale files.+ (and (empty? @dirty-rows) (empty? @disabled-files))+ (do+ (log/info "Remote sync export: no changes to export")+ {:status :success+ :outcome {:kind "push-skipped"}})- ;; Merge: 3-way merge, push the result, and reconcile the local app DB.- :else- (export-merged! src snapshot base-snapshot task-id message sync-timestamp- (spec/extract-entities-for-export)))))+ (not= @plan :remote-sync/incremental-not-possible)+ (incremental-export! @plan @disabled-files task-id snapshot message sync-timestamp)++ :else ;; fall back to full+ (do+ (log/info "Remote sync full export: a pending change can't be applied incrementally")+ (full-export! snapshot task-id message sync-timestamp)))))(catch Exception e;; handle-task-result! records the failure on this result, and skips entirely when the task;; was already cancelled (ended_at set) — so cancellation needs no special case here.@@ -1200,13 +1355,18 @@:conflicts true})))(run-async! "import" branch(fn [task-id]+ (when (branch-changed-since-scheduling? pre-task-branch)+ (log/warnf "Aborting import: remote-sync-branch changed from %s to %s since task was scheduled"+ pre-task-branch (settings/remote-sync-branch))+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"+ {:pre-task-branch pre-task-branch+ :current-branch (settings/remote-sync-branch)})))(import! snapshot task-id(assoc import-args:force? force?:force-deletion? force-deletion?:merge? merge?- :base-snapshot base-snapshot- :pre-task-branch pre-task-branch)))+ :base-snapshot base-snapshot))):on-success on-success)))(defn async-export!@@ -1227,6 +1387,9 @@Returns a RemoteSyncTask."[branch force? message & {:keys [on-success merge?]}](guards/ensure-no-active-task!)+ (when-not (settings/remote-sync-enabled)+ (throw (ex-info "Remote sync source is not enabled. Please configure MB_GIT_SOURCE_REPO_URL environment variable."+ {:status-code 400})))(let [pre-task-branch (settings/remote-sync-branch)source (source/source-from-settings branch)last-task-version (remote-sync.task/last-version)@@ -1239,12 +1402,17 @@(source.p/snapshot-at source last-task-version))](run-async! "export" branch(fn [task-id]+ (when (branch-changed-since-scheduling? pre-task-branch)+ (log/warnf "Aborting export: remote-sync-branch changed from %s to %s since task was scheduled"+ pre-task-branch (settings/remote-sync-branch))+ (throw (ex-info "Branch setting changed since task was scheduled; aborting to protect data integrity"+ {:pre-task-branch pre-task-branch+ :current-branch (settings/remote-sync-branch)})))(export! snapshot task-id message:force? force?:merge? merge?:source source- :base-snapshot base-snapshot- :pre-task-branch pre-task-branch))+ :base-snapshot base-snapshot)):on-success on-success)))(defn preview-export-merge@@ -1270,7 +1438,7 @@no-changes(if-let [base-snapshot (source.p/snapshot-at source base-version)](serdes/with-cache- (if-let [models (spec/extract-entities-for-export)]+ (if-let [models (seq (spec/extract-entities-for-export))](assoc (source/preview-merge models snapshot base-snapshot nil) :diverged? true)(assoc no-changes :diverged? true)));; No merge base — the remote history was rewritten. A merge is impossible, but a force push is@@ -1278,7 +1446,7 @@{:diverged? true :clean? false :reason :history-rewritten:conflicts [] :summary {:added 0 :updated 0 :removed 0}:force-push-casualties (serdes/with-cache- (if-let [models (spec/extract-entities-for-export)]+ (if-let [models (seq (spec/extract-entities-for-export))](source/force-push-casualties-no-base models snapshot){:deleted [] :overwritten []}))}))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source.cljindex 2a03d2d3a9c..7e1be88f6b5 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source.clj@@ -1,11 +1,14 @@(ns metabase-enterprise.remote-sync.source(:require+ [buddy.core.codecs :as codecs]+ [buddy.core.hash :as buddy-hash][clojure.string :as str][metabase-enterprise.remote-sync.merge :as remote-sync.merge][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.source.git :as git][metabase-enterprise.remote-sync.source.ingestable :as ingestable][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.spec :as spec][metabase-enterprise.serialization.core :as serialization][metabase.models.serialization :as serdes][metabase.settings.core :as setting]@@ -31,10 +34,7 @@(when (some (fn [path-filter] (re-matches path-filter path)) path-filters)(source.p/read-file original-snapshot path)))- (write-files! [_ _message _files]- (throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))-- (apply-changes! [_ _message _upserts _delete-paths]+ (open-commit [_](throw (UnsupportedOperationException. "WrappingSnapshot is a read-only ingestion view, not a write target.")))(version [_]@@ -47,20 +47,38 @@(atom nil) (atom []))(seq root-dependencies) (ingestable/wrap-root-dep-ingestable root-dependencies)))-(defn- remote-sync-path+(defn entity->path+ "The repo-relative path an extracted `entity` serializes to, using storage context `opts`."[opts entity](let [resolved (serialization/resolve-storage-path opts entity)dirnames (drop-last resolved)basename (str (last resolved) ".yaml")](str/join File/separator (concat dirnames [basename]))))+(defn entity->content+ "The serialized YAML string for an extracted `entity`."+ [entity]+ (yaml/generate-string (serialization/serialization-deep-sort entity)+ {:dumper-options {:flow-style :block :split-lines false}}))+(defn entity->file-spec"Serializes a single extracted entity into a `{:path :content}` file spec, using storage context`opts` (from `serdes/storage-base-context`)."[opts entity]- {:path (remote-sync-path opts entity)- :content (yaml/generate-string (serialization/serialization-deep-sort entity)- {:dumper-options {:flow-style :block :split-lines false}})})+ {:path (entity->path opts entity)+ :content (entity->content entity)})++(defn content-hash+ "SHA-256 (hex) of a serialized YAML `content` string."+ [^String content]+ (codecs/bytes->hex (buddy-hash/sha256 content)))++(defn row->content-hash+ "SHA-256 (hex) of the serialized YAML for the entity named by `row` ({:model_type :model_id}), or nil if it+ can't be extracted. Hashes the live DB serialization (never on-disk bytes), so it's stable across sync points."+ [row]+ (when-let [entity (first (spec/extract-entities-for-rows [row]))]+ (content-hash (:content (entity->file-spec (serdes/storage-base-context) entity)))))(defn serialize-specs"Serializes a stream of entities into an eager vector of `{:path :content}` file specs. Reports progress@@ -82,31 +100,6 @@spec)))stream)))-(defn store!- "Stores serialized entities from a stream to a remote source and commits the changes.-- Takes a stream (a sequence of serialized entities to be stored), a snapshot (the remote source- implementing the SourceSnapshot protocol where files will be written), a task-id (the RemoteSyncTask- identifier used to track progress updates), and a message (the commit message to use when writing- files to the source).-- Returns `{:version <written-version> :entries [{:model_type :entity_id :path} ...]}`."- [stream snapshot task-id message]- (let [opts (serdes/storage-base-context)- stream-count (bounded-count 10000 stream)- entries (volatile! [])- version (->> stream- (map-indexed (fn [idx entity]- (let [spec (entity->file-spec opts entity)]- (vswap! entries conj {:model_type (-> entity :serdes/meta last :model)- :entity_id (:entity_id entity)- :path (:path spec)})- (remote-sync.task/update-progress!- task-id (-> (min (inc idx) stream-count) (/ stream-count) (* 0.65) (+ 0.3)))- spec)))- (source.p/write-files! snapshot message))]- {:version version :entries @entries}))-(defn- snapshot->specs"Reads a snapshot's managed-directory files into a sequence of `{:path :content}` specs, matching theshape produced by [[serialize-specs]]. Used to read the merge base and remote-tip trees for merging."@@ -140,14 +133,13 @@(defn specs->snapshot"Builds an in-memory read-only SourceSnapshot backed by `specs` (a seq of `{:path :content}`), so merged- content can be loaded into the app DB without writing it to git. `write-files!` is unsupported."+ content can be loaded into the app DB without writing it to git. Writing is unsupported."[specs](let [by-path (into {} (map (juxt :path :content)) specs)](reify source.p/SourceSnapshot(list-files [_] (vec (keys by-path)))(read-file [_ path] (get by-path path))- (write-files! [_ _ _] (throw (ex-info "in-memory merge snapshot is read-only" {})))- (apply-changes! [_ _ _ _] (throw (ex-info "in-memory merge snapshot is read-only" {})))+ (open-commit [_] (throw (ex-info "in-memory merge snapshot is read-only" {})))(version [_] nil))))(defn preview-merge@@ -174,6 +166,20 @@[stream snapshot](remote-sync.merge/force-push-casualties [] (serialize-specs stream nil) (snapshot->specs snapshot)))+(defn- replace-all-files!+ "Wholesale-commit `file-specs` ({:path :content}) to `snapshot`: clear the managed dirs, stage every spec,+ and push. Returns the new version; aborts the commit on any error."+ [snapshot message file-specs]+ (let [commit (source.p/open-commit snapshot)]+ (try+ (source.p/replace-all! commit)+ (doseq [spec file-specs]+ (source.p/stage-upsert! commit spec))+ (source.p/finish-commit! commit message)+ (catch Throwable e+ (source.p/abort-commit! commit)+ (throw e)))))+(defn merge-and-store!"Like [[store!]], but reconciles the freshly serialized local state against a remote branch that hasadvanced beyond the last sync. Performs an entity-identity 3-way merge of:@@ -189,7 +195,7 @@(if (seq conflicts){:status :conflict :conflicts conflicts :summary summary}{:status :success- :version (source.p/write-files! snapshot message merged)+ :version (replace-all-files! snapshot message merged):summary summary})))(defn source-from-settingsdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.cljindex e453ef46cba..01a8103e9fd 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/git.clj@@ -14,9 +14,10 @@(java.net URI)(org.apache.commons.io FileUtils)(org.eclipse.jgit.api Git GitCommand TransportCommand)- (org.eclipse.jgit.dircache DirCache DirCacheEditor$DeletePath DirCacheEditor$DeleteTree- DirCacheEditor$PathEdit DirCacheEntry)+ (org.eclipse.jgit.dircache DirCache DirCacheBuilder DirCacheEditor DirCacheEditor$DeletePath+ DirCacheEditor$DeleteTree DirCacheEditor$PathEdit DirCacheEntry)(org.eclipse.jgit.lib CommitBuilder Constants FileMode ObjectId PersonIdent Ref Repository)+ (org.eclipse.jgit.lib ObjectInserter ObjectReader)(org.eclipse.jgit.revwalk RevCommit RevWalk)(org.eclipse.jgit.transport PushResult RefSpec RemoteRefUpdateRemoteRefUpdate$Status UsernamePasswordCredentialsProvider)@@ -289,68 +290,76 @@(str/replace-first (.getName ^Ref target) "refs/heads/" ""))))(throw (ex-info "Failed to get a default branch for git repository." {:head-ref head-ref})))))-(defn- commit-edits!- "Builds a commit by editing the branch tip's tree in place and pushes it. Seeds an in-core index from- the parent commit's tree — so every unchanged entry and subtree is carried forward by object id — then- applies only the edits: whole subtrees named in `delete-dirs` are removed (used to fully replace a- managed directory), paths in `delete-paths` are removed, and `upserts` (file specs with :path/:content)- are written as new blobs. `writeTree` only rewrites the subtrees on the path to an edit, so the work is- proportional to the number of changes rather than the size of the repo.-- Returns the new commit sha. Throws ExceptionInfo if the write or push fails."- [{:keys [^Git git ^String version ^String branch] :as snapshot} ^String message upserts delete-paths delete-dirs]- (let [repo (.getRepository git)- branch-ref (qualify-branch branch)- parent-id (.resolve repo version)]- (with-open [inserter (.newObjectInserter repo)- reader (.newObjectReader repo)- rev-walk (RevWalk. repo)]- (let [index (DirCache/newInCore)]- ;; Seed the index from the parent tree; unchanged entries and subtrees carry forward by id.- (let [builder (.builder index)]- (when parent-id- (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader- (.getTree (.parseCommit rev-walk parent-id))))- (.finish builder))- ;; Apply only the edits.- (let [editor (.editor index)]- (doseq [^String dir delete-dirs]- (.add editor (DirCacheEditor$DeleteTree. dir)))- (doseq [^String path delete-paths]- (.add editor (DirCacheEditor$DeletePath. path)))- (doseq [{:keys [^String path content]} upserts]- (let [blob-id (.insert inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]- (.add editor (proxy [DirCacheEditor$PathEdit] [path]- (apply [^DirCacheEntry entry]- (.setFileMode entry FileMode/REGULAR_FILE)- (.setObjectId entry blob-id))))))- (.finish editor))- (let [tree-id (.writeTree index inserter)- commit-builder (doto (CommitBuilder.)- (.setTreeId tree-id)- (.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))- (.setCommitter (PersonIdent. "Metabase Library" "[email protected]"))- (.setMessage message))]- (when parent-id- (.setParentId commit-builder parent-id))- (let [commit-id (.insert inserter commit-builder)]- (.flush inserter)- (doto (.updateRef repo branch-ref)- (.setNewObjectId commit-id)- (.update))- (push-branch! snapshot)- (.name commit-id)))))))--(defn- write-files!- "Full export: replace every managed dir wholesale with `files` (managed-dir files not in `files` are- removed), preserving everything outside the managed dirs."- [{:keys [managed-dirs] :as snapshot} ^String message files]- (commit-edits! snapshot message files nil managed-dirs))--(defn- apply-changes!- "Incremental patch: write `upserts`, remove `delete-paths`, and preserve every other file."- [snapshot ^String message upserts delete-paths]- (commit-edits! snapshot message upserts delete-paths nil))+(defn- close-commit-resources! [inserter reader rev-walk]+ (.close ^ObjectInserter inserter)+ (.close ^ObjectReader reader)+ (.close ^RevWalk rev-walk))++;; A commit being built incrementally against a GitSnapshot. Holds the open JGit resources (inserter, reader,+;; rev-walk) and the in-core index/editor; blobs are inserted as files are staged and the tree is written and+;; pushed at finish. Edits the branch tip's tree in place — unchanged entries/subtrees carry forward by object+;; id — so writeTree's work is proportional to the number of changes, not the repo size.+(defrecord GitCommit [snapshot inserter reader rev-walk index editor parent-id]+ source.p/CommitBuilder+ (stage-upsert! [_ {:keys [^String path content]}]+ (let [blob-id (.insert ^ObjectInserter inserter Constants/OBJ_BLOB (.getBytes ^String content "UTF-8"))]+ (.add ^DirCacheEditor editor+ (proxy [DirCacheEditor$PathEdit] [path]+ (apply [^DirCacheEntry entry]+ (.setFileMode entry FileMode/REGULAR_FILE)+ (.setObjectId entry blob-id)))))+ nil)++ (stage-delete! [_ path]+ (.add ^DirCacheEditor editor (DirCacheEditor$DeletePath. ^String path))+ nil)++ (replace-all! [_]+ (doseq [^String dir (:managed-dirs snapshot)]+ (.add ^DirCacheEditor editor (DirCacheEditor$DeleteTree. dir)))+ nil)++ (finish-commit! [_ message]+ (.finish ^DirCacheEditor editor)+ (let [^Git git (:git snapshot)+ repo (.getRepository git)+ branch-ref (qualify-branch (:branch snapshot))+ tree-id (.writeTree ^DirCache index ^ObjectInserter inserter)+ commit-builder (doto (CommitBuilder.)+ (.setTreeId tree-id)+ (.setAuthor (PersonIdent. "Metabase Library" "[email protected]"))+ (.setCommitter (PersonIdent. "Metabase Library" "[email protected]"))+ (.setMessage ^String message))]+ (when parent-id+ (.setParentId commit-builder parent-id))+ (let [commit-id (.insert ^ObjectInserter inserter commit-builder)]+ (.flush ^ObjectInserter inserter)+ (doto (.updateRef repo branch-ref)+ (.setNewObjectId commit-id)+ (.update))+ (push-branch! snapshot)+ (close-commit-resources! inserter reader rev-walk) ; close only after a successful push+ (.name commit-id))))++ (abort-commit! [_]+ (close-commit-resources! inserter reader rev-walk)+ nil))++(defn- open-commit*+ "Begin a GitCommit against `snapshot`, seeding the in-core index from the parent tree."+ [{:keys [^Git git ^String version] :as snapshot}]+ (let [repo (.getRepository git)+ parent-id (.resolve repo version)+ inserter (.newObjectInserter repo)+ reader (.newObjectReader repo)+ rev-walk (RevWalk. repo)+ index (DirCache/newInCore)]+ (let [^DirCacheBuilder builder (.builder index)]+ (when parent-id+ (.addTree builder (byte-array 0) DirCacheEntry/STAGE_0 reader+ (.getTree (.parseCommit rev-walk parent-id))))+ (.finish builder))+ (->GitCommit snapshot inserter reader rev-walk index (.editor index) parent-id)))(defn branches"Retrieves all branch names from the remote repository.@@ -429,11 +438,8 @@(read-file [this path](read-file this path))- (write-files! [this message files]- (write-files! this message files))-- (apply-changes! [this message upserts delete-paths]- (apply-changes! this message upserts delete-paths))+ (open-commit [this]+ (open-commit* this))(version [this](:version this))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.cljindex f035a58d8a5..dac3be98fb1 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/source/protocol.clj@@ -56,26 +56,8 @@Returns the file contents as a string, or nil if the file doesn't exist.")- (write-files! [snapshot message files]- "Writes multiple files to the source with a commit message.-- Takes a SourceSnapshot instance implementing this protocol, a message (the commit message to use when writing files),- and files (a sequence of file specs). Each file spec is a map with :path and :content keys.-- All existing files within managed directories that are not in the write set are removed.- Files outside managed directories are always preserved.-- Returns the version of the written files.")-- (apply-changes! [snapshot message upserts delete-paths]- "Incrementally updates the source: writes/overwrites `upserts`, removes `delete-paths`,- and PRESERVES every other existing file (unlike `write-files!`, which deletes managed-dir- files absent from the write set).-- Takes a SourceSnapshot instance, a commit message, `upserts` (a sequence of file specs, each- a map with :path and :content), and `delete-paths` (a sequence of path strings to remove).-- Returns the version of the written files.")+ (open-commit [snapshot]+ "Begin building one commit incrementally; returns a CommitBuilder.")(version [snapshot]"Gets a version identifier for the current state of the snapshot.@@ -84,6 +66,26 @@Returns a version identifier string (e.g., a git SHA)."))+(defprotocol CommitBuilder+ "A single commit being built incrementally — stage files one at a time, then finish (or abort). Lets the+ caller stream files into a commit without holding them all at once.++ Lifecycle / resources: a builder holds native resources open (for git, a JGit `ObjectInserter`,+ `ObjectReader`, and `RevWalk`) from `open-commit` until they're released. The caller MUST eventually call+ `finish-commit!` (on success) or `abort-commit!` (on failure) to release them — including when staging+ throws, so callers should wrap staging in `try` and `abort-commit!` in the `catch`/`finally`. A builder+ left open leaks those resources; the lifecycle is the caller's responsibility, not the builder's."+ (stage-upsert! [commit file-spec]+ "Stage one file (a `{:path :content}` map) into the commit. Returns nil.")+ (stage-delete! [commit path]+ "Stage removal of `path` from the commit. Returns nil.")+ (replace-all! [commit]+ "Clear the snapshot's managed dirs, so the staged files replace them wholesale. Returns nil.")+ (finish-commit! [commit message]+ "Write the staged tree, commit with `message`, push, and release resources. Returns the new version.")+ (abort-commit! [commit]+ "Release the commit's resources without committing or pushing. Returns nil."))+(defprotocol Diffable"Optional capability for snapshots that can cheaply report which files changed since a prior version.Implemented by sources backed by real history (git, in-memory versioned test sources); snapshots thatdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 744db12c134..9f196d55965 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -1148,58 +1148,44 @@(defmethod query-export-roots :default [_] nil)-(defn- resolve-targets- "Expands collection IDs to include all descendant collections.- Takes a set of collection IDs and returns a set including the original IDs- plus all IDs of nested collections."- [targets opts]- (when (seq targets)- (merge-with into- (u/traverse targets #(serdes/descendants (first %) (second %) opts))- (u/traverse targets #(serdes/required (first %) (second %))))))+(defn exportable-entities+ "What a full export would serialize: a map of {model-name [id ...]} — the export roots plus their transitive+ `serdes/descendants`/`required` closure — or `{}` when there is no remote-syncable content."+ []+ (let [opts {:include-field-values false+ :include-database-secrets false+ :continue-on-error false+ :skip-archived true}+ root-targets (into []+ (mapcat query-export-roots)+ (vals (enabled-specs)))+ targets (-> #{}+ (into (keys (u/traverse root-targets #(serdes/descendants (first %) (second %) opts))))+ (into (keys (u/traverse root-targets #(serdes/required (first %) (second %))))))]+ (u/group-by first second targets)))(defn extract-entities-for-export"Extracts all entities for remote-sync export based on enabled specs.Returns a lazy sequence of serialized entities ready for storage.- Iterates over enabled specs and uses `query-export-roots` to find root targets- for each model type. Models with `:export-scope :derived` or no export-scope- are expanded from other targets via serdes/descendants.-Only extracts models that:1. Have a spec in remote-sync-specs2. Are currently enabled (based on :enabled? field)3. Are in one of the provided collections (or descendants)"[]- (let [specs (enabled-specs)- ;; Collect all root targets by iterating over enabled specs- root-targets (into []- (comp (map val)- (mapcat query-export-roots)- (filter identity))- specs)]- (when-let [targets (resolve-targets- root-targets- {:include-field-values false- :include-database-secrets false- :continue-on-error false- :skip-archived true})]- (eduction (map (fn [[model ids]]- (serdes/extract-all model {:where [:in :id ids]- :skip-archived true})))- cat- (u/group-by first second (keys targets))))))+ (eduction (map (fn [[model ids]]+ (serdes/extract-all model {:where [:in :id ids]+ :skip-archived true})))+ cat+ (exportable-entities)))(defn extract-entities-for-rows"Serializes the entities named by `rows`, grouped by model type. Each row is a map with a`:model_type` (the serdes model name) and a `:model_id` (the local DB primary key). Returns a lazysequence of serialized entities."[rows]- (let [by-model (reduce (fn [m {:keys [model_type model_id]}]- (update m model_type (fnil conj #{}) model_id))- {}- rows)]+ (let [by-model (u/group-by :model_type :model_id conj #{} rows)](eduction (map (fn [[model ids]](serdes/extract-all model {:where [:in :id ids]:skip-archived true})))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.cljnew file mode 100644index 00000000000..09c23cea9ed--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/content_hash_test.clj@@ -0,0 +1,449 @@+(ns metabase-enterprise.remote-sync.content-hash-test+ "Tests for content-hash-based dirty detection in remote sync (GHY-3933).++ The yellow dot / dirty status must track whether an entity's *serialized form* changed since the last+ sync — not merely whether an update event fired. After a sync, each RemoteSyncObject row records a+ `content_hash` of the entity's serialized YAML. When an update event arrives, the consumer re-serializes+ the entity and compares: an unchanged serialization (a no-op save, or an edit to a non-serialized field+ like a transform's schedule) must leave the row `synced`, so a subsequent push produces no empty commit.++ These tests cover every tracked model type (entity-id, hybrid, and path identities) and the+ null-baseline / real-change / revert edge cases."+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.remote-sync.content-hash-test]}}}}}}+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.models.remote-sync-object :as sync-object]+ [metabase-enterprise.remote-sync.source :as source]+ [metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase.collections.models.collection :as collection]+ [metabase.collections.test-utils :refer [with-library-synced]]+ [metabase.events.core :as events]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(defn- noop-update-status!+ "Seeds a synced RemoteSyncObject for [model-type model-id] whose `content_hash` matches the entity's+ current serialization, publishes a no-op `topic` carrying the unchanged `object`, and returns the row's+ status afterward. Asserts a baseline hash is computable for the model type (i.e. serialization works).+ `extra-payload` is merged into the event (e.g. {:previous-object …} for events whose schema requires it)."+ ([model-type model-id topic object]+ (noop-update-status! model-type model-id topic object {}))+ ([model-type model-id topic object extra-payload]+ (t2/delete! :model/RemoteSyncObject)+ (let [row {:model_type model-type :model_id model-id}+ hash (source/row->content-hash row)]+ (is (some? hash) (str "row->content-hash should be computable for " model-type))+ (t2/insert! :model/RemoteSyncObject+ (merge row {:model_name "Baseline"+ :status "synced"+ :content_hash hash+ :status_changed_at (t/offset-date-time)}))+ (events/publish-event! topic (merge {:object object :user-id (mt/user->id :rasta)} extra-payload))+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))))++;;; ------------------------------------------- entity-id models -------------------------------------------++(deftest card-noop-update-stays-synced-test+ (testing "A no-op Card update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (is (= "synced" (noop-update-status! "Card" (:id card) :event/card-update card {:previous-object card}))))))++(deftest dashboard-noop-update-stays-synced-test+ (testing "A no-op Dashboard update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Dashboard dash {:name "Dash" :collection_id (:id coll)}]+ (is (= "synced" (noop-update-status! "Dashboard" (:id dash) :event/dashboard-update dash))))))++(deftest document-noop-update-stays-synced-test+ (testing "A no-op Document update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Document doc {:collection_id (u/the-id coll)}]+ (is (= "synced" (noop-update-status! "Document" (:id doc) :event/document-update doc))))))++(deftest timeline-noop-update-stays-synced-test+ (testing "A no-op Timeline update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Timeline tl {:name "TL" :collection_id (:id coll)}]+ (is (= "synced" (noop-update-status! "Timeline" (:id tl) :event/timeline-update tl))))))++(deftest snippet-noop-update-stays-synced-test+ (testing "A no-op NativeQuerySnippet update keeps it synced when the Library is synced (GHY-3933)"+ (with-library-synced+ (mt/with-temp [:model/Collection sc {:name "Snippets" :namespace "snippets"}+ :model/NativeQuerySnippet snip {:name "Snip" :content "SELECT 1" :collection_id (:id sc)}]+ (is (= "synced" (noop-update-status! "NativeQuerySnippet" (:id snip) :event/snippet-update snip)))))))++(deftest collection-noop-update-stays-synced-test+ (testing "A no-op Collection update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}]+ (is (= "synced" (noop-update-status! "Collection" (:id coll) :event/collection-update coll))))))++;;; ------------------------------------------- path & hybrid models -------------------------------------------++(deftest table-noop-update-stays-synced-test+ (testing "A no-op Table update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Database db {:name "DB"}+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)+ :is_published true :collection_id (:id coll)}]+ (is (= "synced" (noop-update-status! "Table" (:id table) :event/table-update table))))))++(deftest field-noop-update-stays-synced-test+ (testing "A no-op Field update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Database db {:name "DB"}+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)+ :is_published true :collection_id (:id coll)}+ :model/Field field {:name "F" :table_id (:id table) :base_type :type/Integer}]+ (is (= "synced" (noop-update-status! "Field" (:id field) :event/field-update field))))))++(deftest segment-noop-update-stays-synced-test+ (testing "A no-op Segment update keeps it synced (GHY-3933)"+ (mt/with-temp [:model/Database db {:name "DB"}+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)+ :is_published true :collection_id (:id coll)}+ :model/Segment seg {:name "S" :table_id (:id table)+ :definition {:source-table (:id table) :filter [:> [:field 1 nil] 0]}}]+ (is (= "synced" (noop-update-status! "Segment" (:id seg) :event/segment-update seg))))))++(deftest measure-noop-update-stays-synced-test+ (testing "A no-op Measure update keeps it synced (GHY-3933)"+ (mt/with-model-cleanup [:model/Measure]+ (mt/with-temp [:model/Database db {:name "DB"}+ :model/Collection coll {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db)+ :is_published true :collection_id (:id coll)}+ :model/Field _ {:name "F" :table_id (:id table) :base_type :type/Integer}+ :model/Measure measure {:name "M" :table_id (:id table)}]+ (is (= "synced" (noop-update-status! "Measure" (:id measure) :event/measure-update measure)))))))++;;; ------------------------------------------- transforms models (setting-gated) -------------------------------------------++(deftest transform-noop-update-stays-synced-test+ (testing "A no-op Transform update keeps it synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:namespace collection/transforms-ns :location "/"}+ :model/Transform tr {:name "T" :collection_id (:id coll)}]+ (is (= "synced" (noop-update-status! "Transform" (:id tr) :event/transform-update tr))))))))++(deftest transform-tag-noop-update-stays-synced-test+ (testing "A no-op TransformTag update keeps it synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]+ (mt/with-temp [:model/TransformTag tag {:name "Tag"}]+ (is (= "synced" (noop-update-status! "TransformTag" (:id tag) :event/transform-tag-update tag))))))))++(deftest python-library-noop-update-stays-synced-test+ (testing "A no-op PythonLibrary update keeps it synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-transforms true remote-sync-enabled true]+ (mt/with-model-cleanup [:model/PythonLibrary]+ (let [lib (t2/insert-returning-instance! :model/PythonLibrary {:path "lib.py" :source "# x"})]+ (is (= "synced" (noop-update-status! "PythonLibrary" (:id lib) :event/python-library-update lib)))))))))++;;; ------------------------------------------- edge cases -------------------------------------------++(deftest null-baseline-update-marks-dirty-test+ (testing "An update with no recorded baseline hash still marks dirty — exports the first time (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card card {:name "C" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (t2/delete! :model/RemoteSyncObject)+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "C"+ :status "synced" :status_changed_at (t/offset-date-time)})+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})+ (is (= "update" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))+ "A null content_hash baseline must be treated as dirty"))))++(deftest changed-content-marks-dirty-test+ (testing "A real change to serialized content marks dirty (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card card {:name "Original" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (t2/delete! :model/RemoteSyncObject)+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "Original"+ :status "synced"+ :content_hash (source/row->content-hash {:model_type "Card" :model_id (:id card)})+ :status_changed_at (t/offset-date-time)})+ (t2/update! :model/Card (:id card) {:name "Renamed"})+ (events/publish-event! :event/card-update {:object (t2/select-one :model/Card :id (:id card))+ :previous-object card+ :user-id (mt/user->id :rasta)})+ (is (= "update" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))+ "A changed serialization must mark the row dirty"))))++(deftest revert-to-baseline-resyncs-test+ (testing "An update whose content matches the synced baseline clears a stale dirty flag (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card card {:name "C" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (t2/delete! :model/RemoteSyncObject)+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name "C"+ :status "update"+ :content_hash (source/row->content-hash {:model_type "Card" :model_id (:id card)})+ :status_changed_at (t/offset-date-time)})+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))+ "Content matching the baseline must clear a stale dirty flag")+ (is (not (sync-object/dirty?))))))++;;; ------------------------------------------- mark-rows-synced! -------------------------------------------++(deftest content-metadata-matches-row-hash-test+ (testing "a full export writes a content_hash matching per-row row->content-hash for every identity flavor —+ including the extract-query overrides (Collection, NativeQuerySnippet), hybrids (Measure), and path+ models (Field) that have no entity_id (GHY-3933)"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-type :read-write]+ (mt/with-temp [:model/Database db {:name "DB"}+ :model/Collection rs {:is_remote_synced true :name "RS" :location "/"}+ :model/Collection data {:is_remote_synced true :type collection/library-data-collection-type :name "Data"}+ :model/Collection snips {:name "Snippets" :namespace "snippets"}+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id rs)}+ :model/NativeQuerySnippet snip {:name "Snip" :content "SELECT 1" :collection_id (:id snips)}+ :model/Table table {:name "T" :schema "PUBLIC" :db_id (:id db) :is_published true :collection_id (:id data)}+ :model/Field field {:name "F" :table_id (:id table) :base_type :type/Integer}+ :model/Measure measure {:name "M" :table_id (:id table)}]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})+ rows (mapv (fn [[mt id]] {:model_type mt :model_id id})+ [["Card" (:id card)] ["Collection" (:id rs)] ["NativeQuerySnippet" (:id snip)]+ ["Field" (:id field)] ["Measure" (:id measure)]])]+ (t2/delete! :model/RemoteSyncObject)+ (doseq [row rows]+ (t2/insert! :model/RemoteSyncObject (merge row {:model_name "x" :status "synced"+ :status_changed_at (t/offset-date-time)})))+ ;; force? routes through full-export!, which computes + records the content_hash for every flavor+ (impl/export! (source.p/snapshot (test-helpers/create-mock-source)) task-id "Full export" :force? true)+ (doseq [{:keys [model_type model_id] :as row} rows]+ (is (= (source/row->content-hash row)+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type model_type :model_id model_id))+ (str "written hash should match per-row hash for " model_type))))))))))++(deftest mark-rows-synced!-chunks-rows-test+ (testing "mark-rows-synced! writes every synced row even when they span multiple batches (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card c1 {:name "C1" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}+ :model/Card c2 {:name "C2" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}+ :model/Card c3 {:name "C3" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (t2/delete! :model/RemoteSyncObject)+ (doseq [card [c1 c2 c3]]+ (t2/insert! :model/RemoteSyncObject {:model_type "Card" :model_id (:id card) :model_name (:name card)+ :status "create" :status_changed_at (t/offset-date-time)}))+ ;; synced entries keyed by RSO id, carrying each card's real content hash+ (let [synced (mapv (fn [r] {:id (:id r) :file_path "p"+ :content_hash (source/row->content-hash (select-keys r [:model_type :model_id]))})+ (t2/select [:model/RemoteSyncObject :id :model_type :model_id]))]+ ;; force more than one chunk for the 3 rows+ (with-redefs [impl/app-db-batch-size 2]+ (#'impl/mark-rows-synced! (t2/select-pks-set :model/RemoteSyncObject) synced (t/offset-date-time))))+ (doseq [card [c1 c2 c3]]+ (is (= (source/row->content-hash {:model_type "Card" :model_id (:id card)})+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type "Card" :model_id (:id card)))+ (str "every chunked row should get its hash: " (:name card)))))))++(deftest insert-with-metadata!-chunks-rows-test+ (testing "insert-with-metadata! inserts every row with its content_hash even across multiple chunks (GHY-3933)"+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card c1 {:name "C1" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}+ :model/Card c2 {:name "C2" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}+ :model/Card c3 {:name "C3" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (t2/delete! :model/RemoteSyncObject)+ (let [rows (mapv (fn [c] {:model_type "Card" :model_id (:id c) :model_name (:name c)+ :status "synced" :status_changed_at (t/offset-date-time)})+ [c1 c2 c3])]+ ;; force more than one chunk for the 3 cards+ (with-redefs [impl/app-db-batch-size 2]+ (#'impl/insert-with-metadata! rows [])))+ (doseq [card [c1 c2 c3]]+ (is (= (source/row->content-hash {:model_type "Card" :model_id (:id card)})+ (t2/select-one-fn :content_hash :model/RemoteSyncObject :model_type "Card" :model_id (:id card)))+ (str "every chunked row should be inserted with its hash: " (:name card)))))))++;;; ------------------------------------------- export integration -------------------------------------------+;;; These run a real export! to a mock source — which records the content_hash baseline through the actual+;;; export path (full/incremental) — then fire a no-op update and assert it stays synced. Unlike the+;;; consumer tests above (which seed the hash directly), these exercise the export-side baseline recording.++(defn- export-baseline-then-noop-status!+ "Seeds `create`-status RemoteSyncObject `seeds` (so export has something to write), runs a real export! to+ a mock source (recording content_hash baselines), then publishes a no-op event and returns the resulting+ status of [model-type model-id]. `target` is {:model-type :model-id :topic :object :payload}."+ [seeds {:keys [model-type model-id topic object payload]}]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (t2/delete! :model/RemoteSyncObject)+ (doseq [seed seeds]+ (t2/insert! :model/RemoteSyncObject (merge {:status "create" :status_changed_at (t/offset-date-time)} seed)))+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Initial export")]+ (is (= :success (:status result)) (str "export should succeed: " result))+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))+ (str model-type " should be synced after export")))+ (events/publish-event! topic (merge {:object object :user-id (mt/user->id :rasta)} payload))+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))))++(deftest card-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op Card update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}+ {:model_type "Card" :model_id (:id card) :model_name "Card" :model_collection_id (:id coll)}]+ {:model-type "Card" :model-id (:id card) :topic :event/card-update :object card+ :payload {:previous-object card}})))))))++(deftest dashboard-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op Dashboard update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Dashboard dash {:name "Dash" :collection_id (:id coll)}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}+ {:model_type "Dashboard" :model_id (:id dash) :model_name "Dash" :model_collection_id (:id coll)}]+ {:model-type "Dashboard" :model-id (:id dash) :topic :event/dashboard-update :object dash})))))))++(deftest document-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op Document update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Document doc {:collection_id (u/the-id coll)}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}+ {:model_type "Document" :model_id (:id doc) :model_name "Doc" :model_collection_id (:id coll)}]+ {:model-type "Document" :model-id (:id doc) :topic :event/document-update :object doc})))))))++(deftest timeline-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op Timeline update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS"}+ :model/Timeline tl {:name "TL" :collection_id (:id coll)}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}+ {:model_type "Timeline" :model_id (:id tl) :model_name "TL" :model_collection_id (:id coll)}]+ {:model-type "Timeline" :model-id (:id tl) :topic :event/timeline-update :object tl})))))))++(deftest collection-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op Collection update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS" :location "/"}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "Collection" :model_id (:id coll) :model_name "RS"}]+ {:model-type "Collection" :model-id (:id coll) :topic :event/collection-update :object coll})))))))++(deftest transform-tag-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op TransformTag update stays synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true remote-sync-enabled true]+ (mt/with-temp [:model/TransformTag tag {:name "Tag"}]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "TransformTag" :model_id (:id tag) :model_name "Tag"}]+ {:model-type "TransformTag" :model-id (:id tag) :topic :event/transform-tag-update :object tag}))))))))++(deftest python-library-export-then-noop-stays-synced-test+ (testing "After a real export, a no-op PythonLibrary update stays synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true remote-sync-enabled true]+ (mt/with-model-cleanup [:model/PythonLibrary]+ (let [lib (t2/insert-returning-instance! :model/PythonLibrary {:path "lib.py" :source "# x"})]+ (is (= "synced"+ (export-baseline-then-noop-status!+ [{:model_type "PythonLibrary" :model_id (:id lib) :model_name "lib.py"}]+ {:model-type "PythonLibrary" :model-id (:id lib) :topic :event/python-library-update :object lib})))))))))++(deftest full-export-records-metadata-and-stays-synced-test+ (testing "A forced full export (real store-and-record! + mark-rows-synced!) writes file_path and+ content_hash, and a subsequent no-op update stays synced (GHY-3933)"+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection coll {:is_remote_synced true :name "RS" :location "/"}+ :model/Card card {:name "Card" :dataset_query (mt/mbql-query venues) :collection_id (:id coll)}+ :model/RemoteSyncObject _ {:model_type "Collection" :model_id (:id coll) :model_name "RS" :status "create" :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _ {:model_type "Card" :model_id (:id card) :model_name "Card" :model_collection_id (:id coll) :status "create" :status_changed_at (t/offset-date-time)}+ ;; a removed/delete row whose entity left the synced set — full export must drop it+ :model/RemoteSyncObject leftover {:model_type "Card" :model_id 999999 :model_name "Gone" :status "delete" :status_changed_at (t/offset-date-time)}]+ ;; force? routes through full-export! -> store-and-record! (no incremental fast-path)+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Full export" :force? true)]+ (is (= :success (:status result)) (str "forced full export should succeed: " result))+ (let [row (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))]+ (is (= "synced" (:status row)))+ (is (some? (:content_hash row)) "content_hash recorded from store-and-record!'s serialization")+ (is (some? (:file_path row)) "file_path recorded from store-and-record!'s serialization"))+ (is (nil? (t2/select-one :model/RemoteSyncObject :id (:id leftover)))+ "removed/delete row dropped by full export (as the incremental path does)"))+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id (mt/user->id :rasta)})+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))))+ "a no-op update after a full export stays synced")))))))++;;; ------------------------------------------- import integration -------------------------------------------+;;; These import a snapshot from a mock source — which records the content_hash baseline through the actual+;;; import path (`insert-with-metadata!`) — then fire a no-op update and assert it stays synced. This is the+;;; "right after a pull" scenario from GHY-3933.++(defn- import-then-noop-status!+ "Imports `files` from a mock source (recording content_hash baselines via the import path), looks up the+ imported entity with `lookup` (a 0-arg thunk returning the instance), publishes a no-op `topic`, and+ returns the resulting RemoteSyncObject status. `payload-fn` (optional) returns extra event payload."+ [files model-type lookup topic & {:keys [payload-fn]}]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ ms (test-helpers/create-mock-source :initial-files files)+ result (impl/import! (source.p/snapshot ms) task-id :force? true :force-deletion? true)]+ (is (= :success (:status result)) (str "import should succeed: " result))+ (let [entity (lookup)+ model-id (:id entity)]+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)))+ (str model-type " should be synced after import"))+ (events/publish-event! topic (merge {:object entity :user-id (mt/user->id :rasta)}+ (when payload-fn (payload-fn entity))))+ (:status (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id))))))++(deftest transform-import-then-noop-stays-synced-test+ (testing "After a real import, a no-op Transform update stays synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-model-cleanup [:model/Transform :model/Collection]+ ;; the transform YAML references the "test-data (h2)" database by name; force it to load so the+ ;; import resolves the reference regardless of test order.+ (mt/db)+ (let [coll-eid "transforms-coll-xxxxx"+ tr-eid "test-transform-xxxxxx"+ files {"main" {"collections/transforms/transforms/transforms.yaml"+ (test-helpers/generate-collection-yaml coll-eid "Transforms" :namespace "transforms")+ "collections/transforms/transforms/test_transform.yaml"+ (test-helpers/generate-transform-yaml tr-eid "Test Transform" :collection-id coll-eid)}}]+ (is (= "synced"+ (import-then-noop-status!+ files "Transform"+ #(t2/select-one :model/Transform :entity_id tr-eid)+ :event/transform-update)))))))))++(deftest transform-tag-import-then-noop-stays-synced-test+ (testing "After a real import, a no-op TransformTag update stays synced (GHY-3933)"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-model-cleanup [:model/TransformTag]+ (let [tag-eid "test-transform-tag-xx"+ files {"main" {"transforms/transform_tags/test_tag.yaml"+ (test-helpers/generate-transform-tag-yaml tag-eid "Test Tag")}}]+ (is (= "synced"+ (import-then-noop-status!+ files "TransformTag"+ #(t2/select-one :model/TransformTag :entity_id tag-eid)+ :event/transform-tag-update)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.cljindex 608404e29c9..8b3fb93d230 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj@@ -7,7 +7,7 @@[clojure.test :refer :all][java-time.api :as t][metabase-enterprise.remote-sync.events :as remote-sync.events]- [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.spec :as spec][metabase.collections.models.collection :as collection][metabase.collections.test-utils :refer [with-library-synced]][metabase.events.core :as events]@@ -18,6 +18,14 @@(use-fixtures :once (fixtures/initialize :db))+(defn- mark-synced!+ "Populate the RemoteSyncObject table with synced rows for `imported-data` (a `:by-entity-id` map), as a+ successful import does — used to set up a synced baseline before firing an event."+ [imported-data]+ (t2/delete! :model/RemoteSyncObject)+ (when-let [rows (seq (spec/sync-all-entities! (t/instant) imported-data))]+ (t2/insert! :model/RemoteSyncObject rows)))+;;; Model Change Event Tests(deftest card-create-event-creates-entry-test@@ -514,7 +522,7 @@:model/Card card {:name "Test Card":dataset_query (mt/mbql-query venues):collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Card" #{(:entity_id card)}}})+ (mark-synced! {:by-entity-id {"Card" #{(:entity_id card)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))](is (= "synced" (:status initial-entry)))(events/publish-event! :event/card-update@@ -550,7 +558,7 @@:model/Collection normal-collection {:name "Normal"}:model/Dashboard dashboard {:name "Test Dashboard":collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})+ (mark-synced! {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id dashboard))](is (= "synced" (:status initial-entry)))(events/publish-event! :event/dashboard-update@@ -582,7 +590,7 @@(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}:model/Collection normal-collection {:name "Normal"}:model/Document document {:collection_id (u/the-id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Document" #{(:entity_id document)}}})+ (mark-synced! {:by-entity-id {"Document" #{(:entity_id document)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id document))](is (= "synced" (:status initial-entry)))(events/publish-event! :event/document-update@@ -611,7 +619,7 @@(deftest existing-collection-type-changed-from-remote-synced-test(testing "existing collection type changed from remote-synced is marked as removed"(mt/with-temp [:model/Collection collection {:is_remote_synced true :name "Remote-Sync"}]- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Collection" #{(:entity_id collection)}}})+ (mark-synced! {:by-entity-id {"Collection" #{(:entity_id collection)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id collection))](is (= "synced" (:status initial-entry)))(events/publish-event! :event/collection-update@@ -788,7 +796,7 @@:model/Collection normal-collection {:name "Normal Collection"}:model/Timeline timeline {:name "Test Timeline":collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})+ (mark-synced! {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})(is (= 1 (count (t2/select :model/RemoteSyncObject))))(events/publish-event! :event/timeline-update{:object (assoc timeline :collection_id (:id normal-collection))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex af379af4586..330fa079b52 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -189,13 +189,13 @@;; export! tests-(deftest export!-with-no-source-configured-test- (testing "export! with no source configured throws"- (mt/with-temporary-setting-values [remote-sync-type :read-write]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (is (thrown-with-msg? clojure.lang.ExceptionInfo- #"Remote sync source is not enabled"- (impl/export! nil task-id "Test commit")))))))+(deftest async-export!-with-no-source-configured-test+ (testing "async-export! throws at the API call (not inside the async task) when remote sync isn't configured"+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-url nil]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Remote sync source is not enabled"+ (impl/async-export! "main" false "Test commit"))))))(deftest export!-with-no-remote-synced-collections-test(testing "export! is a no-op success when nothing is dirty (no remote-synced content to export)"@@ -1672,8 +1672,13 @@ serdes/meta:(version [_] version)(list-files [_] [])(read-file [_ _] nil)- (write-files! [_ _ _] version)- (apply-changes! [_ _ _ _] version)))+ (open-commit [_]+ (reify source.p/CommitBuilder+ (stage-upsert! [_ _] nil)+ (stage-delete! [_ _] nil)+ (replace-all! [_] nil)+ (finish-commit! [_ _] "written-version")+ (abort-commit! [_] nil)))))(defn- export-test-source"A minimal Source whose snapshot-at returns a snapshot at the requested version."@@ -1770,18 +1775,16 @@ serdes/meta:(deftest export!-refuses-when-diverged-without-merge-flag-test(testing "when the remote advanced and neither force? nor merge? is set, export! refuses without writing or merging"(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]- (let [merged? (atom false)- stored? (atom false)]+ (let [merged? (atom false)](with-redefs [remote-sync.task/last-version (constantly "base-B")spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})- source/store! (fn [_ _ _ _] (reset! stored? true) "v")]+ source/merge-and-store! (fn [_ _ _ _ _] (reset! merged? true) {:status :success})](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":source (export-test-source):base-snapshot (export-test-snapshot "base-B"))](is (= :conflict (:status result)))(is (false? @merged?) "no merge without the merge flag")- (is (false? @stored?) "nothing written")+ ;; :conflict short-circuits before any write — the version is never advanced(is (nil? (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))(deftest export!-force-overwrites-without-merging-test@@ -1789,17 +1792,16 @@ serdes/meta:(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}](let [merged? (atom false)](with-redefs [remote-sync.task/last-version (constantly "base-B")- spec/extract-entities-for-export (constantly [{:dummy true}])- source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})- ;; force? routes through full-export!, which uses store!'s {:version :entries} result.- source/store! (fn [& _] {:version "forced-version" :entries []})]+ spec/exportable-entities (constantly {"Card" [999999999]}) ; absent id -> empty extraction; routing is what's under test+ source/merge-and-store! (fn [& _] (reset! merged? true) {:status :success})](let [result (impl/export! (export-test-snapshot "remote-R") task-id "msg":force? true:source (export-test-source):base-snapshot (export-test-snapshot "base-B"))](is (= :success (:status result)))(is (false? @merged?) "force? skips the merge path")- (is (= "forced-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))+ ;; force? routes through full-export!, which commits via the snapshot and advances the version+ (is (= "written-version" (:version (t2/select-one :model/RemoteSyncTask :id task-id))))))))))(deftest export!-no-merge-when-not-diverged-test(testing "when the remote has not advanced, export! takes the normal (non-merge) export path"@@ -1885,7 +1887,7 @@ serdes/meta:{:merged [{:path "collections/x.yaml" :content "y"}]:conflicts []:summary {:added 1 :updated 0 :removed 0}})- ;; simulate the load marking everything synced (what sync-objects! does), then running+ ;; simulate the load marking everything synced (what the import insert does), then running;; the in-transaction finalize (restore-dirty + set-version)impl/load-snapshot! (fn [_ _ _ & {:keys [finalize!]}](t2/update! :model/RemoteSyncObjectdiff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.cljindex 8ade5adf547..edab7ea9fcb 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/incremental_import_test.clj@@ -96,7 +96,7 @@path (atom :fallback)](with-redefs [impl/incremental-load-snapshot!(fn [& args] (let [r (apply real args)]- (reset! path (if (= r @#'impl/incremental-not-possible) :fallback :incremental))+ (reset! path (if (= r :remote-sync/incremental-not-possible) :fallback :incremental))r))][(import-at! src "v1" :force? false) @path])))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.cljindex 8c7c0c09b8d..c523d51b4d8 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj@@ -134,8 +134,7 @@:model/NativeQuerySnippet snippet {:name "Existing Snippet" :content "SELECT 1" :collection_id coll-id}](is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))"Should have no snippet tracking entries initially")- ;; Enable snippet sync- (rs-events/sync-snippet-tracking! true)+ (rs-events/enable-snippet-tracking!);; Verify tracking entries created(is (t2/exists? :model/RemoteSyncObject:model_type "Collection"@@ -168,8 +167,7 @@(is (= 2 (t2/count :model/RemoteSyncObject :model_type [:in ["Collection" "NativeQuerySnippet"]]:model_id [:in [coll-id (:id snippet)]]))"Should have 2 tracking entries")- ;; Disable snippet sync- (rs-events/sync-snippet-tracking! false)+ (rs-events/disable-snippet-tracking!);; Verify tracking entries removed(is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))"Snippet tracking entries should be removed")@@ -417,6 +415,6 @@ is_sample: false(let [files-after-export (get @(:files-atom mock-source) "main")](is (not (some #(str/includes? % "archived_snippet") (keys files-after-export)))"Archived snippet file should be deleted after export")))- (testing "RemoteSyncObject entry is updated to synced after export"- (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id)))- "RemoteSyncObject entry for archived snippet should have synced status")))))))))+ (testing "the archived snippet's tracking row is dropped after export (it left the synced set)"+ (is (nil? (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id))+ "RemoteSyncObject entry for the archived snippet should be deleted")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 69cc8774109..156c801ee34 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -16,6 +16,24 @@(set! *warn-on-reflection* true)+(defn- write-files!+ "Test seeding helper: wholesale-write `files` ({:path :content}) to `snapshot` via the commit builder+ (clear the managed dirs, stage every file, push). Returns the new version."+ [snapshot message files]+ (let [c (source.p/open-commit snapshot)]+ (source.p/replace-all! c)+ (doseq [f files] (source.p/stage-upsert! c f))+ (source.p/finish-commit! c message)))++(defn- apply-changes!+ "Test helper: incremental patch via the commit builder — stage `upserts`, remove `delete-paths`, push.+ Returns the new version."+ [snapshot message upserts delete-paths]+ (let [c (source.p/open-commit snapshot)]+ (doseq [u upserts] (source.p/stage-upsert! c u))+ (doseq [p delete-paths] (source.p/stage-delete! c p))+ (source.p/finish-commit! c message)))+(defn- git-working-branch"The working branch in the given repo"[{:keys [^Git git]}]@@ -210,11 +228,11 @@(str thirddir-path "path2.txt") "File 2 in third dir"}:branches ["branch-1" "branch-2"])](testing "All files in managed dirs not in write set are removed; root files outside managed dirs are preserved"- (source.p/write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}- {:path (str subdir-path "path.txt") :content "Updated subdir content"}- {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}- {:path (str thirddir-path "path.txt") :content "Updated third dir content"}- {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])+ (write-files! (source.p/snapshot master) "Update 1" [{:path "master.txt" :content "Updated master content"}+ {:path (str subdir-path "path.txt") :content "Updated subdir content"}+ {:path (str subdir-path "path3.txt") :content "Updated subdir content 3"}+ {:path (str thirddir-path "path.txt") :content "Updated third dir content"}+ {:path (str thirddir-path "path3.txt") :content "Updated third dir content 3"}])(is (= ["Update 1" "Initial commit"] (map :message (git/log master))))(let [master-snap (source.p/snapshot master)];; otherdir files are removed because collections/ is a managed dir and those files weren't in the write set@@ -240,7 +258,7 @@(git/list-files (assoc remote :version "master"))))(is (= ["Update 1" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))))(testing "Writing only to collections/ removes all other collection files"- (source.p/write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])+ (write-files! (source.p/snapshot master) "Update 2" [{:path (str thirddir-path "path.txt") :content "Only third dir content"}])(is (= [(str thirddir-path "path.txt")"master.txt""master2.txt"]@@ -255,10 +273,10 @@(str subdir "edit.yaml") "old content"(str subdir "remove.yaml") "delete me"})](testing "apply-changes! overwrites/adds upserts, removes delete-paths, and PRESERVES every other file"- (source.p/apply-changes! (source.p/snapshot master) "Incremental"- [{:path (str subdir "edit.yaml") :content "new content"}- {:path (str subdir "new.yaml") :content "brand new"}]- [(str subdir "remove.yaml")])+ (apply-changes! (source.p/snapshot master) "Incremental"+ [{:path (str subdir "edit.yaml") :content "new content"}+ {:path (str subdir "new.yaml") :content "brand new"}]+ [(str subdir "remove.yaml")])(is (= ["Incremental" "Initial commit"] (map :message (git/log master))))(let [snap (source.p/snapshot master)](is (= [(str subdir "edit.yaml")@@ -285,9 +303,9 @@"collections/a/deep/sibling.yaml" "deep sibling""collections/b/edit.yaml" "old""notes.txt" "root note"})]- (source.p/apply-changes! (source.p/snapshot master) "Incremental deep"- [{:path "collections/b/edit.yaml" :content "new"}]- [])+ (apply-changes! (source.p/snapshot master) "Incremental deep"+ [{:path "collections/b/edit.yaml" :content "new"}]+ [])(let [snap (source.p/snapshot master)](is (= "new" (source.p/read-file snap "collections/b/edit.yaml")))(is (= "deep keep" (source.p/read-file snap "collections/a/deep/nested/keep.yaml"))@@ -303,8 +321,8 @@"transforms/t1/t.yaml" "a transform""transforms/t2/t.yaml" "another transform""notes.txt" "root note"})]- (source.p/write-files! (source.p/snapshot master) "Full"- [{:path "collections/new/new.yaml" :content "new collection"}])+ (write-files! (source.p/snapshot master) "Full"+ [{:path "collections/new/new.yaml" :content "new collection"}])(let [snap (source.p/snapshot master)](is (= ["collections/new/new.yaml" "notes.txt"] (source.p/list-files snap))"transforms/ (managed, no upserts) fully removed; collections/ reconciled to the write set; non-managed notes.txt preserved")@@ -318,9 +336,9 @@(mt/with-temp-dir [remote-dir nil](let [[master _remote] (init-source! "master" remote-dir:files {"collections/a/keep.yaml" "keep"})]- (source.p/apply-changes! (source.p/snapshot master) "Delete missing + add"- [{:path "collections/a/new.yaml" :content "new"}]- ["collections/a/gone.yaml"])+ (apply-changes! (source.p/snapshot master) "Delete missing + add"+ [{:path "collections/a/new.yaml" :content "new"}]+ ["collections/a/gone.yaml"])(let [snap (source.p/snapshot master)](is (= ["collections/a/keep.yaml" "collections/a/new.yaml"] (source.p/list-files snap)))(is (= "new" (source.p/read-file snap "collections/a/new.yaml")))@@ -335,10 +353,10 @@"remove.txt" "delete me""deep/nested/keep.txt" "deep unchanged"})from-version (:version (source.p/snapshot master))]- (source.p/apply-changes! (source.p/snapshot master) "Change set"- [{:path "edit.txt" :content "new content"}- {:path "add.txt" :content "brand new"}]- ["remove.txt"])+ (apply-changes! (source.p/snapshot master) "Change set"+ [{:path "edit.txt" :content "new content"}+ {:path "add.txt" :content "brand new"}]+ ["remove.txt"])(let [snap (source.p/snapshot master)](testing "added / modified / deleted are reported in their own buckets"(is (= {:added #{"add.txt"}@@ -386,7 +404,7 @@;; Make source be behind again(git-working-add! remote "only-on-remote.txt" "Initially on remote")(git-working-commit! remote "Only on remote")- (source.p/write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])+ (write-files! (source.p/snapshot master) "Added to source" [{:path "initially-source.txt" :content "Initially on source"}])(testing "Remote has the new commit with just the files committed, but only version is in history"(is (= ["Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "master")))))(is (= ["additional-file.txt" "initially-source.txt" "master.txt" "only-on-remote.txt" "subdir/path.txt"] (git/list-files (assoc remote :version "master"))))@@ -400,8 +418,8 @@(git-working-commit! remote "New-branch on remote")(is (= ["New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch")))))(is (nil? (git/log new-branch)))- (source.p/write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}- {:path "new-branch-file.txt" :content "Updated on source"}])+ (write-files! (source.p/snapshot new-branch) "New-branch on source" [{:path "new-branch-source.txt" :content "Initially on source"}+ {:path "new-branch-file.txt" :content "Updated on source"}])(is (= ["New-branch on source" "New-branch on remote" "Added to source" "Only on remote" "Added additional file" "Initial commit"] (map :message (git/log (assoc remote :branch "new-branch"))))))))))(deftest git-source-using-commit-ref@@ -410,8 +428,8 @@:files {"master.txt" "File in master""subdir/path.txt" "File in subdir"})old-master (source.p/snapshot master)]- (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}- {:path "new-file.txt" :content "New file in master"}])+ (write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated file in master"}+ {:path "new-file.txt" :content "New file in master"}])(is (= "File in master" (source.p/read-file old-master "master.txt")))(is (= "Updated file in master" (source.p/read-file (source.p/snapshot master) "master.txt")))(is (= ["master.txt" "subdir/path.txt"] (source.p/list-files old-master))))))@@ -427,7 +445,7 @@(is (= (git/commit-sha master "master") initial-version)"version should match the commit id for the branch")(testing "version changes after writing files"- (source.p/write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])+ (write-files! (source.p/snapshot master) "Update file" [{:path "master.txt" :content "Updated content"}])(let [new-version (source.p/version (source.p/snapshot master))](is (not= initial-version new-version) "version should change after commit")(is (= 40 (count new-version)) "new version should also be a full SHA-1 hash")@@ -469,13 +487,13 @@(str old-col-path "cards/card2.yaml") "Card 2 in old col"(str kept-col-path "dashboards/dash1.yaml") "Dashboard in kept col"})](testing "Writing to a managed dir removes all stale files in ALL managed dirs"- (source.p/write-files! (source.p/snapshot master) "Rename database"- [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}- {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}- {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}- {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}- {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}- {:path "snippets/old_snippet.yaml" :content "Old snippet"}])+ (write-files! (source.p/snapshot master) "Rename database"+ [{:path "databases/new_db/new_db.yaml" :content "Renamed database"}+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}+ {:path (str old-col-path "cards/card1.yaml") :content "Card in old col"}+ {:path (str old-col-path "cards/card2.yaml") :content "Card 2 in old col"}+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard in kept col"}+ {:path "snippets/old_snippet.yaml" :content "Old snippet"}])(let [files (set (source.p/list-files (source.p/snapshot master)))](is (contains? files "databases/new_db/new_db.yaml") "New database file should exist")(is (contains? files "databases/new_db/schemas/public.yaml") "New schema file should exist")@@ -485,11 +503,11 @@(is (contains? files "snippets/old_snippet.yaml") "Written snippet file should remain")(is (contains? files "unmanaged/keep_me.txt") "Unmanaged files should be untouched")))(testing "Entity moved between collections removes files from old collection"- (source.p/write-files! (source.p/snapshot master) "Move card to new collection"- [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}- {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}- {:path "databases/new_db/new_db.yaml" :content "Renamed database"}- {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])+ (write-files! (source.p/snapshot master) "Move card to new collection"+ [{:path (str new-col-path "cards/card1.yaml") :content "Card moved to new col"}+ {:path (str kept-col-path "dashboards/dash1.yaml") :content "Dashboard still here"}+ {:path "databases/new_db/new_db.yaml" :content "Renamed database"}+ {:path "databases/new_db/schemas/public.yaml" :content "Same schema"}])(let [files (set (source.p/list-files (source.p/snapshot master)))](is (contains? files (str new-col-path "cards/card1.yaml")) "Moved card should exist in new collection")(is (contains? files (str kept-col-path "dashboards/dash1.yaml")) "Kept collection files should remain")@@ -505,9 +523,9 @@:files {(str col-path "cards/eid123_OldCardName.yaml") "Card with old name"(str col-path "cards/eid456_OtherCard.yaml") "Other card"})](testing "Entity renamed within a collection removes the old-named file"- (source.p/write-files! (source.p/snapshot master) "Rename card"- [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}- {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])+ (write-files! (source.p/snapshot master) "Rename card"+ [{:path (str col-path "cards/eid123_NewCardName.yaml") :content "Card with new name"}+ {:path (str col-path "cards/eid456_OtherCard.yaml") :content "Other card"}])(let [files (set (source.p/list-files (source.p/snapshot master)))](is (contains? files (str col-path "cards/eid123_NewCardName.yaml")) "Renamed card should exist")(is (contains? files (str col-path "cards/eid456_OtherCard.yaml")) "Other card should still exist")diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.cljindex 281b6fb3a6a..31895f28fe7 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/wrapping_source_test.clj@@ -12,12 +12,8 @@(read-file [_ path](get files path))- (write-files! [_ _message new-files]- (into {} (map (juxt :path :content) new-files))- "written-files-version")-- (apply-changes! [_ _message _upserts _delete-paths]- "written-files-version")+ (open-commit [_]+ (throw (UnsupportedOperationException. "open-commit not implemented in this test mock")))(version [_]"mock-version"))@@ -102,9 +98,6 @@(deftest wrapping-source-is-read-only-test(testing "WrappingSnapshot is a read-only ingestion view: write operations throw rather than write"(let [mock-source (->MockSource {"collections/foo.yaml" "foo-content"})- wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])- file-spec {:path "collections/foo.yaml" :content "foo-content"}]- (is (thrown? UnsupportedOperationException- (source.p/write-files! wrapped-snap "test commit" [file-spec])))+ wrapped-snap (source/->WrappingSnapshot (source.p/snapshot mock-source) [#"collections/.*"])](is (thrown? UnsupportedOperationException- (source.p/apply-changes! wrapped-snap "test commit" [file-spec] []))))))+ (source.p/open-commit wrapped-snap))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.cljindex 20a0eb4955c..e3d19543dcf 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source_test.clj@@ -1,55 +1,16 @@(ns metabase-enterprise.remote-sync.source-test(:require- [clojure.string :as str][clojure.test :refer :all][metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as th][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [toucan2.core :as t2]))+ [metabase.test.fixtures :as fixtures]))(use-fixtures :once (fixtures/initialize :db))(use-fixtures :each th/clean-remote-sync-state)-(defrecord MockSourceSnapshot [written-files]- source.p/SourceSnapshot- (list-files [_]- [])-- (read-file [_ _path]- nil)-- (write-files! [_ message files]- (let [realized-files (vec files)]- (reset! written-files {:message message :files realized-files}))- "mock-written-version")-- (apply-changes! [_ message upserts _delete-paths]- (reset! written-files {:message message :files (vec upserts)})- "mock-written-version")-- (version [_]- "mock-version"))--(defrecord MockSource [written-files]- source.p/Source- (create-branch [_ _branch _base]- nil)-- (branches [_]- ["main"])-- (default-branch [_]- "main")-- (snapshot [_]- (->MockSourceSnapshot written-files))-- (snapshot-at [_ _version]- (->MockSourceSnapshot written-files)))-(defn- create-test-entity"Creates a test entity with serdes/meta for testing"[id label model-name]@@ -71,135 +32,18 @@(reify source.p/SourceSnapshot(list-files [_] (vec (keys by-path)))(read-file [_ p] (get by-path p))- (write-files! [_ message files]- (reset! written {:message message :files (vec files)})- "merged-version")- (apply-changes! [_ message upserts _delete-paths]- (reset! written {:message message :files (vec upserts)})- "merged-version")+ (open-commit [_]+ (let [staged (atom [])]+ (reify source.p/CommitBuilder+ (stage-upsert! [_ spec] (swap! staged conj spec) nil)+ (stage-delete! [_ _path] nil)+ (replace-all! [_] nil)+ (finish-commit! [_ message]+ (reset! written {:message message :files @staged})+ "merged-version")+ (abort-commit! [_] nil))))(version [_] "remote-tip"))))-(deftest store!-basic-test- (testing "store! writes files to source with correct structure"- (mt/with-temp [:model/User user {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password123"}- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"- :initiated_by (:id user)}]- (let [written-files (atom nil)- mock-source (->MockSource written-files)- test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")- (create-test-entity "test-id-2" "entity-two" "Card")]]- (is (= "mock-written-version" (:version (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit message"))))- (testing "write-files! was called with correct message"- (is (= "Test commit message" (:message @written-files))))- (testing "write-files! was called with correct number of files"- (is (= 2 (count (:files @written-files)))))- (testing "each file has path and content"- (doseq [file (:files @written-files)]- (is (contains? file :path) "File should have :path")- (is (contains? file :content) "File should have :content")- (is (string? (:path file)) "Path should be a string")- (is (string? (:content file)) "Content should be a string")))- (testing "file paths end with .yaml"- (doseq [file (:files @written-files)]- (is (str/ends-with? (:path file) ".yaml")- "File paths should end with .yaml")))- (testing "file content is valid YAML containing entity data"- (doseq [file (:files @written-files)]- (is (str/includes? (:content file) "serdes/meta")- "Content should include serdes/meta")- (is (or (str/includes? (:content file) "test-id-1")- (str/includes? (:content file) "test-id-2"))- "Content should include entity ID")))))))--(deftest store!-progress-tracking-test- (testing "store! updates task progress as files are written"- (mt/with-temp [:model/User user {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password123"}- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"- :initiated_by (:id user)}]- (let [written-files (atom nil)- mock-source (->MockSource written-files)- test-entities [(create-test-entity "test-id-1" "entity-one" "Collection")- (create-test-entity "test-id-2" "entity-two" "Card")- (create-test-entity "test-id-3" "entity-three" "Dashboard")]]- (let [initial-task (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (nil? (:progress initial-task)) "Progress should be nil initially"))- (source/store! test-entities (source.p/snapshot mock-source) task-id "Test commit")- (let [final-task (t2/select-one :model/RemoteSyncTask :id task-id)]- (is (some? (:progress final-task)) "Progress should be updated after store!")- (is (> (:progress final-task) 0.3) "Progress should be greater than 0.3")- (is (<= (:progress final-task) 0.95) "Progress should be at most 0.95")- (is (some? (:last_progress_report_at final-task))- "last_progress_report_at should be set"))))))--(deftest store!-empty-stream-test- (testing "store! handles empty stream correctly"- (mt/with-temp [:model/User user {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password123"}- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"- :initiated_by (:id user)}]- (let [written-files (atom nil)- mock-source (->MockSource written-files)]- (source/store! [] (source.p/snapshot mock-source) task-id "Empty commit")- (testing "write-files! was called even with empty stream"- (is (some? @written-files)))- (testing "files list is empty"- (is (empty? (:files @written-files))))))))--(deftest store!-deterministic-yaml-test- (testing "store! produces deterministic YAML output with consistent key ordering"- (mt/with-temp [:model/User user {:first_name "Test"- :last_name "User"- :email "[email protected]"- :password "password123"}- :model/RemoteSyncTask {task-id :id} {:sync_task_type "export"- :initiated_by (:id user)}]- (let [written-files-1 (atom nil)- written-files-2 (atom nil)- mock-source-1 (->MockSource written-files-1)- mock-source-2 (->MockSource written-files-2)- ;; Create entity with keys in non-alphabetical order to test sorting- test-entity {:serdes/meta [{:id "test-id" :label "test-entity" :model "Card"}]- :z_last_field "should be last (unknown)"- :a_first_field "should be after known fields (unknown)"- :name "Test Card"- :description "A test card"- :entity_id "test-id"- :display "table"- :dataset_query {:z_field 1 :a_field 2 :database 3}- :visualization_settings {:zebra true :apple false}}]- ;; Store the same entity twice- (source/store! [test-entity] (source.p/snapshot mock-source-1) task-id "First commit")- (source/store! [test-entity] (source.p/snapshot mock-source-2) task-id "Second commit")- (testing "YAML content is identical between runs"- (let [content-1 (-> @written-files-1 :files first :content)- content-2 (-> @written-files-2 :files first :content)]- (is (= content-1 content-2)- "YAML content should be identical for the same entity")))- (testing "known keys appear in serialization-order.edn order"- (let [content (-> @written-files-1 :files first :content)- name-pos (str/index-of content "name:")- description-pos (str/index-of content "description:")- display-pos (str/index-of content "display:")]- ;; Per serialization-order.edn, Card order is: name, description, entity_id, ... display- (is (< name-pos description-pos)- "name should appear before description")- (is (< description-pos display-pos)- "description should appear before display")))- (testing "unknown keys are sorted alphabetically after known keys"- (let [content (-> @written-files-1 :files first :content)- a-first-pos (str/index-of content "a_first_field:")- z-last-pos (str/index-of content "z_last_field:")]- (is (< a-first-pos z-last-pos)- "a_first_field should appear before z_last_field (alphabetical)")))))))-(deftest merge-and-store!-clean-merge-test(testing "when local and remote changed different entities, merge-and-store! writes the union with no conflict"(mt/with-temp [:model/RemoteSyncTask {task-id :id} {:sync_task_type "export"}]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 3afbf5b947e..57358755ae0 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -194,39 +194,32 @@ width: fixed;; git read-file contract).(get-in @files-atom [branch path])))- (write-files! [_this _message files]- (case fail-mode- :write-files-error (throw (Exception. "Failed to write files"))- :store-error (throw (Exception. "Store failed"))- :network-error (throw (java.net.UnknownHostException. "Remote host not found"))- ;; Default success case - remove all files in managed dirs, then add new files- (let [write-entries (remove #(str/blank? (:path %)) files)- write-paths (into #{} (map :path) write-entries)- current-files (get @files-atom branch {})- ;; Keep files outside managed dirs or in the write set- kept-files (into {}- (filter (fn [[path _]]- (or (not (contains? managed-dirs (top-level-dir path)))- (contains? write-paths path))))- current-files)- ;; Add new files- final-files (into kept-files (map (juxt :path :content) write-entries))]- (swap! files-atom assoc branch final-files)))- "write-files-version")-- (apply-changes! [_this _message upserts delete-paths]- (case fail-mode- :apply-changes-error (throw (Exception. "Failed to apply changes"))- :network-error (throw (java.net.UnknownHostException. "Remote host not found"))- ;; Default: overwrite/add upserts, remove delete-paths, preserve every other file.- (let [write-entries (remove #(str/blank? (:path %)) upserts)- delete-set (into #{} (remove str/blank?) delete-paths)]- (swap! files-atom update branch- (fn [current]- (as-> (or current {}) files- (apply dissoc files delete-set)- (into files (map (juxt :path :content)) write-entries))))))- "apply-changes-version")+ (open-commit [_this]+ (let [upserts (atom [])+ deletes (atom #{})+ replace-all? (atom false)]+ (reify source.p/CommitBuilder+ (stage-upsert! [_ file-spec] (swap! upserts conj file-spec) nil)+ (stage-delete! [_ path] (swap! deletes conj path) nil)+ (replace-all! [_] (reset! replace-all? true) nil)+ (finish-commit! [_ _message]+ (case fail-mode+ :write-files-error (throw (Exception. "Failed to write files"))+ :store-error (throw (Exception. "Store failed"))+ :apply-changes-error (throw (Exception. "Failed to apply changes"))+ :network-error (throw (java.net.UnknownHostException. "Remote host not found"))+ (let [managed (if @replace-all? (set managed-dirs) #{})]+ (swap! files-atom update branch+ (fn [current]+ (as-> (or current {}) files+ (into {} (remove (fn [[p _]] (or (managed (top-level-dir p))+ (contains? @deletes p))))+ files)+ (into files (comp (remove #(str/blank? (:path %))) (map (juxt :path :content)))+ @upserts))))))+ ;; version string discriminates a wholesale replace (replace-all!) from an incremental patch+ (if @replace-all? "write-files-version" "apply-changes-version"))+ (abort-commit! [_] nil))))(version [_this]"mock-version"))@@ -288,8 +281,8 @@ width: fixedUnlike [[create-mock-source]] (which always reports \"mock-version\"), this lets a test controlversions: `:current` is the version `snapshot` reports, and `:trees` maps version -> {path content}.`snapshot-at` returns a snapshot for a version present in `trees`, or nil for an unknown version (to- model a base orphaned by a force-push/rebase). `write-files!` records the written set under a fresh- version, advances `:current`, and returns the new version so an export can fast-forward onto it."+ model a base orphaned by a force-push/rebase). Committing (via `open-commit`) records the written set+ under a fresh version, advances `:current`, and returns the new version so an export can fast-forward onto it."[& {:keys [current trees branch managed-dirs]:or {current "v-remote" branch "main" managed-dirs ingest/legal-top-level-paths}}](let [managed (set managed-dirs)@@ -312,30 +305,27 @@ width: fixedsource.p/SourceSnapshot(list-files [_] (vec (keys (get-in @state [:trees version] {}))))(read-file [_ path] (get-in @state [:trees version path]))- (write-files! [_ _message files]- (let [n (:counter (swap! state update :counter inc))- new-version (str "written-" n)- kept (into {}- (remove (fn [[p _]] (contains? managed (top-level-dir p))))- (get-in @state [:trees version] {}))- tree (into kept- (comp (remove #(str/blank? (:path %)))- (map (juxt :path :content)))- files)]- (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))- new-version))- (apply-changes! [_ _message upserts delete-paths]- (let [n (:counter (swap! state update :counter inc))- new-version (str "written-" n)- delete-set (into #{} (remove str/blank?) delete-paths)- tree (as-> (get-in @state [:trees version] {}) t- (apply dissoc t delete-set)- (into t- (comp (remove #(str/blank? (:path %)))- (map (juxt :path :content)))- upserts))]- (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))- new-version))+ (open-commit [_]+ (let [staged-upserts (atom [])+ staged-deletes (atom #{})+ replace-all? (atom false)]+ (reify source.p/CommitBuilder+ (stage-upsert! [_ file-spec] (swap! staged-upserts conj file-spec) nil)+ (stage-delete! [_ path] (swap! staged-deletes conj path) nil)+ (replace-all! [_] (reset! replace-all? true) nil)+ (finish-commit! [_ _message]+ (let [replace-dirs (if @replace-all? managed #{}) ; `managed` = (set managed-dirs)+ n (:counter (swap! state update :counter inc))+ new-version (str "written-" n)+ tree (as-> (get-in @state [:trees version] {}) t+ (into {} (remove (fn [[p _]] (or (replace-dirs (top-level-dir p))+ (contains? @staged-deletes p)))) t)+ (into t (comp (remove #(str/blank? (:path %)))+ (map (juxt :path :content)))+ @staged-upserts))]+ (swap! state #(-> % (assoc-in [:trees new-version] tree) (assoc :current new-version)))+ new-version))+ (abort-commit! [_] nil))))(version [_] version)))](reify source.p/Source(branches [_] [branch])diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.cljindex 6a279468330..08f4be77ec2 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers_test.clj@@ -5,6 +5,15 @@[metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as th]))+(defn- write-files!+ "Wholesale-write `files` ({:path :content}) to `snapshot` via the commit builder (clear managed dirs,+ stage every file, push)."+ [snapshot message files]+ (let [c (source.p/open-commit snapshot)]+ (source.p/replace-all! c)+ (doseq [f files] (source.p/stage-upsert! c f))+ (source.p/finish-commit! c message)))+(deftest mock-source-write-files-managed-dir-cleanup-test(testing "MockSource removes files in managed dirs not in write set"(let [source (th/create-mock-source@@ -14,8 +23,8 @@"other/file4.yaml" "content4"}}:managed-dirs #{"collections"})snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Write only abc"- [{:path "collections/abc/file1.yaml" :content "new-content1"}])+ (write-files! snapshot "Write only abc"+ [{:path "collections/abc/file1.yaml" :content "new-content1"}])(is (= #{"collections/abc/file1.yaml" "other/file4.yaml"}(set (source.p/list-files snapshot)))"Only written files in managed dirs should remain; unmanaged dirs untouched"))))@@ -27,8 +36,8 @@"unmanaged/file2.yaml" "content2"}}:managed-dirs #{"collections"})snapshot (source.p/snapshot source)]- (source.p/write-files! snapshot "Write collections"- [{:path "collections/abc/file1.yaml" :content "new-content"}])+ (write-files! snapshot "Write collections"+ [{:path "collections/abc/file1.yaml" :content "new-content"}])(is (= #{"collections/abc/file1.yaml" "unmanaged/file2.yaml"}(set (source.p/list-files snapshot)))"Unmanaged directory files should be preserved"))))@@ -41,8 +50,8 @@:managed-dirs #{"collections" "snippets"})snapshot (source.p/snapshot source)];; Write only to collections, nothing to snippets- (source.p/write-files! snapshot "Write only collections"- [{:path "collections/abc/file1.yaml" :content "new-content"}])+ (write-files! snapshot "Write only collections"+ [{:path "collections/abc/file1.yaml" :content "new-content"}])(is (= #{"collections/abc/file1.yaml"}(set (source.p/list-files snapshot)))"Snippets dir should be cleaned even though no snippet files were written"))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex 09ac4c2ef28..422c3393696 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -73,6 +73,38 @@(is (= "update" (:status entry))"Transform should have 'update' status after modification")))))))+(deftest transform-noop-update-does-not-redirty-test+ (testing "A transform-update event whose serialized form is unchanged must not re-dirty a synced transform (GHY-3933)"+ ;; Repro for the \"yellow dot with no changes\" report: editing a non-serialized aspect of a transform+ ;; (e.g. its schedule/run_trigger) — or any no-op re-save — fires :event/transform-update and flips the+ ;; RemoteSyncObject to \"update\", lighting the dirty indicator. But the serialized YAML is identical, so a+ ;; subsequent push produces a commit with no files. The dirty state should track the serialized form, not+ ;; the mere occurrence of an update event.+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxxxx" :location "/"}+ :model/Transform transform {:name "Synced Transform" :collection_id coll-id}+ :model/RemoteSyncObject _rso1 {:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "Transform" :model_id (:id transform) :model_name "Synced Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}]+ ;; Establish a synced baseline by exporting to the mock remote.+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Initial export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Transform" :model_id (:id transform))))+ "Transform should be synced after export"))+ ;; Re-publish an update event for the unchanged transform — the serialized form is identical to+ ;; what was just exported, so this must NOT re-dirty it.+ (events/publish-event! :event/transform-update {:object (t2/select-one :model/Transform :id (:id transform))})+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Transform" :model_id (:id transform))))+ "A no-op update must leave the transform synced (GHY-3933)")+ (is (not (sync-object/dirty?))+ "Instance must not report dirty state after a no-op transform update (GHY-3933)"))))))))+(deftest transform-tag-event-creates-sync-object-test(testing "Creating a transform tag creates a RemoteSyncObject entry when transform sync is enabled"(mt/with-premium-features #{:transforms-basic}diff --git a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsxindex 686182239a6..1433c072446 100644--- a/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx+++ b/frontend/src/metabase/nav/components/search/SearchBar/SearchBar.styled.tsx@@ -40,7 +40,6 @@ export const SearchInputContainer = styled.div<{}return css`background-color: var(--mb-color-background_page-primary);-&:hover {background-color: var(--mb-color-background_page-secondary);}diff --git a/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yaml b/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yamlnew file mode 100644index 00000000000..ea79396193e--- /dev/null+++ b/resources/migrations/063/20260618_ghy_3933_remote_sync_object_content_hash.yaml@@ -0,0 +1,22 @@+## Add content_hash column to remote_sync_object for content-based dirty detection (GHY-3933)++databaseChangeLog:++ - changeSet:+ id: v63.2026-06-18T00:00:00+ author: ericnormand+ comment: Add remote_sync_object.content_hash to record a hash of each entity's serialized form as of the last sync.+ changes:+ - addColumn:+ tableName: remote_sync_object+ columns:+ - column:+ name: content_hash+ type: ${text.type}+ remarks: SHA-256 (hex) of this entity's serialized YAML as of the last sync or null for no last sync.+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: remote_sync_object+ columnName: content_hash