The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
feat: support remote sync for NativeQuerySnippets (#68497)
Edward Pagetb8ed3bd4d013
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.cljindex 3e0dc052769..2bcf4ec3ae1 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj@@ -13,10 +13,10 @@- Card, Dashboard, Document, NativeQuerySnippet, Timeline, Collection- Table (when published in a remote-synced collection)- Field, Segment (when belonging to a published table in a remote-synced collection)- - Transform, TransformTag, transforms-namespace Collections (when remote-sync-transforms setting is enabled)"+ - Transform, TransformTag, transforms-namespace Collections (when remote-sync-transforms setting is enabled)+ - NativeQuerySnippet, snippets-namespace Collections (when Library is remote-synced)"(:require[java-time.api :as t]- [metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.spec :as spec][metabase.collections.core :as collections][metabase.events.core :as events]@@ -24,7 +24,41 @@[methodical.core :as methodical][toucan2.core :as t2]))-;; Helper functions for model change tracking+(defn sync-snippet-tracking!+ "Called when the Library collection's remote sync status changes.+ When enabled: mark all existing snippets and snippets-namespace collections+ as 'create' for initial sync.+ When disabled: remove all snippet-related tracking entries."+ [enabled?]+ (let [timestamp (t/offset-date-time)]+ (if enabled?+ (do+ ;; Mark all snippets-namespace collections for initial sync+ (doseq [coll (t2/select [:model/Collection :id :name] :namespace "snippets")]+ (t2/insert! :model/RemoteSyncObject+ {:model_type "Collection"+ :model_id (:id coll)+ :model_name (:name coll)+ :status "create"+ :status_changed_at timestamp}))+ ;; Mark all existing snippets for initial sync+ (doseq [snippet (t2/select [:model/NativeQuerySnippet :id :name :collection_id])]+ (t2/insert! :model/RemoteSyncObject+ {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name (:name snippet)+ :model_collection_id (:collection_id snippet)+ :status "create"+ :status_changed_at timestamp})))+ (let [snippet-coll-ids (t2/select-pks-set :model/Collection :namespace "snippets")]+ (t2/delete! :model/RemoteSyncObject+ :model_type "NativeQuerySnippet")+ (when (seq snippet-coll-ids)+ (t2/delete! :model/RemoteSyncObject+ :model_type "Collection"+ :model_id [:in snippet-coll-ids]))))))++;;; ----------------------------------------- Helper Functions ---------------------------------------------------------(defn- create-or-update-remote-sync-object-entry!"Creates or updates a remote sync object entry for a model change.@@ -50,18 +84,12 @@:model_table_name (:table_name model-details):status status:status_changed_at (t/offset-date-time)}))-- ;; If the entity was created and then removed/deleted before sync, just delete the entry from tracking(and (= "create" (:status existing)) (contains? #{"removed" "delete"} status))(t2/delete! :model/RemoteSyncObject (:id existing))-- ;; Just update the status, object doesn't exist to update other info(= "delete" (:status existing))(t2/update! :model/RemoteSyncObject (:id existing){:status status:status_changed_at (t/offset-date-time)})-- ;; If the entry was created, the status should remain create until synced(not (= "create" (:status existing)))(let [model-details (hydrate-details-fn model-id)](t2/update! :model/RemoteSyncObject (:id existing)@@ -82,7 +110,6 @@(let [model-type (:model-type model-spec)existing (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)](cond- ;; No existing entry - create new one(not existing)(let [model-details (spec/hydrate-model-details model-spec model-id)fields (spec/build-sync-object-fields model-spec model-details)]@@ -92,18 +119,12 @@:status status:status_changed_at (t/offset-date-time)}fields)))-- ;; Created then removed/deleted before sync - delete the entry(and (= "create" (:status existing)) (contains? #{"removed" "delete"} status))(t2/delete! :model/RemoteSyncObject (:id existing))-- ;; Already deleted - just update status(= "delete" (:status existing))(t2/update! :model/RemoteSyncObject (:id existing){:status status:status_changed_at (t/offset-date-time)})-- ;; Status is not "create" - update with new details(not= "create" (:status existing))(let [model-details (spec/hydrate-model-details model-spec model-id)fields (spec/build-sync-object-fields model-spec model-details)]@@ -122,14 +143,11 @@existing-entry (t2/select-one :model/RemoteSyncObject :model_type model-type :model_id model-id)status (spec/determine-status model-spec topic object)](cond- ;; Model is eligible for synceligible?(do(log/infof "Creating remote sync object entry for %s %s (status: %s)"model-type model-id status)(create-or-update-sync-object-from-spec! model-spec model-id status))-- ;; Model was synced but no longer eligible - mark as removed(and existing-entry (not eligible?))(do(log/infof "%s %s moved out of sync scope, marking as removed" model-type model-id)@@ -141,15 +159,9 @@[model-spec](let [event-kws (spec/event-keywords model-spec)parent-kw (:parent event-kws)]- ;; Derive event hierarchy(derive parent-kw :metabase/event)(doseq [[_event-type event-kw] (dissoc event-kws :parent)](derive event-kw parent-kw))-- ;; Register the handler using methodical's runtime API- ;; (defmethod requires compile-time dispatch values, so we use add-primary-method!)- ;; Note: We use #'events/publish-event! (the var) because add-primary-method! expects a var/atom- ;; The method signature matches defmethod: [topic event], not [next-method topic event](methodical/add-primary-method!#'events/publish-event!parent-kw@@ -157,8 +169,6 @@(handle-model-event-from-spec model-spec topic event)))));;; --------------------------------- Spec-based Event Registration (Non-Collection) ------------------------------------;; Register event handlers for models that use standard spec-based handling.-;; Collection has special handling and is registered separately below.(doseq [[_model-key model-spec] (dissoc spec/remote-sync-specs :model/Collection)](register-events-for-spec! model-spec))@@ -168,7 +178,6 @@;; a collection becomes remote-synced). This handler is kept separate from the standard;; spec-based registration.-;; Collection create/update events - derive from common parent for shared handling(derive ::collection-change-event :metabase/event)(derive :event/collection-create ::collection-change-event)(derive :event/collection-update ::collection-change-event)@@ -178,46 +187,43 @@[id](t2/select-one [:model/Collection :name [:id :collection_id]] :id id))-(defn- transforms-namespace-collection?- "Check if this is a transforms-namespace collection."- [collection]- (= (keyword (:namespace collection)) :transforms))--(defn- should-sync-collection?- "Check if a collection should be synced - either remote-synced or transforms-namespace with setting enabled."- [collection]- (or (collections/remote-synced-collection? collection)- (and (settings/remote-sync-transforms)- (transforms-namespace-collection? collection))))--(defn- hydrate-table-details- "Hydrates details for a Table. For tables, table_id and table_name refer to themselves."- [id]- (when-let [table (t2/select-one [:model/Table :name :collection_id] :id id)]- (assoc table :table_id id :table_name (:name table))))-(defn- track-published-tables-in-collection!"When a collection becomes remote-synced, find all published tables in itand create 'create' entries for them in RemoteSyncObject (if not already tracked)."[collection-id](let [published-tables (t2/select :model/Table:collection_id collection-id- :is_published true)]+ :is_published true)+ table-spec (spec/remote-sync-specs :model/Table)](doseq [table published-tables](let [existing (t2/select-one :model/RemoteSyncObject:model_type "Table":model_id (:id table))]- ;; Only create entry if not already tracked or was marked as removed/synced(when (or (nil? existing)(contains? #{"removed" "synced"} (:status existing)))(log/infof "Creating remote sync object entry for published table %s in newly remote-synced collection %s"(:id table) collection-id)- (create-or-update-remote-sync-object-entry! "Table" (:id table) "create" hydrate-table-details))))))+ (create-or-update-sync-object-from-spec! table-spec (:id table) "create"))))))++(defn- handle-library-sync-status-change!+ "When the Library collection's is_remote_synced status changes, trigger snippet sync tracking.+ This ensures all snippets are tracked/untracked when Library sync is enabled/disabled."+ [is-now-synced?]+ (let [snippets-already-tracked? (t2/exists? :model/RemoteSyncObject :model_type "NativeQuerySnippet")]+ (cond+ (and is-now-synced? (not snippets-already-tracked?))+ (do+ (log/info "Library collection became remote-synced, enabling snippet sync tracking")+ (sync-snippet-tracking! true))+ (and (not is-now-synced?) snippets-already-tracked?)+ (do+ (log/info "Library collection is no longer remote-synced, disabling snippet sync tracking")+ (sync-snippet-tracking! false)))))(methodical/defmethod events/publish-event! ::collection-change-event[topic event](let [{:keys [object]} event- should-sync? (should-sync-collection? object)+ should-sync? (spec/should-sync-collection? object)is-remote-synced? (collections/remote-synced-collection? object)existing-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id object))was-synced? (and existing-entry@@ -227,25 +233,16 @@(case topic:event/collection-create "create":event/collection-update "update"))]+ (when (and (= topic :event/collection-update)+ (spec/library-collection? object))+ (handle-library-sync-status-change! is-remote-synced?))(cond- ;; Collection should be synced (remote-synced or transforms-namespace with setting enabled)should-sync?(do(log/infof "Creating remote sync object entry for collection %s (status: %s)" (:id object) status)(create-or-update-remote-sync-object-entry! "Collection" (:id object) status hydrate-collection-details)- ;; If collection just became remote-synced, track all published tables in it- ;; (only for actual remote-synced collections, not transforms-namespace)(when (and is-remote-synced? (not was-synced?))- (track-published-tables-in-collection! (:id object)))- ;; When a transforms-namespace collection is archived, also mark child transforms for deletion- (when (and (:archived object) (transforms-namespace-collection? object))- (let [transform-ids (t2/select-pks-set :model/Transform :collection_id (:id object))]- (doseq [transform-id transform-ids]- (log/infof "Marking transform %s for deletion (parent collection archived)" transform-id)- (create-or-update-remote-sync-object-entry! "Transform" transform-id "delete"- (fn [id] (t2/select-one [:model/Transform :name [:collection_id :model_collection_id]] :id id)))))))-- ;; Collection was synced but no longer should be - mark as removed+ (track-published-tables-in-collection! (:id object))))(and existing-entry (not should-sync?))(do(log/infof "Collection %s no longer needs syncing, marking as removed" (:id object))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 5a18137968a..d665063f08e 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -23,11 +23,6 @@[toucan2.core :as t2])(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))-(defn- all-top-level-remote-synced-collections- "Returns a vector of primary keys for all top-level remote-synced collections."- []- (t2/select-pks-vec :model/Collection :is_remote_synced true))-(defn- sync-objects!"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records andinserts new ones for each imported entity, marking them as 'synced' with the given timestamp.@@ -49,9 +44,12 @@entity-id based model, deletes entities whose entity_id is not in the imported set.Models with :scope-key in their spec are scoped to synced collections (using :id for Collection, :collection_id- for others). Models without :scope-key (like TransformTag) are deleted globally by entity_id."+ for others). Models without :scope-key (like TransformTag) are deleted globally by entity_id.++ Uses specs-for-deletion to process models in dependency order (models with FK references to+ Collection are deleted before Collection itself)."[synced-collection-ids imported-entities-by-model]- (doseq [[model-key model-spec] (spec/specs-by-identity-type :entity-id)+ (doseq [[model-key model-spec] (spec/specs-for-deletion):let [serdes-model (:model-type model-spec)entity-ids (get imported-entities-by-model serdes-model [])entity-id-clause (if (seq entity-ids)@@ -142,7 +140,9 @@(log/infof (:message <>)))(let [path-filters (cond-> [#"collections/.*" #"databases/.*" #"actions/.*"](settings/remote-sync-transforms)- (conj #"transforms/.*"))+ (conj #"transforms/.*")+ (settings/library-is-remote-synced?)+ (conj #"snippets/.*"))ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters path-filters})(source.ingestable/wrap-progress-ingestable task-id 0.7))load-result (serdes/with-cache@@ -177,7 +177,7 @@:field_name (-> path last :id)}))))](remote-sync.task/update-progress! task-id 0.8)(t2/with-transaction [_conn]- (remove-unsynced! (all-top-level-remote-synced-collections) imported-entities-by-model)+ (remove-unsynced! (spec/all-syncable-collection-ids) imported-entities-by-model)(sync-objects! sync-timestamp imported-entities-by-model table-paths field-paths)(when (and (nil? (collection/remote-synced-collection)) (= :read-write (settings/remote-sync-type)))(collection/create-remote-synced-collection!)))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex 659d408aed6..d0f7aa51247 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -193,3 +193,11 @@(when (and (contains? settings k)(not (and (= k :remote-sync-token) obfuscated?)))(setting/set! k (k settings))))))))++(defn library-is-remote-synced?+ "Returns true if the Library collection exists and is remote-synced.+ When true, all snippets and snippet collections should be synced."+ []+ (boolean+ (when-let [library (collection/library-collection)]+ (collection/remote-synced-collection? library))))diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 50f5fc2379f..42c131b090d 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -33,13 +33,17 @@- :model-key - Toucan2 model keyword (e.g., :model/Card)- :identity - Identity strategy: :entity-id, :path, or :hybrid- :path-keys - For path-based: vector of path components [:database :schema :table :field]+ - :delete-after - Optional vector of model keys that must be deleted AFTER this model.+ Used to handle FK constraints during import cleanup. For example, if Card+ has :delete-after [:model/Collection], Card will be deleted before Collection.- :events - Event configuration map::prefix - Event keyword prefix (e.g., :event/card):types - Vector of event types to handle [:create :update :delete]- :eligibility - Eligibility configuration:- :type - :collection, :published-table, :parent-table, or :setting- :collection - For :collection type: :remote-synced, :transforms-namespace, or :any+ :type - :collection, :published-table, :parent-table, :setting, or :library-synced+ :collection - For :collection type: :remote-synced, :transforms-namespace, :snippets-namespace, or :any:setting - For :setting type: setting keyword to check+ (Note: :library-synced type uses the library-is-remote-synced? setting to determine eligibility)- :archived-key - Key to check for archived status (nil if model has no archived concept)- :tracking - RemoteSyncObject field configuration::select-fields - Fields to select for hydration@@ -51,14 +55,16 @@If nil, deletions are global (by entity_id only).- :export-path - Export path construction::type - Path type: :collection-entity, :table-path, :field-path, :segment-path,- :transform-path, :transform-tag-path+ :transform-path, :transform-tag-path, :snippet-path, :measure-path- :side-effects - Optional side effects::on-first-sync - Function to call when model first becomes synced- - :enabled? - true or setting keyword (e.g., :remote-sync-transforms)"+ - :enabled? - true, or setting keyword (e.g., :remote-sync-transforms, :library-synced).+ When :library-synced, uses the library-is-remote-synced? setting."{:model/Card{:model-type "Card":model-key :model/Card:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/card:types [:create :update :delete]}:eligibility {:type :collection@@ -77,6 +83,7 @@{:model-type "Dashboard":model-key :model/Dashboard:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/dashboard:types [:create :update :delete]}:eligibility {:type :collection@@ -94,6 +101,7 @@{:model-type "Document":model-key :model/Document:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/document:types [:create :update :delete]}:eligibility {:type :collection@@ -111,22 +119,24 @@{:model-type "NativeQuerySnippet":model-key :model/NativeQuerySnippet:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/snippet:types [:create :update :delete]}- :eligibility {:type :collection- :collection :remote-synced}+ :eligibility {:type :library-synced} ; sync all snippets when Library is remote-synced:archived-key :archived- :tracking {:select-fields [:name :id] ; snippets don't have collection_id directly- :field-mappings {:model_name :name}}- :removal {:statuses #{"removed"}- :scope-key :collection_id}- :export-path {:type :collection-entity}- :enabled? true}+ :tracking {:select-fields [:name :collection_id]+ :field-mappings {:model_name :name+ :model_collection_id :collection_id}}+ :removal {:statuses #{"removed" "delete"}} ; no scope-key = global deletion+ :export-path {:type :snippet-path}+ :export-scope :all ; export all snippets+ :enabled? :library-synced}:model/Timeline{:model-type "Timeline":model-key :model/Timeline:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/timeline:types [:create :update :delete]}:eligibility {:type :collection@@ -255,6 +265,7 @@{:model-type "Transform":model-key :model/Transform:identity :entity-id+ :delete-after [:model/Collection] ; has collection_id FK:events {:prefix :event/transform:types [:create :update :delete]}:eligibility {:type :setting@@ -290,9 +301,10 @@"Returns true if the spec is currently enabled based on its :enabled? value."[{:keys [enabled?]}](cond- (true? enabled?) true- (keyword? enabled?) (boolean (setting/get-value-of-type :boolean enabled?))- :else false))+ (true? enabled?) true+ (= enabled? :library-synced) (rs-settings/library-is-remote-synced?)+ (keyword? enabled?) (boolean (setting/get-value-of-type :boolean enabled?))+ :else false))(defn enabled-specs"Returns a map of model-key -> spec for all currently enabled specs."@@ -321,15 +333,56 @@(filter (fn [[_ spec]] (= (:identity spec) identity-type)))(enabled-specs)))+(defn specs-for-deletion+ "Returns specs with :entity-id identity type, topologically sorted for safe deletion.+ Uses :delete-after declarations to ensure models are deleted before their dependencies.+ If model A has :delete-after [:model/B], then A will be deleted before B."+ []+ (let [entity-id-specs (specs-by-identity-type :entity-id)+ ;; Build dependency graph: model -> set of models that must be deleted after it+ ;; (i.e., the models listed in this model's :delete-after)+ delete-after-map (reduce-kv+ (fn [m model-key spec]+ (assoc m model-key (set (get spec :delete-after))))+ {}+ entity-id-specs)+ ;; Invert to get: model -> set of models that must be deleted before it+ ;; If A has :delete-after [B], then B must wait for A to be deleted first+ must-wait-for (reduce-kv+ (fn [m model-key deps]+ (reduce (fn [m' dep]+ (update m' dep (fnil conj #{}) model-key))+ m+ deps))+ {}+ delete-after-map)+ ;; Simple topological sort - repeatedly take models with no remaining dependencies+ sorted (loop [remaining (set (keys entity-id-specs))+ result []]+ (if (empty? remaining)+ result+ (let [;; Find models whose dependencies are all already processed+ ready (filter (fn [k]+ (every? #(not (remaining %))+ (get must-wait-for k)))+ remaining)]+ (if (empty? ready)+ ;; Cycle detected or no progress - just append remaining+ (into result remaining)+ (recur (apply disj remaining ready)+ (into result ready))))))]+ (mapv (fn [k] [k (entity-id-specs k)]) sorted)))+(defn excluded-model-types"Returns a set of model type strings that should be excluded from dirty detection- based on current settings. Models with a setting-based :enabled? that is currently- false will be excluded."+ based on current settings. Models with a setting-based or library-synced :enabled?+ that is currently false will be excluded."[](->> remote-sync-specs(filter (fn [[_ spec]]- (and (keyword? (:enabled? spec))- (not (setting/get-value-of-type :boolean (:enabled? spec))))))+ (let [enabled? (:enabled? spec)]+ (and (keyword? enabled?)+ (not (spec-enabled? spec))))))(map (fn [[_ spec]] (:model-type spec)))set))@@ -340,11 +393,48 @@;;; ------------------------------------------------ Eligibility Checking -----------------------------------------------(defn- transforms-namespace-collection?+(defn transforms-namespace-collection?"Check if this is a transforms-namespace collection."[object](= (keyword (:namespace object)) :transforms))+(defn snippets-namespace-collection?+ "Check if this is a snippets-namespace collection."+ [object]+ (= (keyword (:namespace object)) :snippets))++(defn library-collection?+ "Check if this is the Library collection."+ [collection]+ (= (:type collection) "library"))++(defn should-sync-collection?+ "Check if a collection should be synced - either remote-synced, transforms-namespace with setting enabled,+ or snippets-namespace with Library synced."+ [collection]+ (or (collections/remote-synced-collection? collection)+ (and (rs-settings/remote-sync-transforms)+ (transforms-namespace-collection? collection))+ (and (rs-settings/library-is-remote-synced?)+ (snippets-namespace-collection? collection))))++(defn all-syncable-collection-ids+ "Returns a vector of all collection IDs that are eligible for remote sync.+ This includes:+ - Collections with is_remote_synced=true+ - Transforms-namespace collections (when remote-sync-transforms setting is enabled)+ - Snippets-namespace collections (when Library is remote-synced)++ Used by import cleanup to determine which collections to scope deletions to."+ []+ (into []+ cat+ [(t2/select-pks-vec :model/Collection :is_remote_synced true)+ (when (rs-settings/remote-sync-transforms)+ (t2/select-pks-vec :model/Collection :namespace (name collections/transforms-ns)))+ (when (rs-settings/library-is-remote-synced?)+ (t2/select-pks-vec :model/Collection :namespace "snippets"))]))+(defmulti check-eligibility"Determines if a model instance should be tracked for remote sync.Dispatches on the eligibility type defined in the spec."@@ -363,10 +453,16 @@(and (rs-settings/remote-sync-transforms)(transforms-namespace-collection? object))+ :snippets-namespace+ (and (rs-settings/library-is-remote-synced?)+ (snippets-namespace-collection? object))+:any(or (collections/remote-synced-collection? (or collection-id object))(and (rs-settings/remote-sync-transforms)- (transforms-namespace-collection? object)))+ (transforms-namespace-collection? object))+ (and (rs-settings/library-is-remote-synced?)+ (snippets-namespace-collection? object)))false)))@@ -387,6 +483,10 @@[{:keys [eligibility]} _object](boolean (setting/get-value-of-type :boolean (:setting eligibility))))+(defmethod check-eligibility :library-synced+ [_spec _object]+ (rs-settings/library-is-remote-synced?))+(defmethod check-eligibility :default[_ _]false)@@ -456,12 +556,16 @@(defn- transform-entity-for-serdes"Transforms entity fields to serdes format for path generation.- - Converts integer table_id to [db-name schema table-name] format"+ - Converts integer table_id to [db-name schema table-name] format+ - Converts integer collection_id to entity_id string for collection path lookup"[entity](cond-> entity;; Transform table_id for Segment (and other table-based entities)(and (:table_id entity) (integer? (:table_id entity)))- (assoc :table_id (serdes/*export-table-fk* (:table_id entity)))))+ (assoc :table_id (serdes/*export-table-fk* (:table_id entity)))+ ;; Transform collection_id for snippet and other collection-based entities+ (and (:collection_id entity) (integer? (:collection_id entity)))+ (assoc :collection_id (t2/select-one-fn :entity_id :model/Collection :id (:collection_id entity)))))(defn- entity->serdes-path"Builds the file path for an entity using serdes/storage-path.@@ -728,7 +832,7 @@[{:keys [export-scope]}](case (or export-scope :derived):root-collections- ;; Collection model: query for root-level remote-synced + transforms-namespace collections+ ;; Collection model: query for root-level remote-synced + transforms-namespace + snippets-namespace collections;; Excludes archived collections - their files are handled by the removal logic(concat(t2/select-fn-set (juxt (constantly "Collection") :id)@@ -743,6 +847,13 @@{:where [:and[:= :namespace (name collections/transforms-ns)][:= :location "/"]+ [:not :archived]]}))+ (when (rs-settings/library-is-remote-synced?)+ (t2/select-fn-set (juxt (constantly "Collection") :id)+ :model/Collection+ {:where [:and+ [:= :namespace "snippets"]+ [:= :location "/"][:not :archived]]}))):derived;; Other collection-based models: no root query, derived from collection expansion@@ -764,6 +875,18 @@(defmethod query-export-roots :published-table [_] nil) ; Derived via serdes/descendants(defmethod query-export-roots :parent-table [_] nil) ; Derived via serdes/descendants+(defmethod query-export-roots :library-synced+ [{:keys [export-scope model-key model-type archived-key] :as spec}]+ (when (spec-enabled? spec)+ (case export-scope+ :all+ ;; NativeQuerySnippet: all non-archived instances+ (if archived-key+ (t2/select-fn-set (juxt (constantly model-type) :id) model-key archived-key false)+ (t2/select-fn-set (juxt (constantly model-type) :id) model-key))+ ;; Default: no root query+ nil)))+(defmethod query-export-roots :default [_] nil)(defn- resolve-targets@@ -804,6 +927,7 @@:continue-on-error false:skip-archived true})](eduction (map (fn [[model ids]]- (serdes/extract-all model {:where [:in :id ids]})))+ (serdes/extract-all model {:where [:in :id ids]+ :skip-archived true})))cat(u/group-by first second (keys targets))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.cljindex 197e924a75a..f0482fe8fbd 100644--- a/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_studio/api/table_test.clj@@ -4,7 +4,7 @@[clojure.test :refer :all][metabase-enterprise.data-studio.api.table :as api.table][metabase.collections.models.collection :as collection]- [metabase.collections.test-helpers :refer [without-library]]+ [metabase.collections.test-utils :refer [without-library]][metabase.permissions.models.permissions-group :as perms-group][metabase.sync.core :as sync][metabase.test :as mt]diff --git a/enterprise/backend/test/metabase_enterprise/library/api_test.clj b/enterprise/backend/test/metabase_enterprise/library/api_test.cljindex f23b8e1f00a..d608aca0361 100644--- a/enterprise/backend/test/metabase_enterprise/library/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/library/api_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer :all][metabase.collections.models.collection :as collection]- [metabase.collections.test-helpers :refer [without-library]]+ [metabase.collections.test-utils :refer [without-library]][metabase.test :as mt][toucan2.core :as t2]))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.cljindex 4bf5b8fca78..b46a40eb9a7 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj@@ -7,6 +7,8 @@[java-time.api :as t][metabase-enterprise.remote-sync.events :as remote-sync.events][metabase-enterprise.remote-sync.impl :as impl]+ [metabase.collections.models.collection :as collection]+ [metabase.collections.test-utils :refer [with-library-synced]][metabase.events.core :as events][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -239,142 +241,87 @@(first entries)))))))(deftest snippet-create-event-creates-entry-test- (testing "snippet-create event creates remote sync object entry with create status"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-create- {:object snippet :user-id (mt/user->id :rasta)})- (let [entries (t2/select :model/RemoteSyncObject)]- (is (= 1 (count entries)))- (is (=? {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :status "create"}- (first entries)))))))+ (testing "snippet-create event creates remote sync object entry with create status when Library is synced"+ (with-library-synced+ (mt/with-temp [:model/Collection snippet-collection {:name "Snippets" :namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id (:id snippet-collection)}]+ (t2/delete! :model/RemoteSyncObject)+ (events/publish-event! :event/snippet-create+ {:object snippet :user-id (mt/user->id :rasta)})+ (let [entries (t2/select :model/RemoteSyncObject)]+ (is (= 1 (count entries)))+ (is (=? {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :status "create"}+ (first entries))))))))(deftest snippet-update-event-creates-entry-test- (testing "snippet-update event creates or updates remote sync object entry with update status"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-update- {:object snippet :user-id (mt/user->id :rasta)})- (let [entries (t2/select :model/RemoteSyncObject)]- (is (= 1 (count entries)))- (is (=? {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :status "update"}- (first entries)))))))+ (testing "snippet-update event creates or updates remote sync object entry with update status when Library is synced"+ (with-library-synced+ (mt/with-temp [:model/Collection snippet-collection {:name "Snippets" :namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id (:id snippet-collection)}]+ (t2/delete! :model/RemoteSyncObject)+ (events/publish-event! :event/snippet-update+ {:object snippet :user-id (mt/user->id :rasta)})+ (let [entries (t2/select :model/RemoteSyncObject)]+ (is (= 1 (count entries)))+ (is (=? {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :status "update"}+ (first entries))))))))(deftest snippet-update-archived-sets-delete-test- (testing "snippet-update event with archived=true sets delete status"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-update- {:object (assoc snippet :archived true)- :user-id (mt/user->id :rasta)})- (let [entries (t2/select :model/RemoteSyncObject)]- (is (= 1 (count entries)))- (is (=? {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :status "delete"}- (first entries)))))))--(deftest snippet-delete-event-creates-entry-test- (testing "snippet-delete event creates remote sync object entry with delete status"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-delete- {:object snippet :user-id (mt/user->id :rasta)})- (let [entries (t2/select :model/RemoteSyncObject)]- (is (= 1 (count entries)))- (is (=? {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :status "delete"}- (first entries)))))))--(deftest snippet-event-in-normal-collection-no-entry-test- (testing "snippet events in non-remote-sync collections don't create entries"- (mt/with-temp [:model/Collection normal-collection {:name "Normal" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Normal Snippet"- :content "SELECT 1"- :collection_id (:id normal-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-create- {:object snippet :user-id (mt/user->id :rasta)})- (let [entries (t2/select :model/RemoteSyncObject)]- (is (= 0 (count entries)))))))--(deftest existing-snippet-moved-out-of-remote-synced-collection-test- (testing "snippet moved out of remote-synced collection is marked as removed"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/Collection normal-collection {:name "Normal" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {"NativeQuerySnippet" #{(:entity_id snippet)}} nil nil)-- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))]- (is (= "synced" (:status initial-entry)))+ (testing "snippet-update event with archived=true sets delete status when Library is synced"+ (with-library-synced+ (mt/with-temp [:model/Collection snippet-collection {:name "Snippets" :namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id (:id snippet-collection)}]+ (t2/delete! :model/RemoteSyncObject)(events/publish-event! :event/snippet-update- {:object (assoc snippet :collection_id (:id normal-collection))+ {:object (assoc snippet :archived true):user-id (mt/user->id :rasta)})+ (let [entries (t2/select :model/RemoteSyncObject)]+ (is (= 1 (count entries)))+ (is (=? {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :status "delete"}+ (first entries))))))))- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))]- (is (= "removed" (:status update-entry)))- (is (= (:id initial-entry) (:id update-entry)))))- "done")))--(deftest new-snippet-moved-out-of-remote-synced-collection-test- (testing "new snippet moved out of remote-synced collection are not tracked"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/Collection normal-collection {:name "Normal" :namespace "snippets"}- :model/NativeQuerySnippet snippet {:name "Test Snippet"- :content "SELECT 1"- :collection_id (:id remote-sync-collection)}]- (t2/delete! :model/RemoteSyncObject)- (events/publish-event! :event/snippet-create- {:object snippet :user-id (mt/user->id :rasta)})- (let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))]- (is (= "create" (:status initial-entry)))- (events/publish-event! :event/snippet-update- {:object (assoc snippet :collection_id (:id normal-collection))- :user-id (mt/user->id :rasta)})- (let [update-entry (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))]- (is (nil? update-entry)))))))+(deftest snippet-delete-event-creates-entry-test+ (testing "snippet-delete event creates remote sync object entry with delete status when Library is synced"+ (with-library-synced+ (mt/with-temp [:model/Collection snippet-collection {:name "Snippets" :namespace "snippets"}+ :model/NativeQuerySnippet snippet {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id (:id snippet-collection)}]+ (t2/delete! :model/RemoteSyncObject)+ (events/publish-event! :event/snippet-delete+ {:object snippet :user-id (mt/user->id :rasta)})+ (let [entries (t2/select :model/RemoteSyncObject)]+ (is (= 1 (count entries)))+ (is (=? {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :status "delete"}+ (first entries))))))))-(deftest snippet-moved-into-remote-synced-collection-test- (testing "snippet moved into remote-synced collection gets tracked"- (mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync" :namespace "snippets"}- :model/Collection normal-collection {:name "Normal" :namespace "snippets"}+(deftest snippet-event-when-library-not-synced-no-entry-test+ (testing "snippet events don't create entries when Library is not synced"+ (when-let [library (collection/library-collection)]+ (t2/update! :model/Collection (:id library) {:is_remote_synced false}))+ (mt/with-temp [:model/Collection snippet-collection {:name "Snippets" :namespace "snippets"}:model/NativeQuerySnippet snippet {:name "Test Snippet":content "SELECT 1"- :collection_id (:id normal-collection)}]+ :collection_id (:id snippet-collection)}](t2/delete! :model/RemoteSyncObject)(events/publish-event! :event/snippet-create{:object snippet :user-id (mt/user->id :rasta)})- (is (= 0 (count (t2/select :model/RemoteSyncObject))))-- ;; Move snippet to remote-synced collection- (events/publish-event! :event/snippet-update- {:object (assoc snippet :collection_id (:id remote-sync-collection))- :user-id (mt/user->id :rasta)})(let [entries (t2/select :model/RemoteSyncObject)]- (is (= 1 (count entries)))- (is (=? {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :status "update"}- (first entries)))))))+ (is (= 0 (count entries)))))))(deftest collection-create-event-creates-entry-test(testing "collection-create event creates remote sync object entry with create status"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.cljindex 9d6154aa582..4eeabf5628d 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/models/remote_sync_object_test.clj@@ -4,6 +4,7 @@(:require[clojure.test :refer :all][metabase-enterprise.remote-sync.models.remote-sync-object :as rs-object]+ [metabase-enterprise.remote-sync.spec :as spec][metabase-enterprise.remote-sync.test-helpers :as th][metabase.test :as mt][metabase.test.fixtures :as fixtures]))@@ -452,86 +453,88 @@(deftest dirty-objects-all-model-types-including-new-test(testing "dirty-objects handles all supported model types including Table, Field, and Segment"- (mt/with-temp- [:model/Collection collection {:name "Test Collection"- :location "/"}- :model/Collection snip-collection {:name "Snippet Collection"- :namespace "snippets"- :location "/"}- :model/Card card {:name "Card Item"- :collection_id (:id collection)- :creator_id (mt/user->id :rasta)- :display "table"- :dataset_query (mt/native-query {:query "SELECT 1"})}- :model/Dashboard dashboard {:name "Dashboard Item"+ ;; Mock excluded-model-types to return empty set so all model types (including NativeQuerySnippet) are included+ (with-redefs [spec/excluded-model-types (constantly #{})]+ (mt/with-temp+ [:model/Collection collection {:name "Test Collection"+ :location "/"}+ :model/Collection snip-collection {:name "Snippet Collection"+ :namespace "snippets"+ :location "/"}+ :model/Card card {:name "Card Item"+ :collection_id (:id collection)+ :creator_id (mt/user->id :rasta)+ :display "table"+ :dataset_query (mt/native-query {:query "SELECT 1"})}+ :model/Dashboard dashboard {:name "Dashboard Item"+ :collection_id (:id collection)+ :creator_id (mt/user->id :rasta)}+ :model/Document document {:name "Document Item":collection_id (:id collection):creator_id (mt/user->id :rasta)}- :model/Document document {:name "Document Item"- :collection_id (:id collection)- :creator_id (mt/user->id :rasta)}- :model/NativeQuerySnippet snippet {:name "Snippet Item"- :collection_id (:id snip-collection)- :creator_id (mt/user->id :rasta)- :content "SELECT * FROM table"}- :model/Table table {:name "Table Item"- :collection_id (:id collection)- :is_published true}- :model/Field field {:name "field_item"- :table_id (:id table)- :base_type :type/Text- :database_type "TEXT"}- :model/Segment segment {:name "Segment Item"- :table_id (:id table)- :creator_id (mt/user->id :rasta)- :definition {:source-table (:id table)- :filter [:= [:field (:id field) nil] "test"]}}- :model/RemoteSyncObject _ {:model_type "Collection"- :model_id (:id collection)- :model_name "Test Collection"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Card"- :model_id (:id card)- :model_name "Card Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Dashboard"- :model_id (:id dashboard)- :model_name "Dashboard Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Document"- :model_id (:id document)- :model_name "Document Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "NativeQuerySnippet"- :model_id (:id snippet)- :model_name "Snippet Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Table"- :model_id (:id table)- :model_name "Table Item"- :model_table_id (:id table)- :model_table_name "Table Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Field"- :model_id (:id field)- :model_name "field_item"- :model_table_id (:id table)- :model_table_name "Table Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}- :model/RemoteSyncObject _ {:model_type "Segment"- :model_id (:id segment)- :model_name "Segment Item"- :model_table_id (:id table)- :model_table_name "Table Item"- :status "pending"- :status_changed_at (java.time.OffsetDateTime/now)}]- (let [dirty-items (rs-object/dirty-objects)- models (set (map :model dirty-items))]- (is (= 8 (count dirty-items)))- (is (= #{"collection" "card" "dashboard" "document" "nativequerysnippet" "table" "field" "segment"} models))))))+ :model/NativeQuerySnippet snippet {:name "Snippet Item"+ :collection_id (:id snip-collection)+ :creator_id (mt/user->id :rasta)+ :content "SELECT * FROM table"}+ :model/Table table {:name "Table Item"+ :collection_id (:id collection)+ :is_published true}+ :model/Field field {:name "field_item"+ :table_id (:id table)+ :base_type :type/Text+ :database_type "TEXT"}+ :model/Segment segment {:name "Segment Item"+ :table_id (:id table)+ :creator_id (mt/user->id :rasta)+ :definition {:source-table (:id table)+ :filter [:= [:field (:id field) nil] "test"]}}+ :model/RemoteSyncObject _ {:model_type "Collection"+ :model_id (:id collection)+ :model_name "Test Collection"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Card"+ :model_id (:id card)+ :model_name "Card Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Dashboard"+ :model_id (:id dashboard)+ :model_name "Dashboard Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Document"+ :model_id (:id document)+ :model_name "Document Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Snippet Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Table"+ :model_id (:id table)+ :model_name "Table Item"+ :model_table_id (:id table)+ :model_table_name "Table Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Field"+ :model_id (:id field)+ :model_name "field_item"+ :model_table_id (:id table)+ :model_table_name "Table Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}+ :model/RemoteSyncObject _ {:model_type "Segment"+ :model_id (:id segment)+ :model_name "Segment Item"+ :model_table_id (:id table)+ :model_table_name "Table Item"+ :status "pending"+ :status_changed_at (java.time.OffsetDateTime/now)}]+ (let [dirty-items (rs-object/dirty-objects)+ models (set (map :model dirty-items))]+ (is (= 8 (count dirty-items)))+ (is (= #{"collection" "card" "dashboard" "document" "nativequerysnippet" "table" "field" "segment"} models)))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.cljnew file mode 100644index 00000000000..1dbe7033473--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/snippets_test.clj@@ -0,0 +1,443 @@+(ns metabase-enterprise.remote-sync.snippets-test+ "Tests for snippet sync functionality in remote-sync.+ Snippets are synced globally when the Library collection is remote-synced."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase-enterprise.remote-sync.events :as rs-events]+ [metabase-enterprise.remote-sync.impl :as impl]+ [metabase-enterprise.remote-sync.models.remote-sync-object :as sync-object]+ [metabase-enterprise.remote-sync.settings :as settings]+ [metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.spec :as spec]+ [metabase-enterprise.remote-sync.test-helpers :as test-helpers]+ [metabase.collections.models.collection :as collection]+ [metabase.collections.test-utils :refer [with-library-not-synced with-library-synced]]+ [metabase.events.core :as events]+ [metabase.search.core :as search]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db))++(defn- clean-remote-sync-state+ "Test fixture that cleans up remote sync state before and after each test."+ [f]+ (try+ (t2/delete! :model/RemoteSyncObject)+ (f)+ (finally+ (t2/delete! :model/RemoteSyncObject))))++#_{:clj-kondo/ignore [:metabase/validate-deftest]}+(use-fixtures :each (fn [f]+ (mt/with-dynamic-fn-redefs [search/reindex! (constantly nil)]+ (clean-remote-sync-state f))))++;;; ------------------------------------------- Event Handler Tests -------------------------------------------++(deftest snippet-event-creates-sync-object-when-library-synced-test+ (testing "Creating a snippet creates a RemoteSyncObject entry when Library is remote-synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]+ (events/publish-event! :event/snippet-create {:object snippet :user-id (mt/user->id :rasta)})+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet))+ "Snippet should be tracked when Library is remote-synced"))))))++(deftest snippet-event-ignored-when-library-not-synced-test+ (testing "Snippet events are ignored when Library is not remote-synced"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]+ (events/publish-event! :event/snippet-create {:object snippet :user-id (mt/user->id :rasta)})+ (is (not (t2/exists? :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet)))+ "Snippet should NOT be tracked when Library is not remote-synced"))))))++(deftest snippet-event-updates-sync-object-test+ (testing "Updating a snippet updates the RemoteSyncObject entry when Library is synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]+ ;; Delete auto-created entry from snippet creation event and set up test state+ (t2/delete! :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))+ (mt/with-temp [:model/RemoteSyncObject _rso {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Test Snippet"+ :status "synced"+ :status_changed_at (t/offset-date-time)}]+ (events/publish-event! :event/snippet-update {:object snippet :user-id (mt/user->id :rasta)})+ (let [entry (t2/select-one :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet))]+ (is (= "update" (:status entry))+ "Snippet should have 'update' status after modification"))))))))++(deftest archived-snippet-marked-for-deletion-test+ (testing "Archiving a snippet marks it for deletion when Library is synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]+ ;; Delete auto-created entry from snippet creation event and set up test state+ (t2/delete! :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))+ (mt/with-temp [:model/RemoteSyncObject _rso {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Test Snippet"+ :status "synced"+ :status_changed_at (t/offset-date-time)}]+ (let [archived-snippet (assoc snippet :archived true)]+ (events/publish-event! :event/snippet-update {:object archived-snippet :user-id (mt/user->id :rasta)})+ (let [entry (t2/select-one :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet))]+ (is (= "delete" (:status entry))+ "Archived snippet should be marked for deletion")))))))))++;;; ------------------------------------------- Snippets-Namespace Collection Tests -------------------------------------------++(deftest snippets-namespace-collection-tracked-when-library-synced-test+ (testing "Snippets-namespace collection creates RemoteSyncObject entry when Library is synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:name "Snippets Collection" :namespace :snippets :location "/"}]+ (events/publish-event! :event/collection-create {:object coll :user-id (mt/user->id :rasta)})+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id (:id coll))+ "Snippets-namespace collection should be tracked when Library is synced"))))))++(deftest snippets-namespace-collection-ignored-when-library-not-synced-test+ (testing "Snippets-namespace collection is NOT tracked when Library is not synced"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection coll {:name "Snippets Collection" :namespace :snippets :location "/"}]+ (events/publish-event! :event/collection-create {:object coll :user-id (mt/user->id :rasta)})+ (is (not (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id (:id coll)))+ "Snippets-namespace collection should NOT be tracked when Library is not synced"))))))++;;; ------------------------------------------- Sync Tracking Enable/Disable Tests -------------------------------------------++(deftest enable-snippet-sync-marks-all-snippets-test+ (testing "Enabling snippet sync (via Library becoming synced) marks all existing snippets for initial sync"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Snippets Collection" :namespace :snippets}+ :model/NativeQuerySnippet snippet {:name "Existing Snippet" :content "SELECT 1" :collection_id coll-id}]+ (is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))+ "Should have no snippet tracking entries initially")+ ;; Enable snippet sync+ (rs-events/sync-snippet-tracking! true)+ ;; Verify tracking entries created+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "Collection"+ :model_id coll-id+ :status "create")+ "Snippets-namespace collection should be marked for initial sync")+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :status "create")+ "Snippet should be marked for initial sync"))))))++(deftest disable-snippet-sync-removes-all-tracking-test+ (testing "Disabling snippet sync removes all snippet-related tracking entries"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Snippets Collection" :namespace :snippets}+ :model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1" :collection_id coll-id}+ :model/RemoteSyncObject _rso1 {:model_type "Collection"+ :model_id coll-id+ :model_name "Snippets Collection"+ :status "synced"+ :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Test Snippet"+ :model_collection_id coll-id+ :status "synced"+ :status_changed_at (t/offset-date-time)}]+ (is (= 2 (t2/count :model/RemoteSyncObject :model_type [:in ["Collection" "NativeQuerySnippet"]]+ :model_id [:in [coll-id (:id snippet)]]))+ "Should have 2 tracking entries")+ ;; Disable snippet sync+ (rs-events/sync-snippet-tracking! false)+ ;; Verify tracking entries removed+ (is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))+ "Snippet tracking entries should be removed")+ (is (zero? (t2/count :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))+ "Snippets-namespace collection tracking entry should be removed"))))))++;;; ------------------------------------------- Dirty Check Tests -------------------------------------------++(deftest snippets-included-in-dirty-check-when-library-synced-test+ (testing "Snippets are included in dirty check when Library is synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Dirty Snippet" :content "SELECT 1"}]+ ;; Delete auto-created entry and insert with specific status+ (t2/delete! :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id (:id snippet))+ (mt/with-temp [:model/RemoteSyncObject _rso {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Dirty Snippet"+ :status "update"+ :status_changed_at (t/offset-date-time)}]+ (is (sync-object/dirty?)+ "Should report dirty state when snippet has changes")))))))++(deftest snippets-excluded-from-dirty-check-when-library-not-synced-test+ (testing "Snippets are excluded from dirty check when Library is not synced"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Dirty Snippet" :content "SELECT 1"}+ :model/RemoteSyncObject _rso {:model_type "NativeQuerySnippet"+ :model_id (:id snippet)+ :model_name "Dirty Snippet"+ :status "update"+ :status_changed_at (t/offset-date-time)}]+ (is (not (sync-object/dirty?))+ "Should NOT report dirty state when Library is not synced"))))))++;;; ------------------------------------------- Library Sync Status Change Tests -------------------------------------------++(deftest library-sync-status-change-triggers-snippet-tracking-test+ (testing "When Library collection's is_remote_synced changes, snippet tracking is updated"+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "Test Snippet" :content "SELECT 1"}]+ ;; Ensure library exists and is not synced initially+ (when-let [library (collection/library-collection)]+ (t2/update! :model/Collection (:id library) {:is_remote_synced false}))+ (t2/delete! :model/RemoteSyncObject :model_type "NativeQuerySnippet")+ (is (zero? (t2/count :model/RemoteSyncObject :model_type "NativeQuerySnippet"))+ "Should have no snippet tracking initially")+ ;; Enable Library sync and verify snippets get tracked+ (when-let [library (collection/library-collection)]+ (t2/update! :model/Collection (:id library) {:is_remote_synced true})+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id (:id library))+ :user-id (mt/user->id :rasta)})+ (is (t2/exists? :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet))+ "Snippet should be tracked after Library becomes synced")+ ;; Disable Library sync and verify snippets get untracked+ (t2/update! :model/Collection (:id library) {:is_remote_synced false})+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id (:id library))+ :user-id (mt/user->id :rasta)})+ (is (not (t2/exists? :model/RemoteSyncObject+ :model_type "NativeQuerySnippet"+ :model_id (:id snippet)))+ "Snippet should be untracked after Library becomes un-synced"))))))++;;; ------------------------------------------- Helper Function Tests -------------------------------------------++(deftest library-is-remote-synced-helper-test+ (testing "library-is-remote-synced? returns correct value"+ (with-library-synced+ (is (true? (settings/library-is-remote-synced?))+ "Should return true when Library is synced"))+ (with-library-not-synced+ (is (false? (settings/library-is-remote-synced?))+ "Should return false when Library is not synced"))))++;;; ------------------------------------------- Import/Export Tests -------------------------------------------++(defn- generate-snippets-namespace-collection-yaml+ "Generates YAML content for a snippets-namespace collection."+ [entity-id name]+ (format "name: %s+description: null+entity_id: %s+slug: %s+created_at: '2024-08-28T09:46:18.671622Z'+archived: false+type: null+parent_id: null+personal_owner_id: null+namespace: snippets+authority_level: null+serdes/meta:+- id: %s+ label: %s+ model: Collection+archive_operation_id: null+archived_directly: null+is_sample: false+"+ name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))++(defn- generate-snippet-yaml+ "Generates YAML content for a NativeQuerySnippet."+ [entity-id name content & {:keys [collection-id]}]+ (format "name: %s+description: null+entity_id: %s+content: '%s'+archived: false+template_tags: null+created_at: '2024-08-28T09:46:18.671622Z'+creator_id: [email protected]+collection_id: %s+serdes/meta:+- id: %s+ label: %s+ model: NativeQuerySnippet+"+ name entity-id content (or collection-id "null")+ entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))++(deftest export-includes-snippets-when-library-synced-test+ (testing "Export includes snippets when Library is synced"+ (with-library-synced+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-enabled true]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id}+ {:name "Snippets Collection"+ :namespace :snippets+ :entity_id "snippets-coll-xxxxx"+ :location "/"}+ :model/NativeQuerySnippet {snippet-id :id snippet-eid :entity_id}+ {:name "Test Snippet"+ :content "SELECT 1"+ :collection_id coll-id+ :entity_id "test-snippet-xxxxxxxx"}+ :model/RemoteSyncObject _rso1 {:model_type "Collection"+ :model_id coll-id+ :model_name "Snippets Collection"+ :status "create"+ :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "NativeQuerySnippet"+ :model_id snippet-id+ :model_name "Test Snippet"+ :model_collection_id coll-id+ :status "create"+ :status_changed_at (t/offset-date-time)}]+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (some #(str/includes? % coll-eid) (keys files-after-export))+ "Export should include the snippets-namespace collection")+ (is (some #(str/includes? % snippet-eid) (keys files-after-export))+ "Export should include the snippet"))))))))))++(deftest import-snippets-from-yaml-test+ (testing "Import brings in snippets from YAML files"+ (mt/with-model-cleanup [:model/NativeQuerySnippet :model/Collection :model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (let [coll-entity-id "snippets-coll-importx"+ snippet-entity-id "test-snippet-importxx"+ test-files {"main" {(str "collections/" coll-entity-id "_snippets_collection/" coll-entity-id "_snippets_collection.yaml")+ (generate-snippets-namespace-collection-yaml coll-entity-id "Snippets Collection")+ (str "snippets/" coll-entity-id "_snippets_collection/" snippet-entity-id "_test_snippet.yaml")+ (generate-snippet-yaml snippet-entity-id "Test Snippet" "SELECT 42" :collection-id coll-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (t2/exists? :model/Collection :entity_id coll-entity-id :namespace "snippets")+ "Snippets-namespace collection should be imported")+ (is (t2/exists? :model/NativeQuerySnippet :entity_id snippet-entity-id)+ "Snippet should be imported")+ (when-let [snippet (t2/select-one :model/NativeQuerySnippet :entity_id snippet-entity-id)]+ (is (= "Test Snippet" (:name snippet)))+ (is (= "SELECT 42" (:content snippet)))))))))))++(deftest all-syncable-collection-ids-includes-all-namespace-types-test+ (testing "Snippet collections are included when Library is synced"+ (with-library-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection {snippet-coll-id :id} {:name "Snippets Collection"+ :namespace :snippets+ :location "/"}]+ (is (contains? (set (spec/all-syncable-collection-ids)) snippet-coll-id))))))+ (testing "Snippet collections are NOT included when Library is not synced"+ (with-library-not-synced+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection {snippet-coll-id :id} {:name "Snippets Collection"+ :namespace :snippets+ :location "/"}]+ (is (not (contains? (set (spec/all-syncable-collection-ids)) snippet-coll-id)))))))+ (testing "Transforms collections are included when setting is enabled"+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-temp [:model/Collection {transforms-coll-id :id} {:name "Transforms Collection"+ :namespace :transforms+ :location "/"}]+ (is (contains? (set (spec/all-syncable-collection-ids)) transforms-coll-id)))))+ (testing "Transforms collections are NOT included when setting is disabled"+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-temp [:model/Collection {transforms-coll-id :id} {:name "Transforms Collection"+ :namespace :transforms+ :location "/"}]+ (is (not (contains? (set (spec/all-syncable-collection-ids)) transforms-coll-id))))))+ (testing "Regular remote-synced collections are always included"+ (mt/with-temporary-setting-values [remote-sync-enabled true]+ (mt/with-temp [:model/Collection {synced-coll-id :id} {:name "Synced Collection"+ :is_remote_synced true+ :location "/"}]+ (is (contains? (set (spec/all-syncable-collection-ids)) synced-coll-id))))))++(deftest export-deletes-archived-snippet-files-test+ (testing "export! deletes files from source for snippets with 'delete' status (archived snippets)"+ (with-library-synced+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (mt/with-temporary-setting-values [remote-sync-type :read-write+ remote-sync-enabled true]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})+ ;; Use valid 21-char NanoID format entity_ids+ coll-entity-id (u/generate-nano-id)+ snippet-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id}+ {:name "Snippets Collection"+ :namespace :snippets+ :entity_id coll-entity-id+ :location "/"}+ :model/NativeQuerySnippet {snippet-id :id snippet-eid :entity_id}+ {:name "Archived Snippet"+ :content "SELECT 1"+ :collection_id coll-id+ :entity_id snippet-entity-id+ :archived true}+ :model/RemoteSyncObject _rso1 {:model_type "Collection"+ :model_id coll-id+ :model_name "Snippets Collection"+ :status "synced"+ :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "NativeQuerySnippet"+ :model_id snippet-id+ :model_name "Archived Snippet"+ :model_collection_id coll-id+ :status "delete"+ :status_changed_at (t/offset-date-time)}]+ (let [removal-paths (spec/build-all-removal-paths)+ snippet-removal-path (first (filter #(str/includes? % snippet-eid) removal-paths))+ _ (is (some? snippet-removal-path) "Should have a removal path for the archived snippet")+ initial-files {"main" {(str snippet-removal-path ".yaml")+ (generate-snippet-yaml snippet-eid "Archived Snippet" "SELECT 1" :collection-id coll-eid)}}+ mock-source (test-helpers/create-mock-source :initial-files initial-files)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (not (some #(str/includes? % snippet-eid) (keys files-after-export)))+ "Archived snippet file should be deleted after export")))+ (testing "RemoteSyncObject entry is updated to synced after export"+ (is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "NativeQuerySnippet" :model_id snippet-id)))+ "RemoteSyncObject entry for archived snippet should have synced status")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.cljindex 4d0d719ed15..a21369d3e0a 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj@@ -39,7 +39,7 @@(deftest all-specs-have-valid-eligibility-test(testing "Every spec has a valid eligibility type"- (let [valid-eligibility-types #{:collection :published-table :parent-table :setting}]+ (let [valid-eligibility-types #{:collection :published-table :parent-table :setting :library-synced}](doseq [[model-key spec] spec/remote-sync-specs](testing (str "Spec for " model-key)(is (contains? valid-eligibility-types (get-in spec [:eligibility :type]))@@ -81,7 +81,7 @@(deftest all-specs-have-valid-export-path-test(testing "Every spec has valid export-path configuration"(let [valid-path-types #{:collection-entity :table-path :field-path- :segment-path :measure-path :transform-path :transform-tag-path}]+ :segment-path :measure-path :transform-path :transform-tag-path :snippet-path}](doseq [[model-key spec] spec/remote-sync-specs](testing (str "Spec for " model-key)(is (contains? valid-path-types (get-in spec [:export-path :type]))@@ -151,7 +151,10 @@(testing "excluded-model-types when transforms enabled"(mt/with-temporary-setting-values [remote-sync-transforms true](let [excluded (spec/excluded-model-types)]- (is (empty? excluded))))))+ ;; NativeQuerySnippet is still excluded because Library isn't remote-synced+ (is (not (contains? excluded "Transform")))+ (is (not (contains? excluded "TransformTag")))+ (is (contains? excluded "NativeQuerySnippet"))))))(deftest spec-enabled?-test(testing "spec-enabled? with always-enabled spec"@@ -243,7 +246,7 @@(spec/select-fields-for-sync "Card")))(is (= [:name :collection_id](spec/select-fields-for-sync "Dashboard")))- (is (= [:name :id]+ (is (= [:name :collection_id](spec/select-fields-for-sync "NativeQuerySnippet"))))(testing "select-fields-for-sync returns default for unknown type"diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex f7cefdf5e1a..d63dd63c754 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -8,6 +8,7 @@[metabase-enterprise.remote-sync.models.remote-sync-object :as sync-object][metabase-enterprise.remote-sync.settings :as settings][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.remote-sync.spec :as spec][metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.collections.models.collection :as collection][metabase.events.core :as events]@@ -56,14 +57,13 @@(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true](mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}- :model/Transform transform {:name "Test Transform" :collection_id coll-id}]- (t2/insert! :model/RemoteSyncObject- {:model_type "Transform"- :model_id (:id transform)- :model_name "Test Transform"- :model_collection_id coll-id- :status "synced"- :status_changed_at (t/offset-date-time)})+ :model/Transform transform {:name "Test Transform" :collection_id coll-id}+ :model/RemoteSyncObject _rso {:model_type "Transform"+ :model_id (:id transform)+ :model_name "Test Transform"+ :model_collection_id coll-id+ :status "synced"+ :status_changed_at (t/offset-date-time)}](events/publish-event! :event/transform-update {:object transform})(let [entry (t2/select-one :model/RemoteSyncObject:model_type "Transform"@@ -155,12 +155,9 @@:model/Transform transform {:name "Test Transform" :collection_id coll-id}:model/TransformTag tag {:name "Test Tag"}](mt/with-temporary-setting-values [remote-sync-transforms true]- (is (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id coll-id)- "Collection tracking entry should exist")- (is (t2/exists? :model/RemoteSyncObject :model_type "Transform" :model_id (:id transform))- "Transform tracking entry should exist")- (is (t2/exists? :model/RemoteSyncObject :model_type "TransformTag" :model_id (:id tag))- "TransformTag tracking entry should exist")+ (is (= 3 (t2/count :model/RemoteSyncObject :model_type [:in ["Collection" "Transform" "TransformTag"]]+ :model_id [:in [coll-id (:id transform) (:id tag)]]))+ "Should have 3 tracking entries")(settings/sync-transform-tracking! false)(is (zero? (t2/count :model/RemoteSyncObject :model_type [:in ["Transform" "TransformTag"]]))"Transform and TransformTag tracking entries should be removed")@@ -173,13 +170,12 @@(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true](mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}- :model/Transform transform {:name "Dirty Transform" :collection_id coll-id}]- (t2/insert! :model/RemoteSyncObject- {:model_type "Transform"- :model_id (:id transform)- :model_name "Dirty Transform"- :status "update"- :status_changed_at (t/offset-date-time)})+ :model/Transform transform {:name "Dirty Transform" :collection_id coll-id}+ :model/RemoteSyncObject _rso {:model_type "Transform"+ :model_id (:id transform)+ :model_name "Dirty Transform"+ :status "update"+ :status_changed_at (t/offset-date-time)}](is (sync-object/dirty?)"Should report dirty state when transform has changes"))))))@@ -189,13 +185,12 @@(mt/with-temporary-setting-values [remote-sync-transforms falseremote-sync-enabled true](mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}- :model/Transform transform {:name "Dirty Transform" :collection_id coll-id}]- (t2/insert! :model/RemoteSyncObject- {:model_type "Transform"- :model_id (:id transform)- :model_name "Dirty Transform"- :status "update"- :status_changed_at (t/offset-date-time)})+ :model/Transform transform {:name "Dirty Transform" :collection_id coll-id}+ :model/RemoteSyncObject _rso {:model_type "Transform"+ :model_id (:id transform)+ :model_name "Dirty Transform"+ :status "update"+ :status_changed_at (t/offset-date-time)}](is (not (sync-object/dirty?))"Should NOT report dirty state when transform sync is disabled"))))))@@ -204,13 +199,12 @@(mt/with-premium-features #{:transforms}(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true]- (mt/with-temp [:model/TransformTag tag {:name "Dirty Tag"}]- (t2/insert! :model/RemoteSyncObject- {:model_type "TransformTag"- :model_id (:id tag)- :model_name "Dirty Tag"- :status "update"- :status_changed_at (t/offset-date-time)})+ (mt/with-temp [:model/TransformTag tag {:name "Dirty Tag"}+ :model/RemoteSyncObject _rso {:model_type "TransformTag"+ :model_id (:id tag)+ :model_name "Dirty Tag"+ :status "update"+ :status_changed_at (t/offset-date-time)}](is (sync-object/dirty?)"Should report dirty state when transform tag has changes"))))))@@ -219,13 +213,12 @@(mt/with-premium-features #{:transforms}(mt/with-temporary-setting-values [remote-sync-transforms falseremote-sync-enabled true]- (mt/with-temp [:model/TransformTag tag {:name "Dirty Tag"}]- (t2/insert! :model/RemoteSyncObject- {:model_type "TransformTag"- :model_id (:id tag)- :model_name "Dirty Tag"- :status "update"- :status_changed_at (t/offset-date-time)})+ (mt/with-temp [:model/TransformTag tag {:name "Dirty Tag"}+ :model/RemoteSyncObject _rso {:model_type "TransformTag"+ :model_id (:id tag)+ :model_name "Dirty Tag"+ :status "update"+ :status_changed_at (t/offset-date-time)}](is (not (sync-object/dirty?))"Should NOT report dirty state when transform sync is disabled"))))))@@ -235,28 +228,28 @@(mt/with-temporary-setting-values [remote-sync-type :read-writeremote-sync-transforms trueremote-sync-enabled true]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxx" :location "/"}- :model/Transform transform {:name "Export Transform" :collection_id coll-id}]- (let [saved-coll (t2/select-one :model/Collection :id coll-id)]- (is (= :transforms (:namespace saved-coll))- (str "Collection should have transforms namespace, got: " (:namespace saved-coll)))- (is (= "/" (:location saved-coll))- (str "Collection should have root location, got: " (:location saved-coll))))- (let [found-colls (t2/select-fn-set :entity_id :model/Collection :namespace (name collection/transforms-ns))]- (is (contains? found-colls coll-eid)- (str "Query should find the collection. Found: " found-colls ", expected to contain: " coll-eid)))- (t2/insert! :model/RemoteSyncObject- [{:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}- {:model_type "Transform" :model_id (:id transform) :model_name "Export Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}])- (let [mock-source (test-helpers/create-mock-source)- result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]- (is (= :success (:status result))- (str "Export should succeed. Result: " result))- (let [files-after-export (get @(:files-atom mock-source) "main")]- (is (some #(str/includes? % coll-eid)- (keys files-after-export))- "Export should include the transforms-namespace collection itself")))))))))+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxx" :location "/"}+ :model/Transform transform {:name "Export Transform" :collection_id coll-id}+ :model/RemoteSyncObject _rso1 {:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "Transform" :model_id (:id transform) :model_name "Export Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}]+ (let [saved-coll (t2/select-one :model/Collection :id coll-id)]+ (is (= :transforms (:namespace saved-coll))+ (str "Collection should have transforms namespace, got: " (:namespace saved-coll)))+ (is (= "/" (:location saved-coll))+ (str "Collection should have root location, got: " (:location saved-coll))))+ (let [found-colls (t2/select-fn-set :entity_id :model/Collection :namespace (name collection/transforms-ns))]+ (is (contains? found-colls coll-eid)+ (str "Query should find the collection. Found: " found-colls ", expected to contain: " coll-eid)))+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (some #(str/includes? % coll-eid)+ (keys files-after-export))+ "Export should include the transforms-namespace collection itself"))))))))))(deftest export-excludes-transforms-when-setting-disabled-test(testing "Export excludes transforms when setting is disabled"@@ -264,25 +257,25 @@(mt/with-temporary-setting-values [remote-sync-type :read-writeremote-sync-transforms falseremote-sync-enabled true]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxx" :location "/"}]- (let [saved-coll (t2/select-one :model/Collection :id rs-coll-id)]- (is (true? (:is_remote_synced saved-coll))- (str "Collection should have is_remote_synced true, got: " (:is_remote_synced saved-coll)))- (is (= "/" (:location saved-coll))- (str "Collection should have root location, got: " (:location saved-coll))))- (let [found-colls (t2/select-fn-set :entity_id :model/Collection :is_remote_synced true :location "/")]- (is (contains? found-colls rs-coll-eid)- (str "Query should find the collection. Found: " found-colls ", expected to contain: " rs-coll-eid)))- (t2/insert! :model/RemoteSyncObject- {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)})- (let [mock-source (test-helpers/create-mock-source)- result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]- (is (= :success (:status result))- (str "Export should succeed. Result: " result))- (let [files-after-export (get @(:files-atom mock-source) "main")]- (is (not (some #(str/includes? % "transforms/") (keys files-after-export)))- "Export should NOT include transform files when setting is disabled")))))))))+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxx" :location "/"}+ :model/RemoteSyncObject _rso {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)}]+ (let [saved-coll (t2/select-one :model/Collection :id rs-coll-id)]+ (is (true? (:is_remote_synced saved-coll))+ (str "Collection should have is_remote_synced true, got: " (:is_remote_synced saved-coll)))+ (is (= "/" (:location saved-coll))+ (str "Collection should have root location, got: " (:location saved-coll))))+ (let [found-colls (t2/select-fn-set :entity_id :model/Collection :is_remote_synced true :location "/")]+ (is (contains? found-colls rs-coll-eid)+ (str "Query should find the collection. Found: " found-colls ", expected to contain: " rs-coll-eid)))+ (let [mock-source (test-helpers/create-mock-source)+ result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result))+ (str "Export should succeed. Result: " result))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (not (some #(str/includes? % "transforms/") (keys files-after-export)))+ "Export should NOT include transform files when setting is disabled"))))))))))(defn- generate-transform-yaml"Generates YAML content for a transform."@@ -362,56 +355,169 @@ is_sample: false(mt/with-temporary-setting-values [remote-sync-type :read-writeremote-sync-transforms trueremote-sync-enabled true]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id}- {:name "Transforms Collection"- :namespace collection/transforms-ns- :entity_id "arch-transforms-collx"- :location "/"}- :model/Transform {transform-id :id transform-eid :entity_id}- {:name "Child Transform"- :collection_id coll-id- :entity_id "child-transform-xxxxx"}- :model/Transform {root-transform-id :id root-transform-eid :entity_id}- {:name "Root Transform"- :collection_id nil- :entity_id "root-transform-xxxxxx"}]- (t2/insert! :model/RemoteSyncObject- [{:model_type "Collection"- :model_id coll-id- :model_name "Transforms Collection"- :status "synced"- :status_changed_at (t/offset-date-time)}- {:model_type "Transform"- :model_id transform-id- :model_name "Child Transform"- :model_collection_id coll-id- :status "synced"- :status_changed_at (t/offset-date-time)}- {:model_type "Transform"- :model_id root-transform-id- :model_name "Root Transform"- :model_collection_id nil- :status "create"- :status_changed_at (t/offset-date-time)}])- (let [initial-files {"main" {(str "collections/" coll-eid "_transforms_collection/" coll-eid "_transforms_collection.yaml")- (generate-transforms-namespace-collection-yaml coll-eid "Transforms Collection")- (str "collections/" coll-eid "_transforms_collection/transforms/" transform-eid "_child_transform.yaml")- (generate-transform-yaml transform-eid "Child Transform")}}- mock-source (test-helpers/create-mock-source :initial-files initial-files)]- (is (some #(str/includes? % coll-eid) (keys (get @(:files-atom mock-source) "main"))))- (is (some #(str/includes? % transform-eid) (keys (get @(:files-atom mock-source) "main"))))- (t2/update! :model/Collection coll-id {:archived true})- (events/publish-event! :event/collection-update- {:object (t2/select-one :model/Collection :id coll-id)- :user-id (mt/user->id :rasta)})- (is (= "delete" (:status (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))))- (is (= "delete" (:status (t2/select-one :model/RemoteSyncObject :model_type "Transform" :model_id transform-id))))- (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]- (is (= :success (:status result)))- (let [files-after-export (get @(:files-atom mock-source) "main")]- (is (not (some #(str/includes? % coll-eid) (keys files-after-export))))- (is (not (some #(str/includes? % transform-eid) (keys files-after-export))))- ;; Root transform should still be exported- (is (some #(str/includes? % root-transform-eid) (keys files-after-export))- "Root transform should be exported"))))))))))+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id}+ {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id "arch-transforms-collx"+ :location "/"}+ :model/Transform {transform-id :id transform-eid :entity_id}+ {:name "Child Transform"+ :collection_id coll-id+ :entity_id "child-transform-xxxxx"}+ :model/Transform {root-transform-id :id root-transform-eid :entity_id}+ {:name "Root Transform"+ :collection_id nil+ :entity_id "root-transform-xxxxxx"}+ :model/RemoteSyncObject _rso1 {:model_type "Collection"+ :model_id coll-id+ :model_name "Transforms Collection"+ :status "synced"+ :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso2 {:model_type "Transform"+ :model_id transform-id+ :model_name "Child Transform"+ :model_collection_id coll-id+ :status "synced"+ :status_changed_at (t/offset-date-time)}+ :model/RemoteSyncObject _rso3 {:model_type "Transform"+ :model_id root-transform-id+ :model_name "Root Transform"+ :model_collection_id nil+ :status "create"+ :status_changed_at (t/offset-date-time)}]+ (let [initial-files {"main" {(str "collections/" coll-eid "_transforms_collection/" coll-eid "_transforms_collection.yaml")+ (generate-transforms-namespace-collection-yaml coll-eid "Transforms Collection")+ (str "collections/" coll-eid "_transforms_collection/transforms/" transform-eid "_child_transform.yaml")+ (generate-transform-yaml transform-eid "Child Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files initial-files)]+ (is (some #(str/includes? % coll-eid) (keys (get @(:files-atom mock-source) "main"))))+ (is (some #(str/includes? % transform-eid) (keys (get @(:files-atom mock-source) "main"))))+ (t2/update! :model/Collection coll-id {:archived true})+ (events/publish-event! :event/collection-update+ {:object (t2/select-one :model/Collection :id coll-id)+ :user-id (mt/user->id :rasta)})+ (is (= "delete" (:status (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id coll-id))))+ (let [result (impl/export! (source.p/snapshot mock-source) task-id "Test export")]+ (is (= :success (:status result)))+ (let [files-after-export (get @(:files-atom mock-source) "main")]+ (is (not (some #(str/includes? % coll-eid) (keys files-after-export))))+ (is (not (some #(str/includes? % transform-eid) (keys files-after-export))))+ ;; Root transform should still be exported+ (is (some #(str/includes? % root-transform-eid) (keys files-after-export))+ "Root transform should be exported")))))))))))++;;; ------------------------------------------- Collection Cleanup Tests -------------------------------------------++(deftest transforms-collections-included-in-all-syncable-collection-ids-test+ (testing "Transforms collections are included in all-syncable-collection-ids when setting is enabled"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-temp [:model/Collection {transforms-coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}]+ (is (contains? (set (spec/all-syncable-collection-ids)) transforms-coll-id)+ "Transforms-namespace collection should be included when setting is enabled")))))++ (testing "Transforms collections are NOT included when setting is disabled"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-temp [:model/Collection {transforms-coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}]+ (is (not (contains? (set (spec/all-syncable-collection-ids)) transforms-coll-id))+ "Transforms-namespace collection should NOT be included when setting is disabled"))))))++(deftest import-removes-empty-transforms-collection-not-on-remote-test+ (testing "Import removes empty transforms-namespace collections that don't exist on the remote"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-coll-entity-id (u/generate-nano-id)+ remote-coll-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {local-coll-id :id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id local-coll-entity-id+ :location "/"}]+ (is (t2/exists? :model/Collection :id local-coll-id))+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (not (t2/exists? :model/Collection :id local-coll-id))+ "Local transforms collection should be deleted after import since it wasn't on remote")+ (is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")+ "Remote transforms collection should be imported")))))))))++(deftest import-removes-transform-not-on-remote-test+ (testing "Import removes transforms that don't exist on the remote but keeps the collection"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ coll-entity-id (u/generate-nano-id)+ local-transform-entity-id (u/generate-nano-id)+ remote-transform-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id coll-entity-id+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}]+ (is (t2/exists? :model/Collection :id coll-id))+ (is (t2/exists? :model/Transform :id local-transform-id))+ (let [test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")+ (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")+ (str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")+ (generate-transform-yaml remote-transform-entity-id "Remote Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (t2/exists? :model/Collection :id coll-id)+ "Transforms collection should still exist")+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be deleted after import since it wasn't on remote")+ (is (t2/exists? :model/Transform :entity_id remote-transform-entity-id)+ "Remote transform should be imported")))))))))++(deftest import-removes-transforms-and-collection-not-on-remote-test+ (testing "Import removes transforms and their collection when neither exist on remote"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-coll-entity-id (u/generate-nano-id)+ local-transform-entity-id (u/generate-nano-id)+ remote-coll-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {local-coll-id :id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id local-coll-entity-id+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id local-coll-id}]+ (is (t2/exists? :model/Collection :id local-coll-id))+ (is (t2/exists? :model/Transform :id local-transform-id))+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_transforms/" remote-coll-entity-id "_remote_transforms.yaml")+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be deleted after import")+ (is (not (t2/exists? :model/Collection :id local-coll-id))+ "Local transforms collection should be deleted after import")+ (is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")+ "Remote transforms collection should be imported")))))))))diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/snippets/pages/NewSnippetPage/NewSnippetPage.tsx b/enterprise/frontend/src/metabase-enterprise/data-studio/snippets/pages/NewSnippetPage/NewSnippetPage.tsxindex 167d1b302e8..288cc43b11b 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/snippets/pages/NewSnippetPage/NewSnippetPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/snippets/pages/NewSnippetPage/NewSnippetPage.tsx@@ -1,5 +1,5 @@import { sql } from "@codemirror/lang-sql";-import { useMemo, useState } from "react";+import { useEffect, useMemo, useState } from "react";import type { Route } from "react-router";import { push } from "react-router-redux";import { t } from "ttag";@@ -17,7 +17,10 @@ import { PLUGIN_SNIPPET_FOLDERS } from "metabase/plugins";import { Card, Flex, Stack } from "metabase/ui";import { DataStudioBreadcrumbs } from "metabase-enterprise/data-studio/common/components/DataStudioBreadcrumbs";import { PageContainer } from "metabase-enterprise/data-studio/common/components/PageContainer";-import type { RegularCollectionId } from "metabase-types/api";+import type {+ NativeQuerySnippet,+ RegularCollectionId,+} from "metabase-types/api";import {PaneHeader,@@ -40,28 +43,38 @@ export function NewSnippetPage({ route }: NewSnippetPageProps) {const [description, setDescription] = useState("");const [content, setContent] = useState("");const [isCollectionPickerOpen, setIsCollectionPickerOpen] = useState(false);+ const [savedSnippet, setSavedSnippet] = useState<NativeQuerySnippet | null>(+ null,+ );const [createSnippet, { isLoading: isSaving }] = useCreateSnippetMutation();const isValid = name.length > 0 && content.length > 0;const handleCreateSnippet = async (collectionId: RegularCollectionId | null,) => {- try {- const result = await createSnippet({- name,- content,- description: description.trim().length > 0 ? description.trim() : null,- collection_id: collectionId,- }).unwrap();- dispatch(push(Urls.dataStudioSnippet(result.id)));- } catch (error) {+ const { data: snippet, error } = await createSnippet({+ name,+ content,+ description: description.trim().length > 0 ? description.trim() : null,+ collection_id: collectionId,+ });++ if (error) {sendToast({message: getErrorMessage(error, t`Failed to create snippet`),icon: "warning",});+ } else if (snippet) {+ setSavedSnippet(snippet);}};+ useEffect(() => {+ if (savedSnippet) {+ dispatch(push(Urls.dataStudioSnippet(savedSnippet.id)));+ }+ }, [savedSnippet, dispatch]);+const handleSave = async () => {if (!PLUGIN_SNIPPET_FOLDERS.isEnabled) {await handleCreateSnippet(null);@@ -147,7 +160,10 @@ export function NewSnippetPage({ route }: NewSnippetPageProps) {</Stack></Flex></PageContainer>- <LeaveRouteConfirmModal route={route} isEnabled={!isSaving} />+ <LeaveRouteConfirmModal+ route={route}+ isEnabled={!savedSnippet && !isSaving}+ /><PLUGIN_SNIPPET_FOLDERS.CollectionPickerModalisOpen={isCollectionPickerOpen}onSelect={handleCollectionSelected}diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsxindex 7e546f8299b..add064b1669 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx@@ -59,9 +59,32 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {"include-library": true,});+ // Fetch snippets namespace collections separately+ const { data: snippetCollectionTree = [] } = useListCollectionsTreeQuery({+ namespace: "snippets",+ });++ // Build a set of snippet collection IDs for icon selection+ const snippetCollectionIds = useMemo(() => {+ const ids = new Set<number>();+ const collectIds = (collections: typeof snippetCollectionTree) => {+ for (const collection of collections) {+ if (typeof collection.id === "number") {+ ids.add(collection.id);+ }+ if (collection.children) {+ collectIds(collection.children);+ }+ }+ };+ collectIds(snippetCollectionTree);+ return ids;+ }, [snippetCollectionTree]);+const collectionMap = useMemo(() => {- return buildCollectionMap(collectionTree);- }, [collectionTree]);+ // Merge regular collections with snippet collections+ return buildCollectionMap([...collectionTree, ...snippetCollectionTree]);+ }, [collectionTree, snippetCollectionTree]);const hasRemovals = useMemo(() => {return (@@ -134,15 +157,19 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {(a, b) => a.tableName.localeCompare(b.tableName),);+ const numericCollectionId = Number(collectionId) || undefined;return {pathSegments: getCollectionPathSegments(- Number(collectionId) || undefined,+ numericCollectionId,collectionMap,),- collectionId: Number(collectionId) || undefined,+ collectionId: numericCollectionId,collectionEntity,tableGroups,items: sortByStatus(otherItems),+ isSnippetCollection:+ numericCollectionId !== undefined &&+ snippetCollectionIds.has(numericCollectionId),};}).sort((a, b) =>@@ -151,7 +178,7 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {.join(" / ").localeCompare(b.pathSegments.map((s) => s.name).join(" / ")),);- }, [entities, collectionMap]);+ }, [entities, collectionMap, snippetCollectionIds]);return (<Box>@@ -188,7 +215,11 @@ export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {bdrs="md"><Icon- name="synced_collection"+ name={+ group.isSnippetCollection+ ? "snippet"+ : "synced_collection"+ }size={16}c="text-secondary"/>diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsxindex 9775bffac42..4757e8bfd31 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx@@ -343,4 +343,62 @@ describe("AllChangesView", () => {expect(screen.getByText("Total Users")).toBeInTheDocument();});});++ describe("snippet collections", () => {+ it("should display snippets in snippet collections with hierarchy", async () => {+ const snippetCollection = createMockCollection({+ id: 50,+ name: "My Snippets",+ namespace: "snippets",+ effective_ancestors: [],+ });+ const snippetEntity = createMockRemoteSyncEntity({+ id: 300,+ name: "SELECT Query",+ model: "nativequerysnippet",+ collection_id: 50,+ sync_status: "create",+ });++ setup({+ entities: [snippetEntity],+ collections: [snippetCollection],+ });++ expect(await screen.findByText("My Snippets")).toBeInTheDocument();+ expect(screen.getByText("SELECT Query")).toBeInTheDocument();+ });++ it("should display nested snippet collection hierarchy", async () => {+ const childSnippetCollection = createMockCollection({+ id: 51,+ name: "Child Snippet Folder",+ namespace: "snippets",+ });+ const parentSnippetCollection = createMockCollection({+ id: 50,+ name: "Parent Snippet Folder",+ namespace: "snippets",+ children: [childSnippetCollection],+ });+ const snippetEntity = createMockRemoteSyncEntity({+ id: 300,+ name: "SELECT Query",+ model: "nativequerysnippet",+ collection_id: 51,+ sync_status: "update",+ });++ setup({+ entities: [snippetEntity],+ collections: [parentSnippetCollection],+ });++ expect(+ await screen.findByText("Parent Snippet Folder"),+ ).toBeInTheDocument();+ expect(screen.getByText("Child Snippet Folder")).toBeInTheDocument();+ expect(screen.getByText("SELECT Query")).toBeInTheDocument();+ });+ });});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.tsindex fe72c300217..223e9c3f6dc 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.ts@@ -10,7 +10,7 @@ import { useRemoteSyncDirtyState } from "./use-remote-sync-dirty-state";export function useHasLibraryDirtyChanges(): boolean {const { isVisible: isGitSyncVisible } = useGitSyncVisible();- const { hasDirtyInCollectionTree, isDirty, hasRemovedItems } =+ const { dirty, hasDirtyInCollectionTree, isDirty, hasRemovedItems } =useRemoteSyncDirtyState();const { data: collections = [] } = useListCollectionsTreeQuery(@@ -22,6 +22,12 @@ export function useHasLibraryDirtyChanges(): boolean {{ skip: !isGitSyncVisible },);+ // Fetch snippets-namespace collections to check for dirty snippet collections+ const { data: snippetsCollections = [] } = useListCollectionsTreeQuery(+ { namespace: "snippets" },+ { skip: !isGitSyncVisible },+ );+return useMemo(() => {// Always show dirty if there are removed itemsif (hasRemovedItems) {@@ -32,6 +38,33 @@ export function useHasLibraryDirtyChanges(): boolean {return false;}+ // Check for dirty snippets or snippet collections+ const snippetsTree = buildCollectionTree(snippetsCollections);+ const snippetsCollectionIds = getAllDescendantIds(snippetsTree);+ const numericSnippetCollectionIds = new Set(+ [...snippetsCollectionIds].filter(+ (id): id is number => typeof id === "number",+ ),+ );++ const hasSnippetDirtyChanges = dirty.some((entity) => {+ if (entity.model === "nativequerysnippet") {+ return true;+ }+ if (+ entity.model === "collection" &&+ numericSnippetCollectionIds.has(entity.id)+ ) {+ return true;+ }+ return false;+ });++ if (hasSnippetDirtyChanges) {+ return true;+ }++ // Check for dirty items in the Library collection treeconst libraryCollection = collections.find(isLibraryCollection);if (!libraryCollection) {return false;@@ -49,5 +82,12 @@ export function useHasLibraryDirtyChanges(): boolean {);return hasDirtyInCollectionTree(numericIds);- }, [collections, isDirty, hasRemovedItems, hasDirtyInCollectionTree]);+ }, [+ collections,+ snippetsCollections,+ dirty,+ isDirty,+ hasRemovedItems,+ hasDirtyInCollectionTree,+ ]);}diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.tsindex 45621c08832..2c22256a452 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-library-dirty-changes.unit.spec.ts@@ -11,6 +11,7 @@ import type { Collection, RemoteSyncEntity } from "metabase-types/api";import {createMockCollection,createMockSettings,+ createMockSnippetsCollection,createMockTokenFeatures,createMockUser,} from "metabase-types/api/mocks";@@ -189,4 +190,140 @@ describe("useHasLibraryDirtyChanges", () => {expect(result.current).toBe(false);});});++ describe("snippet dirty state", () => {+ it("returns true when a dirty snippet exists", async () => {+ const { result } = setup({+ collections: [+ createLibraryCollection({ id: 1 }),+ createMockSnippetsCollection({ id: 100 }),+ ],+ dirty: [+ createMockDirtyEntity({+ id: 10,+ model: "nativequerysnippet",+ name: "My Snippet",+ collection_id: 100,+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(true);+ });+ });++ it("returns true when a dirty snippet exists in root (no collection)", async () => {+ const { result } = setup({+ collections: [createLibraryCollection({ id: 1 })],+ dirty: [+ createMockDirtyEntity({+ id: 10,+ model: "nativequerysnippet",+ name: "Root Snippet",+ collection_id: undefined,+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(true);+ });+ });++ it("returns true when a dirty collection in snippets namespace exists", async () => {+ const { result } = setup({+ collections: [+ createLibraryCollection({ id: 1 }),+ createMockSnippetsCollection({ id: 100 }),+ ],+ dirty: [+ createMockDirtyEntity({+ id: 100,+ model: "collection",+ name: "Snippets Folder",+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(true);+ });+ });++ it("returns false when dirty collection is not in snippets namespace", async () => {+ const { result } = setup({+ collections: [+ createLibraryCollection({ id: 1 }),+ createRegularCollection({ id: 2 }),+ createMockSnippetsCollection({ id: 100 }),+ ],+ dirty: [+ createMockDirtyEntity({+ id: 2,+ model: "collection",+ name: "Regular Collection",+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(false);+ });+ });++ it("returns false when dirty snippets exist but git sync is not visible", async () => {+ const { result } = setup({+ isGitSyncVisible: false,+ collections: [+ createLibraryCollection({ id: 1 }),+ createMockSnippetsCollection({ id: 100 }),+ ],+ dirty: [+ createMockDirtyEntity({+ id: 10,+ model: "nativequerysnippet",+ name: "My Snippet",+ collection_id: 100,+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(false);+ });+ });++ it("returns true when nested snippets collection is dirty", async () => {+ const parentSnippetsCollection = createMockSnippetsCollection({+ id: 100,+ name: "Parent Snippets Folder",+ });+ const childSnippetsCollection = createMockSnippetsCollection({+ id: 101,+ name: "Child Snippets Folder",+ });+ // Set up parent-child relationship+ parentSnippetsCollection.children = [childSnippetsCollection];++ const { result } = setup({+ collections: [+ createLibraryCollection({ id: 1 }),+ parentSnippetsCollection,+ childSnippetsCollection,+ ],+ dirty: [+ createMockDirtyEntity({+ id: 101,+ model: "collection",+ name: "Child Snippets Folder",+ }),+ ],+ });++ await waitFor(() => {+ expect(result.current).toBe(true);+ });+ });+ });});diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.tsindex 013ba2bb4f9..c8bdc3ad1a2 100644--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/middleware/model-configs.ts@@ -7,6 +7,7 @@ import { documentApi } from "metabase/api/document";import { fieldApi } from "metabase/api/field";import { measureApi } from "metabase/api/measure";import { segmentApi } from "metabase/api/segment";+import { snippetApi } from "metabase/api/snippet";import { tableApi as coreTableApi } from "metabase/api/table";import { timelineApi } from "metabase/api/timeline";import { timelineEventApi } from "metabase/api/timeline-event";@@ -181,6 +182,12 @@ export const MODEL_MUTATION_CONFIGS: ModelMutationConfig[] = [updateEndpoints: [measureApi.endpoints.updateMeasure.matchFulfilled],invalidation: { type: InvalidationType.Always },},+ {+ modelType: "snippet",+ createEndpoints: [snippetApi.endpoints.createSnippet.matchFulfilled],+ updateEndpoints: [snippetApi.endpoints.updateSnippet.matchFulfilled],+ invalidation: { type: InvalidationType.Always },+ },{modelType: "transform",createEndpoints: [transformApi.endpoints.createTransform.matchFulfilled],diff --git a/frontend/src/metabase-types/api/mocks/collection.ts b/frontend/src/metabase-types/api/mocks/collection.tsindex c7784181f4d..0176291bc55 100644--- a/frontend/src/metabase-types/api/mocks/collection.ts+++ b/frontend/src/metabase-types/api/mocks/collection.ts@@ -81,3 +81,13 @@ export const createMockTransformsCollection = (namespace: "transforms",...opts,});++export const createMockSnippetsCollection = (+ opts?: Partial<Collection>,+): Collection =>+ createMockCollection({+ id: 101,+ name: "Snippets",+ namespace: "snippets",+ ...opts,+ });diff --git a/frontend/src/metabase-types/api/remote-sync.ts b/frontend/src/metabase-types/api/remote-sync.tsindex 78e7bc8f361..1fe5eb0b0ce 100644--- a/frontend/src/metabase-types/api/remote-sync.ts+++ b/frontend/src/metabase-types/api/remote-sync.ts@@ -9,7 +9,7 @@ export type RemoteSyncEntityModel =| "dashboard"| "collection"| "document"- | "snippet"+ | "nativequerysnippet"| "table"| "field"| "segment"diff --git a/frontend/src/metabase/lib/icon.ts b/frontend/src/metabase/lib/icon.tsindex cd2d2d466d4..cf4dc1069f1 100644--- a/frontend/src/metabase/lib/icon.ts+++ b/frontend/src/metabase/lib/icon.ts@@ -21,7 +21,8 @@ export type IconModel =| "timeline"| "question"| "transform"- | "user";+ | "user"+ | "nativequerysnippet";export type ObjectWithModel = {id?: unknown;@@ -52,7 +53,8 @@ export const modelIconMap: Record<IconModel, IconName> = {card: "table2",segment: "segment",metric: "metric",- snippet: "unknown",+ snippet: "snippet",+ nativequerysnippet: "snippet",document: "document",timeline: "calendar",transform: "transform",diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex f7d3d3c0372..b5943d94b7e 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -137,14 +137,17 @@;;; ------------------------------------------------- Serialization ---------------------------------------------------(defmethod serdes/extract-query "NativeQuerySnippet" [_ {:keys [collection-set where]}]- ;; NativeQuerySnippets leave in their own special collections, so the logic is the following:+(defmethod serdes/extract-query "NativeQuerySnippet" [_ {:keys [collection-set where skip-archived]}]+ ;; NativeQuerySnippets live in their own special collections, so the logic is the following:;; - you either are exporting one of those;; - or it was requested as a dependency of some Card, so export it regardless of collection- (t2/reducible-select :model/NativeQuerySnippet (cond-> {:where [:or- [:in :collection_id (remove nil? collection-set)]- (when (some nil? collection-set)- [:= :collection_id nil])]}+ (t2/reducible-select :model/NativeQuerySnippet (cond-> {:where [:and+ (when skip-archived [:not :archived])+ [:or+ (when-let [collection-ids (not-empty (remove nil? collection-set))]+ [:in :collection_id collection-ids])+ (when (some nil? collection-set)+ [:= :collection_id nil])]]}where (sql.helpers/where :or where))))(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]diff --git a/test/metabase/collections/test_helpers.clj b/test/metabase/collections/test_helpers.cljdeleted file mode 100644index 7faeea4b436..00000000000--- a/test/metabase/collections/test_helpers.clj+++ /dev/null@@ -1,39 +0,0 @@-(ns metabase.collections.test-helpers- "Test utilities and macros for collection-related tests."- (:require- [metabase.collections.models.collection :as collection]- [toucan2.core :as t2]))--(defmacro without-library- "Macro that clears existing library collections, executes body, then cleans up any created library collections.-- This is useful for tests that need to work with library collections in isolation without- interference from any existing library collections in the test database.-- Example:- ```clojure- (without-library- (let [lib (collection/create-library-collection!)]- ...))- ```"- [& body]- `(do- ;; Clear existing library collections- (t2/update! (t2/table-name :model/Collection)- :type collection/library-collection-type- {:type nil})- (t2/update! (t2/table-name :model/Collection)- :type collection/library-data-collection-type- {:type nil})- (t2/update! (t2/table-name :model/Collection)- :type collection/library-metrics-collection-type- {:type nil})-- (try- ~@body- (finally- ;; Clean up any created library collections- (t2/delete! :model/Collection- :type [:in [collection/library-collection-type- collection/library-data-collection-type- collection/library-metrics-collection-type]])))))diff --git a/test/metabase/collections/test_utils.clj b/test/metabase/collections/test_utils.cljnew file mode 100644index 00000000000..a82a9a20c07--- /dev/null+++ b/test/metabase/collections/test_utils.clj@@ -0,0 +1,79 @@+(ns metabase.collections.test-utils+ "Test utilities and macros for collection-related tests."+ (:require+ [metabase.collections.models.collection :as collection]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(defmacro without-library+ "Macro that clears existing library collections, executes body, then cleans up any created library collections.++ This is useful for tests that need to work with library collections in isolation without+ interference from any existing library collections in the test database.++ Example:+ ```clojure+ (without-library+ (let [lib (collection/create-library-collection!)]+ ...))+ ```"+ [& body]+ `(do+ ;; Clear existing library collections+ (t2/update! (t2/table-name :model/Collection)+ :type collection/library-collection-type+ {:type nil})+ (t2/update! (t2/table-name :model/Collection)+ :type collection/library-data-collection-type+ {:type nil})+ (t2/update! (t2/table-name :model/Collection)+ :type collection/library-metrics-collection-type+ {:type nil})++ (try+ ~@body+ (finally+ ;; Clean up any created library collections+ (t2/delete! :model/Collection+ :type [:in [collection/library-collection-type+ collection/library-data-collection-type+ collection/library-metrics-collection-type]])))))++(defn do-with-library-synced+ "Implementation for with-library-synced macro.+ If a Library collection exists, temporarily sets is_remote_synced to true.+ If no Library collection exists, creates a temporary one with is_remote_synced true."+ [f]+ (if-let [library (collection/library-collection)]+ (let [original-synced (:is_remote_synced library)]+ (try+ (t2/update! :model/Collection (:id library) {:is_remote_synced true})+ (f)+ (finally+ (t2/update! :model/Collection (:id library) {:is_remote_synced (boolean original-synced)}))))+ ;; No library collection exists, create one for the test+ (mt/with-temp [:model/Collection _ {:name "Library" :type collection/library-collection-type :is_remote_synced true :location "/"}]+ (f))))++(defmacro with-library-synced+ "Sets up Library collection as remote-synced for the duration of body."+ [& body]+ `(do-with-library-synced (fn [] ~@body)))++(defn do-with-library-not-synced+ "Implementation for with-library-not-synced macro.+ If a Library collection exists, temporarily sets is_remote_synced to false."+ [f]+ (if-let [library (collection/library-collection)]+ (let [original-synced (:is_remote_synced library)]+ (try+ (t2/update! :model/Collection (:id library) {:is_remote_synced false})+ (f)+ (finally+ (t2/update! :model/Collection (:id library) {:is_remote_synced (boolean original-synced)}))))+ (f)))++(defmacro with-library-not-synced+ "Ensures Library collection is not remote-synced for the duration of body."+ [& body]+ `(do-with-library-not-synced (fn [] ~@body)))diff --git a/test/metabase/collections_rest/api_test.clj b/test/metabase/collections_rest/api_test.cljindex 3c3f82387e8..33beae1809d 100644--- a/test/metabase/collections_rest/api_test.clj+++ b/test/metabase/collections_rest/api_test.clj@@ -8,7 +8,7 @@[metabase.collections-rest.settings :as collections-rest.settings][metabase.collections.models.collection :as collection][metabase.collections.models.collection-test :as collection-test]- [metabase.collections.test-helpers :refer [without-library]]+ [metabase.collections.test-utils :refer [without-library]][metabase.notification.api.notification-test :as api.notification-test][metabase.notification.test-util :as notification.tu][metabase.permissions.core :as perms]