The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
feat: enable transform setting on first import (#68807)
Edward Paget9c003314beef
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex 4fdbb040381..04f9ba5cef0 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -23,41 +23,55 @@[toucan2.core :as t2])(:import (metabase_enterprise.remote_sync.source.protocol SourceSnapshot)))+(def ^:private transform-models+ "Models that indicate transforms content in a snapshot."+ #{"Transform" "TransformTag" "PythonLibrary"})++(defn- snapshot-has-transforms?+ "Checks if the snapshot contains any Transform, TransformTag, or PythonLibrary entities.+ Used to auto-enable remote-sync-transforms setting during import.++ Uses the ingestable to list all entities and checks their :model metadata."+ [ingestable]+ (let [serdes-paths (serialization/ingest-list ingestable)]+ (some (fn [path]+ (some #(transform-models (:model %)) path))+ serdes-paths)))+(defn- sync-objects!"Populates the remote-sync-object table with imported entities. Deletes all existing RemoteSyncObject records andinserts new ones for each imported entity, marking them as 'synced' with the given timestamp.- Takes a timestamp instant, a map of imported entities grouped by model name (for standard models with entity_id),- and separate path collections for Table and Field models which use name-based identity.+ Takes a timestamp instant and imported-data map from spec/extract-imported-entities.Uses the spec system to determine how to query and build sync objects for each model type."- [timestamp imported-entities-by-model table-paths field-paths]+ [timestamp imported-data](t2/delete! :model/RemoteSyncObject)- (let [all-inserts (spec/sync-all-entities! timestamp imported-entities-by-model table-paths field-paths)]+ (let [all-inserts (spec/sync-all-entities! timestamp imported-data)](when (seq all-inserts)(t2/insert! :model/RemoteSyncObject all-inserts))))(defn- remove-unsynced!"Deletes any remote sync content that was NOT part of the import.- Takes a sequence of remote-synced collection IDs and a map of imported entities grouped by model name. For each- entity-id based model, deletes entities whose entity_id is not in the imported set.+ Takes a sequence of remote-synced collection IDs and imported-data map from spec/extract-imported-entities.+ For each entity-id based model, deletes entities whose entity_id is not in the imported set.Models with :scope-key in their spec are scoped to synced collections (using :id for Collection, :collection_idfor others). Models without :scope-key (like TransformTag) are deleted globally by entity_id.+ Path-based models (Table, Field) are not removed here - they are controlled by published table settings.+Uses specs-for-deletion to process models in dependency order (models with FK references toCollection are deleted before Collection itself)."- [synced-collection-ids imported-entities-by-model]+ [synced-collection-ids {:keys [by-entity-id]}](doseq [[model-key model-spec] (spec/specs-for-deletion):let [serdes-model (:model-type model-spec)- entity-ids (get imported-entities-by-model serdes-model [])+ entity-ids (get by-entity-id serdes-model [])entity-id-clause (if (seq entity-ids)[:not-in entity-ids]:entity_id)- ;; :scope-key determines collection scoping - nil means globalscope-key (get-in model-spec [:removal :scope-key])- ;; Extra conditions to protect certain rows from removal (e.g. built-in TransformTags)extra-conditions (into [] cat (get-in model-spec [:removal :conditions]))]](if scope-key;; Collection-scoped: delete only within synced collections@@ -142,47 +156,23 @@:version (source.p/version snapshot):message (format "Skipping import: snapshot version %s matches last imported version" snapshot-version)}(log/infof (:message <>)))- (let [path-filters (cond-> [#"collections/.*" #"databases/.*" #"actions/.*"]- (settings/remote-sync-transforms)- (conj #"transforms/.*" #"python-libraries/.*")- (settings/library-is-remote-synced?)- (conj #"snippets/.*"))- ingestable-snapshot (->> (source.p/->ingestable snapshot {:path-filters path-filters})- (source.ingestable/wrap-progress-ingestable task-id 0.7))+ (let [path-filters [#"collections/.*" #"databases/.*" #"actions/.*"+ #"transforms/.*" #"python-libraries/.*" #"snippets/.*"]+ base-ingestable (source.p/->ingestable snapshot {:path-filters path-filters})+ has-transforms? (snapshot-has-transforms? base-ingestable)+ ingestable-snapshot (source.ingestable/wrap-progress-ingestable task-id 0.7 base-ingestable)load-result (serdes/with-cache(serialization/load-metabase! ingestable-snapshot))seen-paths (:seen load-result)- data-model-models #{"Table" "Field"}- ;; For standard models (Collection, Card, Segment, etc.), extract entity_id as before- imported-entities-by-model (->> seen-paths- (remove #(data-model-models (:model (last %))))- (map last)- (group-by :model)- (map (fn [[model entities]]- [model (set (map :id entities))]))- (into {}))- ;; For Tables, capture full path for unique identification- ;; Path format: [{:model "Database" :id db-name} {:model "Schema" :id schema}? {:model "Table" :id table-name}]- table-paths (->> seen-paths- (filter #(= "Table" (:model (last %))))- (map (fn [path]- {:db_name (-> path first :id)- :schema (when (= 3 (count path)) (-> path second :id))- :table_name (-> path last :id)})))- ;; For Fields, path format includes Table- ;; [{:model "Database"} {:model "Schema"}? {:model "Table"} {:model "Field" :id field-name}]- field-paths (->> seen-paths- (filter #(= "Field" (:model (last %))))- (map (fn [path]- (let [table-idx (dec (count path))]- {:db_name (-> path first :id)- :schema (when (> (count path) 3) (-> path second :id))- :table_name (-> path (nth (dec table-idx)) :id)- :field_name (-> path last :id)}))))]+ imported-data (spec/extract-imported-entities seen-paths)](remote-sync.task/update-progress! task-id 0.8)+ (when (and has-transforms?+ (not (settings/remote-sync-transforms)))+ (log/info "Detected transforms in remote source, enabling remote-sync-transforms setting")+ (settings/remote-sync-transforms! true))(t2/with-transaction [_conn]- (remove-unsynced! (spec/all-syncable-collection-ids) imported-entities-by-model)- (sync-objects! sync-timestamp imported-entities-by-model table-paths field-paths))+ (remove-unsynced! (spec/all-syncable-collection-ids) imported-data)+ (sync-objects! sync-timestamp imported-data))(remote-sync.task/update-progress! task-id 0.95)(remote-sync.task/set-version!task-iddiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.cljindex d0f7aa51247..88cebef4481 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj@@ -123,7 +123,11 @@(defn- sync-transform-tracking-on-change"Called when remote-sync-transforms setting changes."[_old-value new-value]- (sync-transform-tracking! (boolean new-value)))+ ;; new-value may be a string "true"/"false" or a boolean, so we need to parse it+ (let [enabled? (if (string? new-value)+ (parse-boolean new-value)+ (boolean new-value))]+ (sync-transform-tracking! enabled?)))(defsetting remote-sync-transforms(deferred-tru "Whether to sync transforms via remote-sync. When enabled, all transforms, transform tags, and transform jobs are synced as a single unit (all-or-nothing).")diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.cljindex 8cb0095046d..5becdc7c078 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj@@ -559,6 +559,67 @@{:entity-id (:entity_id instance):path (select-keys instance path-keys)})+;;; ----------------------------------------- Serdes Path Identity Extraction ------------------------------------------++(def ^:private serdes-path-identity-hierarchy+ "Hierarchy for extract-identity-from-serdes-path dispatch. Both :entity-id and :hybrid models+ extract identity the same way (entity_id from last path element)."+ (-> (make-hierarchy)+ (derive :entity-id ::entity-id-extractor)+ (derive :hybrid ::entity-id-extractor)))++(defmulti extract-identity-from-serdes-path+ "Extracts identity data from a serdes path based on the spec's identity strategy. For entity-id+ and hybrid models, returns the entity_id string from the last path element. For path-based models+ like Table and Field, returns a map with database, schema, and table/field names that can be used+ to look up the entity."+ {:arglists '([spec serdes-path])}+ (fn [spec _path] (:identity spec))+ :hierarchy #'serdes-path-identity-hierarchy)++(defmethod extract-identity-from-serdes-path ::entity-id-extractor+ [_ serdes-path]+ (:id (last serdes-path)))++(defmethod extract-identity-from-serdes-path :path+ [_ serdes-path]+ (let [path-map (into {} (map (fn [elem] [(keyword (u/lower-case-en (:model elem))) (:id elem)]) serdes-path))]+ (cond-> {}+ (contains? path-map :database) (assoc :db_name (:database path-map))+ (contains? path-map :schema) (assoc :schema (:schema path-map))+ (contains? path-map :table) (assoc :table_name (:table path-map))+ (contains? path-map :field) (assoc :field_name (:field path-map)))))++(defmethod extract-identity-from-serdes-path :default+ [_ _]+ nil)++(defn extract-imported-entities+ "Processes serdes paths from an import and extracts entity identities grouped by how they should be looked up.+ Returns a map with :by-entity-id containing entity_ids grouped by model type, and :by-path containing+ path lookup maps for models like Table and Field that use path-based identity."+ [seen-paths]+ (reduce+ (fn [acc path]+ (let [model-type (-> path last :model)]+ (if-let [spec (spec-for-model-type model-type)]+ (let [identity-type (:identity spec)+ identity-data (extract-identity-from-serdes-path spec path)]+ (if identity-data+ (case identity-type+ (:entity-id :hybrid)+ (update-in acc [:by-entity-id model-type] (fnil conj #{}) identity-data)++ :path+ (update-in acc [:by-path (:model-key spec)] (fnil conj []) identity-data)++ acc)+ acc))+ acc)))+ {:by-entity-id {}+ :by-path {}}+ seen-paths))+;;; --------------------------------------------- Export Path Construction ---------------------------------------------(defn- transform-entity-for-serdes@@ -799,25 +860,18 @@nil)(defn sync-all-entities!- "Syncs all entities based on specs. Takes:- - timestamp: The sync timestamp- - imported-entities-by-model: Map of model-type string -> set of entity_ids- - table-paths: Collection of table path maps- - field-paths: Collection of field path maps-- Returns a sequence of all sync objects to insert."- [timestamp imported-entities-by-model table-paths field-paths]+ "Builds RemoteSyncObject entries for all imported entities based on their specs. Iterates over enabled+ specs and queries the database to hydrate the fields needed for each sync object. Returns a sequence+ of maps ready for insertion into the RemoteSyncObject table."+ [timestamp {:keys [by-entity-id by-path]}](into [](for [[model-key spec] (enabled-specs):let [identity-type (:identity spec)model-type (:model-type spec)data (case identity-type- :entity-id (get imported-entities-by-model model-type)- :path (case model-key- :model/Table table-paths- :model/Field field-paths- nil)- :hybrid (get imported-entities-by-model model-type))]+ :entity-id (get by-entity-id model-type)+ :path (get by-path model-key)+ :hybrid (get by-entity-id model-type))]:when (seq data)entity (query-entities-for-sync spec data timestamp)]entity)))@@ -835,7 +889,6 @@[{:keys [export-scope]}](case (or export-scope :derived):root-collections- ;; Collection model: query for root-level remote-synced + transforms-namespace + snippets-namespace collections;; Excludes archived collections - their files are handled by the removal logic(concat(t2/select-fn-set (juxt (constantly "Collection") :id)@@ -859,7 +912,6 @@[:= :location "/"][:not :archived]]}))):derived- ;; Other collection-based models: no root query, derived from collection expansionnil))(defmethod query-export-roots :setting@@ -867,27 +919,22 @@(when (spec-enabled? spec)(case export-scope:root-only- ;; Transform: root transforms (collection_id = nil)(t2/select-fn-set (juxt (constantly model-type) :id) model-key :collection_id nil):all- ;; TransformTag: all instances(t2/select-fn-set (juxt (constantly model-type) :id) model-key)- ;; Default: no root querynil)))-(defmethod query-export-roots :published-table [_] nil) ; Derived via serdes/descendants-(defmethod query-export-roots :parent-table [_] nil) ; Derived via serdes/descendants+(defmethod query-export-roots :published-table [_] nil)+(defmethod query-export-roots :parent-table [_] nil)(defmethod query-export-roots :library-synced[{:keys [export-scope model-key model-type archived-key] :as spec}](when (spec-enabled? spec)(case export-scope:all- ;; NativeQuerySnippet: all non-archived instances(if archived-key(t2/select-fn-set (juxt (constantly model-type) :id) model-key archived-key false)(t2/select-fn-set (juxt (constantly model-type) :id) model-key))- ;; Default: no root querynil)))(defmethod query-export-roots :default [_] nil)diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.cljindex 0175ae0e006..bb26f7e5aec 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/events_test.clj@@ -513,7 +513,7 @@:model/Card card {:name "Test Card":dataset_query (mt/mbql-query venues):collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {"Card" #{(:entity_id card)}} nil nil)+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Card" #{(:entity_id card)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Card" :model_id (:id card))](is (= "synced" (:status initial-entry)))@@ -551,7 +551,7 @@:model/Collection normal-collection {:name "Normal"}:model/Dashboard dashboard {:name "Test Dashboard":collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {"Dashboard" #{(:entity_id dashboard)}} nil nil)+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Dashboard" #{(:entity_id dashboard)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Dashboard" :model_id (:id dashboard))](is (= "synced" (:status initial-entry)))@@ -585,7 +585,7 @@(mt/with-temp [:model/Collection remote-sync-collection {:is_remote_synced true :name "Remote-Sync"}:model/Collection normal-collection {:name "Normal"}:model/Document document {:collection_id (u/the-id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {"Document" #{(:entity_id document)}} nil nil)+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Document" #{(:entity_id document)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Document" :model_id (:id document))](is (= "synced" (:status initial-entry)))@@ -616,7 +616,7 @@(deftest existing-collection-type-changed-from-remote-synced-test(testing "existing collection type changed from remote-synced is marked as removed"(mt/with-temp [:model/Collection collection {:is_remote_synced true :name "Remote-Sync"}]- (#'impl/sync-objects! (t/instant) {"Collection" #{(:entity_id collection)}} nil nil)+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Collection" #{(:entity_id collection)}}})(let [initial-entry (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id (:id collection))](is (= "synced" (:status initial-entry)))(events/publish-event! :event/collection-update@@ -765,7 +765,7 @@:model/Collection normal-collection {:name "Normal Collection"}:model/Timeline timeline {:name "Test Timeline":collection_id (:id remote-sync-collection)}]- (#'impl/sync-objects! (t/instant) {"Timeline" #{(:entity_id timeline)}} nil nil)+ (#'impl/sync-objects! (t/instant) {:by-entity-id {"Timeline" #{(:entity_id timeline)}}})(is (= 1 (count (t2/select :model/RemoteSyncObject))))(events/publish-event! :event/timeline-update{:object (assoc timeline :collection_id (:id normal-collection))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.cljindex 2ae1d5e5113..22aac37c8bd 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj@@ -5,6 +5,7 @@[java-time.api :as t][metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]+ [metabase-enterprise.remote-sync.settings :as remote-sync.settings][metabase-enterprise.remote-sync.source :as source][metabase-enterprise.remote-sync.source.protocol :as source.p][metabase-enterprise.remote-sync.test-helpers :as test-helpers]@@ -144,7 +145,6 @@(testing "export! errors when there are no remote-synced collections"(mt/with-temporary-setting-values [remote-sync-type :read-write](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- ;; Create a regular collection (not remote-synced) to verify it's not included(mt/with-temp [:model/Collection {_coll-id :id} {:name "Regular Collection" :type nil :location "/"}](mt/with-temporary-setting-values [remote-sync-transforms false](let [mock-source (test-helpers/create-mock-source)@@ -194,27 +194,22 @@(remote-sync.task/complete-sync-task! (:id export-task))(is (= :success (:status export-result)))- ;; Verify files were written to the mock source atom(let [files-after-export (get @(:files-atom mock-main) "test-branch")](is (map? files-after-export))(is (not-empty files-after-export))- ;; Should have at least collection and card files(is (some #(str/includes? % "collection") (keys files-after-export)))(is (some #(str/includes? % "card") (keys files-after-export))))(t2/delete! :model/RemoteSyncTask :id (:id export-task))- ;; Then import - verify it succeeds and processes the exported files(let [import-task (t2/with-connection [_conn (app-db/app-db) (t2/insert-returning-instance! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})])import-result (impl/import! (source.p/snapshot mock-main) (:id import-task))](remote-sync.task/complete-sync-task! (:id import-task))(is (= :success (:status import-result)))(is (= "Successfully reloaded from git repository" (:message import-result)))- ;; Verify the entities still exist after import(is (t2/exists? :model/Collection :id coll-id))(is (t2/exists? :model/Card :id card-id))- ;; Verify the collection and card still have the correct attributes(let [collection (t2/select-one :model/Collection :id coll-id)card (t2/select-one :model/Card :id card-id)](is (= "Test Collection" (:name collection)))@@ -239,7 +234,6 @@result (impl/import! (source.p/snapshot mock-main) (:id import-task))](is (= :success (:status result)))- ;; Verify the entities still exist (real cleanup would require more complex setup)(is (t2/exists? :model/Card :id card1-id))(is (not (t2/exists? :model/Collection :id coll2-id)))(is (not (t2/exists? :model/Card :id card2-id))))))))@@ -263,12 +257,10 @@(swap! progress-calls conj {:task-id task-id :progress progress}))](let [result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify progress was called with expected values(is (= 5 (count @progress-calls)))(is (= task-id (:task-id (first @progress-calls))))(is (= task-id (:task-id (second @progress-calls))))(is (= task-id (:task-id (nth @progress-calls 2))))- ;; Check progress values are in expected sequence(is (= 0.7 (:progress (nth @progress-calls 2))))(is (= 0.8 (:progress (nth @progress-calls 3))))(is (= 0.95 (:progress (nth @progress-calls 4)))))))))))@@ -288,11 +280,8 @@(swap! progress-calls conj {:task-id task-id :progress progress}))](let [result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :success (:status result)))- ;; Verify progress was called with expected values- ;; 1 initial call at 0.3, then N calls during store! for each entity(is (pos? (count @progress-calls)))(is (= task-id (:task-id (first @progress-calls))))- ;; Check first progress value is expected (before store!)(is (= 0.3 (:progress (first @progress-calls)))))))))))))(deftest import!-resets-remote-sync-object-table-test@@ -452,10 +441,8 @@result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :success (:status result)))(let [files-after-export (get @(:files-atom mock-source) "main")]- ;; Active collection should be written by export(is (some #(str/includes? % "active-coll-xxxxxxxxx") (keys files-after-export))"Active collection files should exist after export")- ;; Both removed collections should be deleted(is (not (some #(str/includes? % "removed-coll-1xxxxxxx") (keys files-after-export)))"First removed collection files should be deleted")(is (not (some #(str/includes? % "removed-coll-2xxxxxxx") (keys files-after-export)))@@ -582,7 +569,6 @@:is_remote_synced true:entity_id "active-coll-xxxxxxxxx":location "/"}]- ;; Mark the table as 'removed' in RemoteSyncObject(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])@@ -613,7 +599,6 @@:definition {:source-table table-id:filter [:> [:field 1 nil] 0]}:entity_id "test-segment-xxxxxxxx"}]- ;; Mark the segment as 'removed' in RemoteSyncObject(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])@@ -646,18 +631,15 @@:definition {:source-table table-id:filter [:> [:field 1 nil] 0]}:entity_id "test-segment-xxxxxxxx"}]- ;; Create RemoteSyncObject entries with 'removed' status(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}{:model_type "Field" :model_id field-id :model_name "test-field" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}{:model_type "Segment" :model_id segment-id :model_name "Test Segment" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])- ;; Verify entries exist before export(is (= 4 (t2/count :model/RemoteSyncObject)))(let [mock-source (test-helpers/create-mock-source)result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")](is (= :success (:status result)))- ;; After export, all entries should have status 'synced'(is (= 4 (t2/count :model/RemoteSyncObject))"All RemoteSyncObject entries should remain after export")(let [table-entry (t2/select-one :model/RemoteSyncObject :model_type "Table" :model_id table-id)@@ -685,12 +667,10 @@:is_remote_synced true:entity_id "active-coll-xxxxxxxxx":location "/"}]- ;; Mark the table as 'removed' in RemoteSyncObject(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Table" :model_id table-id :model_name "test-table" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])- (let [;; Path should include schema: databases/{db}/schemas/{schema}/tables/{table}- initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"+ (let [initial-files {"main" {"databases/test-db/schemas/PUBLIC/tables/test-table/test-table.yaml"(test-helpers/generate-table-yaml "test-table" "test-db" :schema "PUBLIC")}}mock-source (test-helpers/create-mock-source :initial-files initial-files)result (impl/export! (source.p/snapshot mock-source) task-id "Test commit")]@@ -729,7 +709,6 @@:filter [:> [:field 2 nil] 0]}:entity_id "archived-segment-xxxx":archived true}]- ;; Track the collection so export runs(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])(let [mock-source (test-helpers/create-mock-source)@@ -737,7 +716,6 @@(is (= :success (:status result)))(let [files-after-export (get @(:files-atom mock-source) "main")file-keys (keys files-after-export)]- ;; File paths use entity_id, e.g. "active-segment-xxxxxx_active_segment.yaml"(is (some #(str/includes? % "active-segment-xxxxxx") file-keys)"Active segment should be exported")(is (not (some #(str/includes? % "archived-segment-xxxx") file-keys))@@ -762,11 +740,9 @@:filter [:> [:field 1 nil] 0]}:entity_id "archived-seg-xxxxxxxx":archived true}]- ;; Mark the segment as 'delete' in RemoteSyncObject (simulating archival event)(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Segment" :model_id segment-id :model_name "Archived Segment" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])- ;; File path uses slugified name to match what serdes/storage-path generates(let [initial-files {"main" {"databases/test-db/tables/test-table/segments/archived-seg-xxxxxxxx_archived_segment.yaml"(test-helpers/generate-segment-yaml "Archived Segment" "test-table" "test-db")}}mock-source (test-helpers/create-mock-source :initial-files initial-files)@@ -799,7 +775,6 @@mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify RemoteSyncObject entries exist(is (t2/exists? :model/RemoteSyncObject:model_type "Collection":model_id coll-id@@ -829,7 +804,6 @@mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify RemoteSyncObject entry exists for the field(is (t2/exists? :model/RemoteSyncObject:model_type "Field":model_id field-id@@ -866,7 +840,6 @@mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify RemoteSyncObject entry exists for the segment(is (t2/exists? :model/RemoteSyncObject:model_type "Segment":model_id segment-id@@ -890,7 +863,6 @@mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify RemoteSyncObject entry exists for the table(is (t2/exists? :model/RemoteSyncObject:model_type "Table":model_id table-id@@ -954,7 +926,6 @@mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))- ;; Verify RemoteSyncObject entry exists for the measure(is (t2/exists? :model/RemoteSyncObject:model_type "Measure":model_id measure-id@@ -977,7 +948,6 @@{:name "Test Measure":table_id table-id:entity_id "test-measure-xxxxxxxx"}]- ;; Mark the measure as 'removed' in RemoteSyncObject(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Measure" :model_id measure-id :model_name "Test Measure" :model_table_id table-id :model_table_name "test-table" :status "removed" :status_changed_at (t/offset-date-time)}])@@ -1017,7 +987,6 @@:table_id table-id:entity_id "archived-measure-xxxx":archived true}]- ;; Track the collection so export runs(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Test Collection" :status "synced" :status_changed_at (t/offset-date-time)}])(let [mock-source (test-helpers/create-mock-source)@@ -1025,7 +994,6 @@(is (= :success (:status result)))(let [files-after-export (get @(:files-atom mock-source) "main")file-keys (keys files-after-export)]- ;; File paths use entity_id, e.g. "active-measure-xxxxxx_active_measure.yaml"(is (some #(str/includes? % "active-measure-xxxxxx") file-keys)(str "Active measure should be exported. Keys: " (pr-str file-keys)))(is (not (some #(str/includes? % "archived-measure-xxxx") file-keys))@@ -1048,11 +1016,9 @@:table_id table-id:entity_id "archived-meas-xxxxxxx":archived true}]- ;; Mark the measure as 'delete' in RemoteSyncObject (simulating archival event)(t2/insert! :model/RemoteSyncObject[{:model_type "Collection" :model_id coll-id :model_name "Active Collection" :status "synced" :status_changed_at (t/offset-date-time)}{:model_type "Measure" :model_id measure-id :model_name "Archived Measure" :model_table_id table-id :model_table_name "test-table" :status "delete" :status_changed_at (t/offset-date-time)}])- ;; File path uses slugified name to match what serdes/storage-path generates(let [initial-files {"main" {"databases/test-db/tables/test-table/measures/archived-meas-xxxxxxx_archived_measure.yaml"(test-helpers/generate-measure-yaml "Archived Measure" "test-table" "test-db")}}mock-source (test-helpers/create-mock-source :initial-files initial-files)@@ -1064,3 +1030,110 @@(testing "RemoteSyncObject entry is cleaned up after export"(is (= "synced" (:status (t2/select-one :model/RemoteSyncObject :model_type "Measure" :model_id measure-id)))"RemoteSyncObject entry for archived measure should have synced status")))))))))++;; Auto-enable transforms tests++(deftest import!-auto-enables-transforms-setting-when-transforms-detected-test+ (testing "import! auto-enables remote-sync-transforms setting only after successful import with transforms"+ (mt/with-premium-features #{:transforms}+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/RemoteSyncObject]+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ transform-entity-id "auto-enable-trans-xxx"+ test-files {"main" {(str "transforms/" transform-entity-id "_test_transform.yaml")+ (test-helpers/generate-transform-yaml transform-entity-id "Test Transform")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)]+ (is (false? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be initially disabled")+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ "Import should succeed")+ (is (true? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be auto-enabled after successful import with transforms"))))))))++(deftest import!-auto-enables-transforms-setting-when-python-libraries-detected-test+ (testing "import! auto-enables remote-sync-transforms setting only after successful import with python-libraries"+ (mt/with-premium-features #{:transforms}+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/PythonLibrary :model/RemoteSyncObject]+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ lib-entity-id "auto-enable-lib-xxxxx"+ test-files {"main" {(str "python-libraries/" lib-entity-id ".yaml")+ (format "path: uncommon.py+source: |+ # shared code+ def shared_func():+ return 42+entity_id: %s+created_at: '2024-08-28T09:46:18.671622Z'+serdes/meta:+- id: %s+ model: PythonLibrary+" lib-entity-id lib-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)]+ (is (false? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be initially disabled")+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ "Import should succeed")+ (is (true? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be auto-enabled after successful import with python-libraries"))))))))++(deftest import!-keeps-transforms-setting-disabled-when-no-transforms-present-test+ (testing "import! keeps remote-sync-transforms setting disabled when no transforms are present"+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temporary-setting-values [remote-sync-transforms false]+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "No Transforms Coll" :is_remote_synced true :entity_id "no-transforms-coll-xx" :location "/"}]+ (let [test-files {"main" {"collections/no-transforms-coll-xx-_/no-transforms-coll-xx.yaml"+ (test-helpers/generate-collection-yaml "no-transforms-coll-xx" "No Transforms Coll")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)]+ (is (false? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be initially disabled")+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result)))+ (is (false? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should remain disabled when no transforms in remote")))))))))++(deftest import!-does-not-disable-transforms-setting-when-already-enabled-test+ (testing "import! does not modify remote-sync-transforms setting when it's already enabled"+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temporary-setting-values [remote-sync-transforms true]+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "Already Enabled Coll" :is_remote_synced true :entity_id "already-enabled-collx" :location "/"}]+ (let [test-files {"main" {"collections/already-enabled-collx-_/already-enabled-collx.yaml"+ (test-helpers/generate-collection-yaml "already-enabled-collx" "Already Enabled Coll")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)]+ (is (true? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should be initially enabled")+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result)))+ (is (true? (remote-sync.settings/remote-sync-transforms))+ "remote-sync-transforms should remain enabled even when no transforms in remote")))))))))++(deftest import!-includes-all-optional-paths-regardless-of-settings-test+ (testing "import! always includes all optional paths (transforms, python-libraries, snippets)"+ (mt/with-model-cleanup [:model/RemoteSyncTask]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]+ (mt/with-temporary-setting-values [remote-sync-transforms false]+ (mt/with-temp [:model/Collection {_coll-id :id} {:name "All Paths Coll" :is_remote_synced true :entity_id "all-paths-coll-xxxxxx" :location "/"}]+ (let [test-files {"main" {"collections/all-paths-coll-xxxxxx-_/all-paths-coll-xxxxxx.yaml"+ (test-helpers/generate-collection-yaml "all-paths-coll-xxxxxx" "All Paths Coll")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ paths-passed (atom nil)+ original-fn source.p/->ingestable]+ (with-redefs [source.p/->ingestable (fn [snapshot opts]+ (reset! paths-passed (:path-filters opts))+ (original-fn snapshot opts))]+ (let [result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result)))+ (when @paths-passed+ (let [filter-strs (map str @paths-passed)]+ (is (some #(str/includes? % "transforms") filter-strs)+ "transforms path should be included in filters")+ (is (some #(str/includes? % "python-libraries") filter-strs)+ "python-libraries path should be included in filters")+ (is (some #(str/includes? % "snippets") filter-strs)+ "snippets path should be included in filters"))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex ef3604fe310..31e8f9b80f6 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -8,15 +8,8 @@[toucan2.core :as t2]))(defn generate-collection-yaml- "Generates YAML content for a collection.-- Args:- entity-id: The unique identifier for the collection.- name: The name of the collection.- parent-id: Optional parent collection ID.-- Returns:- A string containing the YAML representation of the collection."+ "Generate YAML content for a collection with the given `entity-id` and `name`.+ Optionally accepts `:parent-id` for nested collections."[entity-id name & {:keys [parent-id]}](format "name: %sdescription: null@@ -41,20 +34,10 @@ is_sample: false(or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))(defn generate-v57-collection-yaml- "Generates YAML content for a collection in v57 format.-- In v57, remote-synced collections used `type: remote-synced` instead of- `is_remote_synced: true`. This helper is used to test backward compatibility- when importing exports from v57 Metabase instances.-- Args:- entity-id: The unique identifier for the collection.- name: The name of the collection.- parent-id: Optional parent collection ID.- type: Optional type value (e.g., \"remote-synced\" for v57 remote-synced collections).-- Returns:- A string containing the v57-format YAML representation of the collection."+ "Generate YAML content for a collection in v57 format. In v57, remote-synced collections+ used `type: remote-synced` instead of `is_remote_synced: true`. Used to test backward+ compatibility when importing exports from v57 Metabase instances. Optionally accepts+ `:parent-id` and `:type` (e.g., `\"remote-synced\"`)."[entity-id name & {:keys [parent-id type]}](format "name: %sdescription: null@@ -79,15 +62,8 @@ is_sample: false(or type "null") (or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))(defn generate-card-yaml- "Generates YAML content for a card.-- Args:- entity-id: The unique identifier for the card.- name: The name of the card.- collection-id: The ID of the collection containing this card.-- Returns:- A string containing the YAML representation of the card."+ "Generate YAML content for a card with the given `entity-id`, `name`, and `collection-id`.+ Optionally accepts a `type` (defaults to `\"question\"`)."[entity-id name collection-id & [type]](format "name: %sdescription: null@@ -127,16 +103,8 @@ document_id: nullname entity-id collection-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_") (or type "question")))(defn generate-dashboard-yaml- "Generates YAML content for a dashboard.-- Args:- entity-id: The unique identifier for the dashboard.- name: The name of the dashboard.- collection-id: The ID of the collection containing this dashboard.- dashcards: Optional collection of dashboard cards to include.-- Returns:- A string containing the YAML representation of the dashboard."+ "Generate YAML content for a dashboard with the given `entity-id`, `name`, and `collection-id`.+ Optionally accepts `:dashcards` to include dashboard cards."[entity-id name collection-id & {:keys [dashcards]}](let [dashcards-yaml (if dashcards(str/join "\n" (map (fn [dc]@@ -261,7 +229,7 @@ width: fixed(->MockSourceSnapshot source-id base-url branch fail-mode files-atom)))(defn create-mock-source- "Create a mock Source for testing"+ "Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, and `:initial-files`."[& {:keys [branch fail-mode initial-files]:or {branch "main"fail-mode nil@@ -287,7 +255,8 @@ width: fixed(->MockSource "test-source" "https://test.example.com" branch fail-mode files-atom branches-atom)))(defn clean-object- "Reset the object table before running tests to prevent existing extries from affecting dirty state checks"+ "Test fixture that resets the RemoteSyncObject table before running tests to prevent existing+ entries from affecting dirty state checks."[f](let [old-models (t2/select :model/RemoteSyncObject)](try@@ -299,12 +268,12 @@ width: fixed(t2/insert! :model/RemoteSyncObject old-models))))))(defmacro with-clean-object- "Macro to wrap a body to execute in a clean change log table"+ "Execute `body` with a clean RemoteSyncObject table."[& body]`(clean-object (fn [] ~@body)))(defn clean-task-table- "Reset the task table to an empty state before running tests."+ "Test fixture that resets the RemoteSyncTask table to an empty state before running tests."[f](let [old-models (t2/select :model/RemoteSyncTask)](try@@ -316,18 +285,12 @@ width: fixed(t2/insert! :model/RemoteSyncTask old-models))))))(def clean-remote-sync-state- "Fixture to make sure sync state is clean"+ "Composed test fixture that ensures both RemoteSyncObject and RemoteSyncTask tables are clean."(t/compose-fixtures clean-object clean-task-table))(defn generate-table-yaml- "Generates YAML content for a table.-- Args:- table-name: The name of the table (used as entity_id).- db-name: The name of the database containing this table.-- Returns:- A string containing the YAML representation of the table."+ "Generate YAML content for a table with the given `table-name` and `db-name`.+ Optionally accepts `:schema`, `:is-published` (defaults to true), and `:description`."[table-name db-name & {:keys [schema is-published description]:or {is-published truedescription nil}}]@@ -375,15 +338,8 @@ serdes/meta:table-name))(defn generate-field-yaml- "Generates YAML content for a field.-- Args:- field-name: The name of the field (used as entity_id).- table-name: The name of the table containing this field.- db-name: The name of the database containing this field.-- Returns:- A string containing the YAML representation of the field."+ "Generate YAML content for a field with the given `field-name`, `table-name`, and `db-name`.+ Optionally accepts `:schema`, `:base-type`, `:description`, and `:database-type`."[field-name table-name db-name & {:keys [schema base-type description database-type]:or {base-type "type/Text"database-type "VARCHAR"@@ -446,16 +402,8 @@ database_partitioned: nullfield-name))(defn generate-segment-yaml- "Generates YAML content for a segment.-- Args:- segment-name: The name of the segment (used as entity_id).- table-name: The name of the table containing this segment.- db-name: The name of the database containing this segment.- filter-field-name: The name of the field to use in the filter clause.-- Returns:- A string containing the YAML representation of the segment."+ "Generate YAML content for a segment with the given `segment-name`, `table-name`, and `db-name`.+ Optionally accepts `:schema`, `:description`, `:entity-id`, and `:filter-field-name`."[segment-name table-name db-name & {:keys [schema description entity-id filter-field-name]:or {description "Test segment"filter-field-name "Some Field"}}]@@ -513,15 +461,8 @@ serdes/meta:(str/replace (u/lower-case-en segment-name) #"\s+" "_"))))(defn generate-action-yaml- "Generates YAML content for an action.-- Args:- entity-id: The unique identifier for the action.- name: The name of the action.- model-id: The entity ID of the model (Card) this action belongs to.-- Returns:- A string containing the YAML representation of the action."+ "Generate YAML content for an action with the given `entity-id`, `name`, and `model-id`.+ Optionally accepts `:type` (defaults to `\"implicit\"`) and `:kind` (defaults to `\"row/create\"`)."[entity-id name model-id & {:keys [type kind]:or {type "implicit"kind "row/create"}}]@@ -555,10 +496,41 @@ serdes/meta:entity-id(str/replace (u/lower-case-en name) #"\s+" "_")))+(defn generate-transform-yaml+ "Generate YAML content for a transform with the given `entity-id` and `name`.+ Optionally accepts `:collection-id` for transforms inside a collection."+ [entity-id name & {:keys [collection-id]}]+ (format "name: %s+description: null+entity_id: %s+collection_id: %s+created_at: '2024-08-28T09:46:18.671622Z'+creator_id: [email protected]+source_database_id: test-data (h2)+source:+ type: query+ query:+ database: test-data (h2)+ type: query+ query:+ source-table:+ - test-data (h2)+ - PUBLIC+ - VENUES+target:+ type: table+ name: test_output+ schema: PUBLIC+serdes/meta:+- id: %s+ label: %s+ model: Transform+"+ name entity-id (or collection-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))+(defn generate-measure-yaml- "Generates YAML content for a measure with the given `measure-name`, `table-name`, and `db-name`.- Optional keyword args include `:schema`, `:description`, `:entity-id`, and `:agg-field-name`- for the aggregation field. Returns a string containing the YAML representation of the measure."+ "Generate YAML content for a measure with the given `measure-name`, `table-name`, and `db-name`.+ Optionally accepts `:schema`, `:description`, `:entity-id`, and `:agg-field-name`."[measure-name table-name db-name & {:keys [schema description entity-id agg-field-name]:or {description "Test measure"agg-field-name "Some Field"}}]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.cljindex d83fbf70d5f..1cf885f511e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj@@ -230,7 +230,7 @@remote-sync-enabled true](mt/with-model-cleanup [:model/RemoteSyncTask](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxx" :location "/"}+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Transforms Collection" :namespace collection/transforms-ns :entity_id "transforms-coll-xxxxx" :location "/"}:model/Transform transform {:name "Export Transform" :collection_id coll-id}:model/RemoteSyncObject _rso1 {:model_type "Collection" :model_id coll-id :model_name "Transforms Collection" :status "create" :status_changed_at (t/offset-date-time)}:model/RemoteSyncObject _rso2 {:model_type "Transform" :model_id (:id transform) :model_name "Export Transform" :model_collection_id coll-id :status "create" :status_changed_at (t/offset-date-time)}]@@ -259,7 +259,7 @@remote-sync-enabled true](mt/with-model-cleanup [:model/RemoteSyncTask](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "export" :initiated_by (mt/user->id :rasta)})]- (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxx" :location "/"}+ (mt/with-temp [:model/Collection {rs-coll-id :id rs-coll-eid :entity_id} {:name "Remote Synced" :is_remote_synced true :entity_id "remote-synced-xxxxxxx" :location "/"}:model/RemoteSyncObject _rso {:model_type "Collection" :model_id rs-coll-id :model_name "Remote Synced" :status "create" :status_changed_at (t/offset-date-time)}](let [saved-coll (t2/select-one :model/Collection :id rs-coll-id)](is (true? (:is_remote_synced saved-coll))@@ -277,33 +277,6 @@(is (not (some #(str/includes? % "transforms/") (keys files-after-export)))"Export should NOT include transform files when setting is disabled"))))))))))-(defn- generate-transform-yaml- "Generates YAML content for a transform."- [entity-id name database-id table-id]- (format "name: %s-description: null-entity_id: %s-collection_id: null-created_at: '2024-08-28T09:46:18.671622Z'-creator_id: [email protected]-source:- type: query- query:- database: %d- type: query- query:- source-table: %d-target:- type: table- name: test_output- schema: PUBLIC-serdes/meta:-- id: %s- label: %s- model: Transform-"- name entity-id database-id table-id entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))-(defn- generate-transforms-namespace-collection-yaml"Generates YAML content for a transforms-namespace collection."[entity-id name]@@ -335,14 +308,14 @@ is_sample: false(mt/with-temporary-setting-values [remote-sync-enabled true](mt/with-model-cleanup [:model/Transform :model/Collection](let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- coll-entity-id "transforms-coll-xxx"- transform-entity-id "test-transform-xxxxx"- test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")- (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")- (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_test_transform.yaml")- (generate-transform-yaml transform-entity-id "Test Transform" (mt/id) (mt/id :venues))}}- mock-source (test-helpers/create-mock-source :initial-files test-files)- result (impl/import! (source.p/snapshot mock-source) task-id)]+ coll-entity-id "transforms-coll-xxxxx"+ transform-entity-id "test-transform-xxxxxx"+ test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")+ (generate-transforms-namespace-collection-yaml coll-entity-id "Transforms")+ (str "collections/" coll-entity-id "_transforms/transforms/" transform-entity-id "_test_transform.yaml")+ (test-helpers/generate-transform-yaml transform-entity-id "Test Transform" :collection-id coll-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result)))(is (t2/exists? :model/Collection :entity_id coll-entity-id :namespace "transforms")"Transforms-namespace collection should be imported")@@ -390,7 +363,7 @@ is_sample: false(let [initial-files {"main" {(str "collections/" coll-eid "_transforms_collection/" coll-eid "_transforms_collection.yaml")(generate-transforms-namespace-collection-yaml coll-eid "Transforms Collection")(str "collections/" coll-eid "_transforms_collection/transforms/" transform-eid "_child_transform.yaml")- (generate-transform-yaml transform-eid "Child Transform" (mt/id) (mt/id :venues))}}+ (test-helpers/generate-transform-yaml transform-eid "Child Transform" :collection-id coll-eid)}}mock-source (test-helpers/create-mock-source :initial-files initial-files)](is (some #(str/includes? % coll-eid) (keys (get @(:files-atom mock-source) "main"))))(is (some #(str/includes? % transform-eid) (keys (get @(:files-atom mock-source) "main"))))@@ -404,7 +377,6 @@ is_sample: false(let [files-after-export (get @(:files-atom mock-source) "main")](is (not (some #(str/includes? % coll-eid) (keys files-after-export))))(is (not (some #(str/includes? % transform-eid) (keys files-after-export))))- ;; Root transform should still be exported(is (some #(str/includes? % root-transform-eid) (keys files-after-export))"Root transform should be exported")))))))))))@@ -461,24 +433,24 @@ is_sample: false(mt/with-premium-features #{:transforms}(mt/with-temporary-setting-values [remote-sync-transforms trueremote-sync-enabled true]- (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]- (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})- coll-entity-id (u/generate-nano-id)- local-transform-entity-id (u/generate-nano-id)- remote-transform-entity-id (u/generate-nano-id)]- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"- :namespace collection/transforms-ns- :entity_id coll-entity-id- :location "/"}- :model/Transform {local-transform-id :id} {:name "Local Transform"- :entity_id local-transform-entity-id- :collection_id coll-id}]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ coll-entity-id (u/generate-nano-id)+ local-transform-entity-id (u/generate-nano-id)+ remote-transform-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id coll-entity-id+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform](is (t2/exists? :model/Collection :id coll-id))(is (t2/exists? :model/Transform :id local-transform-id))(let [test-files {"main" {(str "collections/" coll-entity-id "_transforms/" coll-entity-id "_transforms.yaml")(generate-transforms-namespace-collection-yaml coll-entity-id "Transforms Collection")(str "collections/" coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")- (generate-transform-yaml remote-transform-entity-id "Remote Transform" (mt/id) (mt/id :venues))}}+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id coll-entity-id)}}mock-source (test-helpers/create-mock-source :initial-files test-files)result (impl/import! (source.p/snapshot mock-source) task-id)](is (= :success (:status result))@@ -673,7 +645,6 @@ serdes/meta:local-entity-id (:entity_id local-library)](is (t2/exists? :model/PythonLibrary :path "common.py")"Local PythonLibrary should exist before import")- ;; Use the same entity_id as the local library so it gets updated(let [test-files {"main" {(str "python-libraries/" local-entity-id ".yaml")(generate-python-library-yaml local-entity-id "common.py" "# remote version\ndef new_func():\n pass")}}mock-source (test-helpers/create-mock-source :initial-files test-files)@@ -684,3 +655,159 @@ serdes/meta:(is (some? library) "PythonLibrary should still exist")(is (str/includes? (:source library) "# remote version")"PythonLibrary source should be updated with remote version")))))))))))++;;; ------------------------------------------- Transform Sync Behavior Tests -------------------------------------------++(deftest import-preserves-local-transforms-when-setting-disabled-and-remote-has-none-test+ (testing "When remote-sync-transforms is disabled and remote has no transforms, local transforms are preserved"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-transform-entity-id (u/generate-nano-id)+ local-tag-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"+ :entity_id local-tag-entity-id}]+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform should exist before import")+ (is (t2/exists? :model/TransformTag :id local-tag-id)+ "Local tag should exist before import")+ (is (t2/exists? :model/Collection :id coll-id)+ "Local transforms collection should exist before import")+ (let [remote-coll-entity-id (u/generate-nano-id)+ test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")+ (test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform should still exist after import when setting is disabled")+ (is (t2/exists? :model/TransformTag :id local-tag-id)+ "Local tag should still exist after import when setting is disabled")+ (is (t2/exists? :model/Collection :id coll-id)+ "Local transforms collection should still exist after import when setting is disabled")))))))))++(deftest import-replaces-local-transforms-when-remote-has-transforms-test+ (testing "When remote has transforms, local transforms are replaced with remote content (setting auto-enabled)"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-transform-entity-id (u/generate-nano-id)+ local-tag-entity-id (u/generate-nano-id)+ remote-transform-entity-id (u/generate-nano-id)+ remote-coll-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"+ :entity_id local-tag-entity-id}]+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform should exist before import")+ (is (t2/exists? :model/TransformTag :id local-tag-id)+ "Local tag should exist before import")+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")+ (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (settings/remote-sync-transforms)+ "remote-sync-transforms setting should be auto-enabled when remote has transforms")+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be deleted after import when remote has different transforms")+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))+ "Local tag should be deleted after import when remote has different tags")+ (is (t2/exists? :model/Transform :entity_id remote-transform-entity-id)+ "Remote transform should be imported")+ (is (t2/exists? :model/Collection :entity_id remote-coll-entity-id :namespace "transforms")+ "Remote transforms collection should be imported")+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))+ "Local transforms collection should be removed when not on remote")))))))))++(deftest import-removes-all-local-transforms-when-setting-enabled-and-remote-has-none-test+ (testing "When remote-sync-transforms is enabled and remote has no transforms, all local transforms are removed"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms true+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/TransformTag]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-transform-entity-id (u/generate-nano-id)+ local-tag-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id coll-eid :entity_id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :entity_id (u/generate-nano-id)+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}+ :model/TransformTag {local-tag-id :id} {:name "Local Tag"+ :entity_id local-tag-entity-id}]+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform should exist before import")+ (is (t2/exists? :model/TransformTag :id local-tag-id)+ "Local tag should exist before import")+ (is (t2/exists? :model/Collection :id coll-id)+ "Local transforms collection should exist before import")+ (let [remote-coll-entity-id (u/generate-nano-id)+ test-files {"main" {(str "collections/" remote-coll-entity-id "_remote_collection/" remote-coll-entity-id "_remote_collection.yaml")+ (test-helpers/generate-collection-yaml remote-coll-entity-id "Remote Collection")}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be deleted when setting enabled and remote has no transforms")+ (is (not (t2/exists? :model/TransformTag :id local-tag-id))+ "Local tag should be deleted when setting enabled and remote has no transforms")+ (is (not (t2/exists? :model/Collection :entity_id coll-eid))+ "Local transforms collection should be deleted when setting enabled and remote has none")))))))))++(deftest import-auto-enables-setting-when-remote-has-transforms-test+ (testing "When setting is disabled and remote has transforms, setting is auto-enabled and transforms are synced"+ (mt/with-premium-features #{:transforms}+ (mt/with-temporary-setting-values [remote-sync-transforms false+ remote-sync-enabled true]+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform]+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})+ local-transform-entity-id (u/generate-nano-id)+ remote-transform-entity-id (u/generate-nano-id)+ remote-coll-entity-id (u/generate-nano-id)]+ (mt/with-temp [:model/Collection {coll-id :id} {:name "Local Transforms Collection"+ :namespace collection/transforms-ns+ :location "/"}+ :model/Transform {local-transform-id :id} {:name "Local Transform"+ :entity_id local-transform-entity-id+ :collection_id coll-id}]+ (is (not (settings/remote-sync-transforms))+ "remote-sync-transforms should be disabled initially")+ (is (t2/exists? :model/Transform :id local-transform-id)+ "Local transform should exist before import")+ (let [test-files {"main" {(str "collections/" remote-coll-entity-id "_transforms/" remote-coll-entity-id "_transforms.yaml")+ (generate-transforms-namespace-collection-yaml remote-coll-entity-id "Remote Transforms")+ (str "collections/" remote-coll-entity-id "_transforms/transforms/" remote-transform-entity-id "_remote_transform.yaml")+ (test-helpers/generate-transform-yaml remote-transform-entity-id "Remote Transform" :collection-id remote-coll-entity-id)}}+ mock-source (test-helpers/create-mock-source :initial-files test-files)+ result (impl/import! (source.p/snapshot mock-source) task-id)]+ (is (= :success (:status result))+ (str "Import should succeed. Result: " result))+ (is (settings/remote-sync-transforms)+ "remote-sync-transforms setting should be auto-enabled when remote has transforms")+ (is (t2/exists? :model/Transform :entity_id remote-transform-entity-id)+ "Remote transform should be imported")+ (is (not (t2/exists? :model/Transform :id local-transform-id))+ "Local transform should be removed because it's not on the remote")))))))))