Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Track transforms content per row in remote sync (#84655)

Alexander Polyankin8ab517fedec3
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
index b43320023d1..ba4be3da383 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/api.clj
@@ -4,6 +4,7 @@
[medley.core :as m]
[metabase-enterprise.remote-sync.core :as remote-sync.core]
[metabase-enterprise.remote-sync.db :as remote-sync.db]
+ [metabase-enterprise.remote-sync.events :as remote-sync.events]
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-object :as remote-sync.object]
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
@@ -311,6 +312,7 @@
:error (ex-message e)
:status-code 400)
e)))))
+ (remote-sync.events/sync-transforms-tracking!)
(events/publish-event! :event/remote-sync-settings-update
{:details {:remote-sync-type remote-sync-type}
:user-id api/*current-user-id*})
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
index 1060c8895e0..7fb787efdbb 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/db.clj
@@ -14,7 +14,7 @@
(def ^:private ConditionKey
"The column keys used in the `:conditions` / `:cascade-filter` / `:removal-conditions` of a remote-sync model
spec (see `metabase-enterprise.remote-sync.spec`)."
- [:enum :exploration_id :built_in_type :entity_id :collection_id :model_id :archived :archived_at])
+ [:enum :exploration_id :built_in_type :is_default :entity_id :collection_id :model_id :archived :archived_at])
(def ^:private Conditions
"A map of column to value (possibly nil) or Toucan 2 operator-vector value, or nil for none."
@@ -31,9 +31,8 @@
[:removal-conditions Conditions]])
(def ^:private ModelId
- "A `:model_id`: the primary key of the referenced entity, or the `-1` sentinel
- (`metabase-enterprise.remote-sync.settings/transforms-root-id`) standing in for the virtual Transforms root
- Collection."
+ "A `:model_id`: the primary key of the referenced entity, or `-1` for the legacy Transforms root Collection row that
+ startup converts."
[:maybe [:or ms/PositiveInt [:= -1]]])
(mu/defn eligible-children
@@ -59,6 +58,14 @@
conditions :- Conditions]
(apply t2/select-fn-set :id model-key (mapcat identity conditions)))
+(mu/defn existing-ids :- [:set ms/PositiveInt]
+ "The IDs among `ids` of the existing instances of `model-key`."
+ [model-key :- :keyword
+ ids :- [:sequential ms/PositiveInt]]
+ (if (seq ids)
+ (t2/select-pks-set model-key :id [:in ids])
+ #{}))
+
(mu/defn instances-where
"The instances of `model-key` matching `conditions` (a map of column to value or Toucan 2 operator-vector value,
or nil for every instance)."
@@ -567,22 +574,31 @@
(t2/select :model/RemoteSyncObject {:where [:not= :status "synced"]}))
(mu/defn dirty-rso-exists?
- "Whether a RemoteSyncObject of a model type other than `excluded-model-types` is not synced."
+ "Whether a RemoteSyncObject is not synced, counting those of `excluded-model-types` only for a pending removal."
[excluded-model-types :- [:set :string]]
(t2/exists? :model/RemoteSyncObject
{:where [:and
[:not= :status "synced"]
(when (seq excluded-model-types)
- [:not-in :model_type excluded-model-types])]}))
+ [:or
+ [:not-in :model_type excluded-model-types]
+ [:in :status ["removed" "delete"]]])]}))
(mu/defn dirty-rsos
- "The RemoteSyncObjects of model types other than `excluded-model-types` that are not synced."
+ "The RemoteSyncObjects that are not synced, including those of `excluded-model-types` only for a pending removal."
[excluded-model-types :- [:set :string]]
(t2/select :model/RemoteSyncObject
{:where [:and
[:not= :status "synced"]
(when (seq excluded-model-types)
- [:not-in :model_type excluded-model-types])]}))
+ [:or
+ [:not-in :model_type excluded-model-types]
+ [:in :status ["removed" "delete"]]])]}))
+
+(mu/defn rsos-of-types
+ "The RemoteSyncObjects of `model-types`."
+ [model-types :- [:sequential :string]]
+ (t2/select :model/RemoteSyncObject :model_type [:in model-types]))
(mu/defn tracked-model-ids
"The model IDs of the RemoteSyncObjects of `model-type`."
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
index 972821626eb..df3550bcc76 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/events.clj
@@ -19,8 +19,10 @@
(:require
[java-time.api :as t]
[metabase-enterprise.remote-sync.db :as remote-sync.db]
+ [metabase-enterprise.remote-sync.settings :as settings]
[metabase-enterprise.remote-sync.source :as source]
[metabase-enterprise.remote-sync.spec :as spec]
+ [metabase.app-db.cluster-lock :as cluster-lock]
[metabase.collections.core :as collections]
[metabase.events.core :as events]
[metabase.util.log :as log]
@@ -137,6 +139,119 @@
(when (seq action-ids)
(remote-sync.db/delete-rsos-of-models! "Action" action-ids))))
+(defn- transforms-folder-ids
+ "The IDs of the transforms-namespace Collections."
+ []
+ (remote-sync.db/collection-ids-in-namespace (name collections/transforms-ns)))
+
+(defn- transforms-rsos
+ "The RemoteSyncObjects of transforms content: every model that syncs with transforms, and the transforms folders."
+ []
+ (concat (remote-sync.db/rsos-of-types (vec (spec/models-for-setting :remote-sync-transforms)))
+ (when-let [folder-ids (seq (transforms-folder-ids))]
+ (remote-sync.db/rsos-of-models "Collection" (vec folder-ids)))))
+
+(defn- untracked-row
+ "A 'create' RemoteSyncObject row for `instance` of `model-type`."
+ [model-type instance timestamp]
+ (merge {:model_type model-type
+ :model_id (:id instance)
+ :status "create"
+ :status_changed_at timestamp}
+ (spec/build-sync-object-fields (spec/spec-for-model-type model-type) instance)))
+
+(defn- untracked-transforms-rows
+ "A 'create' row for each piece of transforms content that an export writes and that has no row yet."
+ [timestamp]
+ (let [exportable (update-vals (spec/exportable-entities) set)
+ untracked? (fn [model-type tracked instance]
+ (and (contains? (get exportable model-type) (:id instance))
+ (not (contains? tracked (:id instance)))))]
+ (concat
+ (for [model-type (sort (spec/models-for-setting :remote-sync-transforms))
+ :let [model-spec (spec/spec-for-model-type model-type)
+ tracked (remote-sync.db/tracked-model-ids model-type)]
+ instance (remote-sync.db/instances-where (:model-key model-spec) (spec/export-conditions model-spec))
+ :when (untracked? model-type tracked instance)]
+ (untracked-row model-type instance timestamp))
+ (let [tracked (remote-sync.db/tracked-model-ids "Collection")]
+ (for [folder (remote-sync.db/collections-with-names-in-namespace (name collections/transforms-ns))
+ :when (untracked? "Collection" tracked folder)]
+ (untracked-row "Collection" folder timestamp))))))
+
+(defn- removed-rsos-by-existence
+ "`removed` RemoteSyncObjects split into those whose entity still exists (`true`) and those whose entity is gone
+ (`false`)."
+ [removed]
+ (let [existing (into {}
+ (for [[model-type rsos] (group-by :model_type removed)]
+ [model-type (remote-sync.db/existing-ids (:model-key (spec/spec-for-model-type model-type))
+ (mapv :model_id rsos))]))]
+ (group-by #(contains? (existing (:model_type %)) (:model_id %)) removed)))
+
+(defn enable-transforms-tracking!
+ "Tracks the transforms content: a 'removed' row becomes an update again, or a delete when its entity is gone, and
+ exported content without a row gets a 'create' row."
+ []
+ (let [timestamp (t/offset-date-time)
+ {live true gone false} (removed-rsos-by-existence (filterv #(= "removed" (:status %)) (transforms-rsos)))
+ rows (vec (untracked-transforms-rows timestamp))]
+ (when (seq live)
+ (remote-sync.db/set-rsos-status! (mapv :id live) "update" timestamp))
+ (when (seq gone)
+ (remote-sync.db/set-rsos-status! (mapv :id gone) "delete" timestamp))
+ (when (seq rows)
+ (remote-sync.db/insert-rsos! rows))))
+
+(defn disable-transforms-tracking!
+ "Stops tracking the transforms content: rows never pushed are dropped, and pushed ones become 'removed', so the next
+ push deletes their files while the content stays."
+ []
+ (let [timestamp (t/offset-date-time)
+ rsos (transforms-rsos)
+ created (filterv #(= "create" (:status %)) rsos)
+ pushed (filterv #(#{"synced" "update"} (:status %)) rsos)]
+ (when (seq created)
+ (remote-sync.db/delete-rsos! (mapv :id created)))
+ (when (seq pushed)
+ (remote-sync.db/set-rsos-status! (mapv :id pushed) "removed" timestamp))))
+
+(def ^:private legacy-transforms-root-id
+ "The model_id of the Collection RemoteSyncObject that stood for all of transforms before they were tracked per row."
+ -1)
+
+(defn- do-with-transforms-tracking-lock
+ "Calls `thunk` in one transaction, holding the cluster lock that serializes changes to the transforms tracking."
+ [thunk]
+ (cluster-lock/with-cluster-lock ::transforms-tracking
+ (t2/with-transaction [_conn]
+ (thunk))))
+
+(defn convert-legacy-transforms-root!
+ "Replaces the legacy Transforms root RemoteSyncObject with per-row tracking, once across the cluster: a pending
+ 'create' or 'update' tracks the transforms content, a pending 'delete' marks it removed, and the root row goes."
+ []
+ (when (remote-sync.db/rso "Collection" legacy-transforms-root-id)
+ (do-with-transforms-tracking-lock
+ (fn []
+ (when-let [{:keys [status]} (remote-sync.db/rso "Collection" legacy-transforms-root-id)]
+ (case status
+ ("create" "update") (enable-transforms-tracking!)
+ "delete" (disable-transforms-tracking!)
+ nil)
+ (remote-sync.db/delete-rso-of! "Collection" legacy-transforms-root-id))))))
+
+(defn sync-transforms-tracking!
+ "Brings the transforms tracking in line with [[settings/remote-sync-transforms]] on a read-write instance, which is
+ the only kind that publishes."
+ []
+ (when (= :read-write (settings/remote-sync-type))
+ (do-with-transforms-tracking-lock
+ (fn []
+ (if (settings/remote-sync-transforms)
+ (enable-transforms-tracking!)
+ (disable-transforms-tracking!))))))
+
;;; ----------------------------------------- Helper Functions ---------------------------------------------------------
(defn- resolve-status
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index b44bd721369..d0faa82c818 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -14,6 +14,7 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.remote-sync.spec :as spec]
[metabase-enterprise.serialization.core :as serialization]
+ [metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.analytics-interface.core :as analytics]
[metabase.api.common :as api]
[metabase.app-db.cluster-lock :as cluster-lock]
@@ -33,9 +34,9 @@
(set! *warn-on-reflection* true)
(defn- snapshot-has-transforms?
- "Checks if the snapshot contains any Transform, PythonLibrary entities,
- non-built-in TransformTags, or transforms-namespace collections.
- Used to auto-enable/disable remote-sync-transforms setting during import.
+ "Checks if the snapshot contains any Transform, PythonLibrary entities other than the built-in one with its default
+ empty source, non-built-in TransformTags, or transforms-namespace collections.
+ Used to auto-enable remote-sync-transforms setting during import.
Uses the ingestable to list all entities and checks their :model metadata,
then also checks if any Collection entities have namespace=transforms.
@@ -43,7 +44,12 @@
[ingestable]
(let [serdes-paths (serialization/ingest-list ingestable)
models-present (spec/models-in-import serdes-paths)]
- (or (some spec/transform-models models-present)
+ (or (some (disj spec/transform-models "PythonLibrary") models-present)
+ (some (fn [path]
+ (when (= "PythonLibrary" (:model (last path)))
+ (or (not= transforms-python/builtin-entity-id (:id (last path)))
+ (not (str/blank? (:source (serialization/ingest-one ingestable path)))))))
+ serdes-paths)
;; Check for non-built-in TransformTags
(when (contains? models-present "TransformTag")
(some (fn [path]
@@ -425,10 +431,6 @@
(collection/ensure-library-dashboards-collection!)
(when finalize! (finalize!))))
(report 0.9 {:force? true})
- (when (and (not has-transforms?)
- (settings/remote-sync-transforms))
- (log/info "No transforms in remote source, disabling remote-sync-transforms setting")
- (settings/remote-sync-transforms! false))
;; On H2 the reindex's table DDL blocks readers and can deadlock with them, so it must finish
;; inside the task; other app DBs keep the previous behavior of reindexing asynchronously.
(try
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
index 9417adffbd8..572d468596e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/init.clj
@@ -42,6 +42,7 @@
(defn- remote-sync-init []
;; Only stale rows: a fresh open row may belong to a worker on another node sharing this app DB.
(remote-sync.task/supersede-stale-tasks!)
+ (events/convert-legacy-transforms-root!)
(if (settings/remote-sync-enabled)
(do
(when (= :read-only (settings/remote-sync-type))
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
index 193cdf9aa59..f5835b5a8d3 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/settings.clj
@@ -1,8 +1,6 @@
(ns metabase-enterprise.remote-sync.settings
(:require
[clojure.string :as str]
- [java-time.api :as t]
- [metabase-enterprise.remote-sync.db :as remote-sync.db]
[metabase-enterprise.remote-sync.guards :as guards]
[metabase-enterprise.remote-sync.source.git :as git]
[metabase.collections.models.collection :as collection]
@@ -98,56 +96,13 @@
:encryption :no
:default 60)
-(def ^:const transforms-root-id
- "Sentinel value for the virtual Transforms root collection.
- Used to represent the entire transforms feature being enabled/disabled."
- -1)
-
-(defn- replace-transforms-root-rso!
- "Replaces the Transforms root RSO with one in `status`."
- [status]
- (remote-sync.db/delete-rso-of! "Collection" transforms-root-id)
- (remote-sync.db/insert-rso! {:model_type "Collection"
- :model_id transforms-root-id
- :model_name "Transforms"
- :status status
- :status_changed_at (t/offset-date-time)}))
-
-(defn sync-transform-tracking!
- "Records transforms sync state as the Transforms root RSO (model_id -1): 'create' when newly enabled, 'delete'
- when disabled after being tracked, left alone when already in that state (the settings cache also calls this on
- every load), and absent on a read-only instance, which has nothing to publish."
- [enabled?]
- (let [status (:status (remote-sync.db/rso "Collection" transforms-root-id))]
- (cond
- (= :read-only (remote-sync-type))
- (when status
- (remote-sync.db/delete-rso-of! "Collection" transforms-root-id))
-
- enabled?
- (when-not (#{"create" "synced"} status)
- (replace-transforms-root-rso! "create"))
-
- (and status (not= "delete" status))
- (replace-transforms-root-rso! "delete"))))
-
-(defn- sync-transform-tracking-on-change
- "Called when remote-sync-transforms setting changes."
- [_old-value new-value]
- ;; new-value may be a string "true"/"false" or a boolean, so we need to parse it
- (let [enabled? (if (string? new-value)
- (parse-boolean new-value)
- (boolean new-value))]
- (sync-transform-tracking! enabled?)))
-
(defsetting remote-sync-transforms
- (deferred-tru "Whether to sync transforms via remote-sync. When enabled, all transforms, transform tags, and transform jobs are synced as a single unit (all-or-nothing).")
+ (deferred-tru "Whether to sync transforms via remote-sync. When enabled, transforms, transform tags, transform tests, Python libraries, and transforms folders are synced.")
:type :boolean
:visibility :admin
:export? false
:encryption :no
- :default false
- :on-change sync-transform-tracking-on-change)
+ :default false)
(defsetting remote-sync-check-changes-cache-ttl-seconds
(deferred-tru "Time-to-live in seconds for the remote changes check cache. Default is 60 seconds.")
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
index 2057b44e4a4..d7cdb5be96e 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/spec.clj
@@ -73,8 +73,8 @@
:statuses - Set of statuses to check for removal (e.g., #{\"removed\" \"delete\"})
:scope-key - Optional key for scoping deletions (e.g., :collection_id, :id).
If nil, deletions are global (by entity_id only).
- :all-on-setting-disable - Optional setting keyword; when this setting's sentinel
- RSO exists with 'delete' status, remove ALL entities of this type
+ :all-on-setting-disable - Optional setting keyword; the model syncs all or nothing with
+ this setting, so an import that drops it is checked for unsynced local work
- :resources? - Optional. True when the model's YAML files carry resource files (see
`metabase.models.serialization/resource-paths`), whose changes only a full import or export
handles.
@@ -382,6 +382,7 @@
:archived-key nil ; no archived field
:tracking {:select-fields [:path]
:field-mappings {:model_name :path}}
+ :conditions {:is_default false} ; exclude the default common.py from sync
:removal-conditions {:entity_id [:not= transforms-python/builtin-entity-id]} ; protect built-in common.py from deletion
:removal {:statuses #{"removed" "delete"} ; no scope-key = global deletion
:all-on-setting-disable :remote-sync-transforms}
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
index 0327d051352..a3a6e7b211c 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
@@ -46,12 +46,18 @@
(t2/select-one :model/PythonLibrary :path path))
(mu/defn upsert-python-library-source!
- "Insert or update the PythonLibrary at `path`, setting its source to `source`. Returns the ID of the row."
- [path :- :string
- source :- :string]
+ "Insert or update the PythonLibrary at `path`, setting its source to `source` and, on insert only, the columns in
+ `insert-values`. Returns the ID of the row."
+ [path :- :string
+ source :- :string
+ insert-values :- [:map {:closed true}
+ [:entity_id {:optional true} :string]
+ [:is_default {:optional true} :boolean]]]
(mdb/update-or-insert! :model/PythonLibrary
{:path path}
- (constantly {:path path :source source})))
+ (fn [existing]
+ (cond-> {:path path :source source}
+ (nil? existing) (merge insert-values)))))
(mu/defn library-sources-by-path
"A map of path to source for every PythonLibrary."
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
index 40e60636fc0..fe8c417e960 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.transforms-python.models.python-library
(:require
+ [clojure.string :as str]
[metabase-enterprise.transforms-python.db :as transforms-python.db]
[metabase-enterprise.transforms-python.schema]
[metabase.api.common :as api]
@@ -28,9 +29,9 @@
(t2/define-before-update :model/PythonLibrary
[library]
- (if (:path library)
- (update library :path normalize-path)
- library))
+ (cond-> library
+ (seq (dissoc (t2/changes library) :updated_at :is_default)) (assoc :is_default false)
+ (:path library) (update :path normalize-path)))
(doseq [trait [:metabase/model :hook/timestamped? :hook/entity-id]]
(derive :model/PythonLibrary trait))
@@ -78,14 +79,16 @@
[path source]
(let [normalized-path (normalize-path path)]
(validate-path! normalized-path)
- (let [id (transforms-python.db/upsert-python-library-source! normalized-path source)]
+ (let [id (transforms-python.db/upsert-python-library-source!
+ normalized-path source {:entity_id builtin-entity-id :is_default (str/blank? source)})]
(transforms-python.db/python-library id))))
;;; ------------------------------------------------- Serialization --------------------------------------------------
(defmethod serdes/make-spec "PythonLibrary"
[_model-name _opts]
- {:copy [:path :source :entity_id]})
+ {:copy [:path :source :entity_id]
+ :skip [:is_default]})
(defmethod serdes/storage-path "PythonLibrary" [entity _ctx]
[{:label "python-libraries"} {:label (:path entity) :key (:entity_id entity)}])
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj
index 3c04089e910..d517023c139 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/schema.clj
@@ -18,4 +18,5 @@
[:source {:optional true} [:maybe [:or :keyword :string]]]
[:created_at {:optional true} [:maybe ms/TemporalInstant]]
[:updated_at {:optional true} [:maybe ms/TemporalInstant]]
- [:entity_id {:optional true} [:maybe :string]]])
+ [:entity_id {:optional true} [:maybe :string]]
+ [:is_default {:optional true} [:maybe :boolean]]])
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
index 0af73182109..0986c49c07a 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/api_test.clj
@@ -2286,53 +2286,18 @@
(is (true? (settings/remote-sync-transforms))
"Transforms setting should be preserved when not included in request")))))))
-;;; ------------------------------------------- Dirty Endpoint with Transforms Root Tests -------------------------------------------
-
-(deftest dirty-returns-transforms-root-collection-test
- (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection (id=-1) when present"
- (test-helpers/with-clean-object
- (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
- :model_id settings/transforms-root-id
- :model_name "Transforms"
- :status "create"
- :status_changed_at (java.time.OffsetDateTime/now)}]
- (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
- dirty-items (:dirty response)]
- (is (= 1 (count dirty-items)))
- (is (= "Transforms" (:name (first dirty-items))))
- (is (= "collection" (:model (first dirty-items))))
- (is (= settings/transforms-root-id (:id (first dirty-items)))))))))
-
-(deftest dirty-returns-transforms-root-collection-with-delete-status-test
- (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection (id=-1) with delete status"
- (test-helpers/with-clean-object
- (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
- :model_id settings/transforms-root-id
- :model_name "Transforms"
- :status "delete"
- :status_changed_at (java.time.OffsetDateTime/now)}]
- (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
- dirty-items (:dirty response)]
- (is (= 1 (count dirty-items)))
- (is (= "Transforms" (:name (first dirty-items))))
- (is (= "collection" (:model (first dirty-items))))
- (is (= "delete" (:sync_status (first dirty-items))))
- (is (= settings/transforms-root-id (:id (first dirty-items)))))))))
-
-(deftest dirty-returns-transforms-root-collection-when-setting-disabled-test
- (testing "GET /api/ee/remote-sync/dirty returns the Transforms root collection when transforms setting is disabled"
+(deftest dirty-returns-removed-transform-when-setting-disabled-test
+ (testing "GET /api/ee/remote-sync/dirty lists a transform pending removal while transforms sync is disabled"
(test-helpers/with-clean-object
(mt/with-temporary-setting-values [remote-sync-transforms false]
- (mt/with-temp [:model/RemoteSyncObject _ {:model_type "Collection"
- :model_id settings/transforms-root-id
- :model_name "Transforms"
- :status "delete"
- :status_changed_at (java.time.OffsetDateTime/now)}]
- (let [response (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")
- dirty-items (:dirty response)]
- (is (= 1 (count dirty-items)) "Transforms root should be returned even when setting is disabled")
- (is (= "Transforms" (:name (first dirty-items))))
- (is (= "delete" (:sync_status (first dirty-items))))))))))
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "Removed Transform"}
+ :model/RemoteSyncObject _ {:model_type "Transform"
+ :model_id transform-id
+ :model_name "Removed Transform"
+ :status "removed"
+ :status_changed_at (t/offset-date-time)}]
+ (is (=? [{:id transform-id :model "transform" :name "Removed Transform" :sync_status "removed"}]
+ (:dirty (mt/user-http-request :crowberto :get 200 "ee/remote-sync/dirty")))))))))
(deftest settings-collections-not-marked-synced-when-settings-validation-fails-test
(testing "PUT /api/ee/remote-sync/settings does not mark collections as synced when settings validation fails"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 089c96b9abb..fb987714a16 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -1281,8 +1281,8 @@ serdes/meta:
(is (false? (remote-sync.settings/remote-sync-transforms))
"remote-sync-transforms should remain disabled when no transforms in remote")))))))))
-(deftest import!-disables-transforms-setting-when-no-transforms-in-branch-test
- (testing "import! disables remote-sync-transforms setting when importing a branch without transforms"
+(deftest import!-keeps-transforms-setting-enabled-when-no-transforms-in-branch-test
+ (testing "import! never turns remote-sync-transforms off, even for a branch without transforms"
(mt/with-model-cleanup [:model/RemoteSyncTask]
(let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})]
(mt/with-temporary-setting-values [remote-sync-transforms true]
@@ -1294,8 +1294,8 @@ serdes/meta:
"remote-sync-transforms should be initially enabled")
(let [result (impl/import! (source.p/snapshot mock-source) task-id)]
(is (= :success (:status result)))
- (is (false? (remote-sync.settings/remote-sync-transforms))
- "remote-sync-transforms should be disabled when no transforms in remote")))))))))
+ (is (true? (remote-sync.settings/remote-sync-transforms))
+ "remote-sync-transforms should stay enabled when no transforms in remote")))))))))
(deftest import!-includes-all-optional-paths-regardless-of-settings-test
(testing "import! always includes all optional paths (transforms, python-libraries, snippets)"
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
index 6c0f548d04b..25e84dddcf7 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/spec_test.clj
@@ -584,18 +584,18 @@
(is (nil? (spec/removal-conditions {})))))
(deftest python-library-spec-conditions-test
- (testing "PythonLibrary spec has :removal-conditions but no :conditions or :export-conditions"
+ (testing "PythonLibrary spec has :conditions and :removal-conditions but no :export-conditions"
(let [spec (spec/spec-for-model-key :model/PythonLibrary)]
- (is (nil? (:conditions spec))
- "PythonLibrary should have no :conditions")
+ (is (= {:is_default false} (:conditions spec))
+ "PythonLibrary should track and export only libraries that aren't the default one")
(is (nil? (:export-conditions spec))
"PythonLibrary should have no :export-conditions")
(is (= {:entity_id [:not= transforms-python/builtin-entity-id]}
(:removal-conditions spec))
"PythonLibrary should have :removal-conditions protecting builtin entity")))
- (testing "export-conditions returns nil for PythonLibrary (no export filtering)"
+ (testing "export-conditions leaves the default library out of exports"
(let [spec (spec/spec-for-model-key :model/PythonLibrary)]
- (is (nil? (spec/export-conditions spec)))))
+ (is (= {:is_default false} (spec/export-conditions spec)))))
(testing "removal-conditions returns the builtin protection for PythonLibrary"
(let [spec (spec/spec-for-model-key :model/PythonLibrary)]
(is (= {:entity_id [:not= transforms-python/builtin-entity-id]}
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 3457bbfecf6..4bceda44d5d 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -7,6 +7,7 @@
[metabase-enterprise.remote-sync.source.protocol :as source.p]
[metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.transforms-python.core :as transforms-python]
+ [metabase.test :as mt]
[metabase.test.util.thread-local :as tu.thread-local]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -383,7 +384,7 @@ width: fixed
(def ^:private builtin-python-library
"The built-in PythonLibrary created by migration. Recreated by fixture after cleanup."
- {:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id})
+ {:path "common.py" :source "" :entity_id transforms-python/builtin-entity-id :is_default true})
(defn- ensure-builtin-python-library!
"Ensures the built-in common.py PythonLibrary exists, creating it if missing."
@@ -417,10 +418,17 @@ width: fixed
(when (seq old-ns-colls) (t2/insert! :model/Collection old-ns-colls))
(ensure-builtin-python-library!)))))
+(defn reset-transforms-setting
+ "Test fixture that runs with remote-sync-transforms off, since a pull turns it on and nothing turns it off."
+ [f]
+ (mt/with-temporary-setting-values [remote-sync-transforms false]
+ (f)))
+
(def clean-remote-sync-state
- "Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, and optional feature
- model tables (Transform, TransformTag, PythonLibrary) are clean."
- (t/compose-fixtures clean-object (t/compose-fixtures clean-task-table clean-optional-feature-models)))
+ "Composed test fixture that ensures RemoteSyncObject, RemoteSyncTask, optional feature model tables (Transform,
+ TransformTag, PythonLibrary), and the remote-sync-transforms setting are clean."
+ (t/compose-fixtures (t/compose-fixtures reset-transforms-setting clean-object)
+ (t/compose-fixtures clean-task-table clean-optional-feature-models)))
(defn commit-with-temp
"Test fixture (`:each`) that makes `with-temp` COMMIT its rows instead of wrapping the test body in a
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
index c5a1c7f93ce..a58a4d90c91 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/transforms_test.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase-enterprise.remote-sync.events :as remote-sync.events]
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-object :as sync-object]
[metabase-enterprise.remote-sync.settings :as settings]
@@ -155,100 +156,295 @@
:model_id (:id coll)))
"Transforms-namespace collection should NOT be tracked when setting is disabled"))))))
-(deftest enable-transform-sync-marks-all-transforms-test
- (testing "Enabling transform sync creates a single Transforms RSO with model_id=-1 and status 'create'"
+(defn- insert-rso!
+ "Inserts a RemoteSyncObject for `model-type` `model-id` in `status`."
+ [model-type model-id status]
+ (t2/insert! :model/RemoteSyncObject {:model_type model-type :model_id model-id :model_name "T"
+ :status status :status_changed_at (t/offset-date-time)}))
+
+(defn- rso-status
+ "The status of the RemoteSyncObject of `model-type` `model-id`, or nil when there is none."
+ [model-type model-id]
+ (t2/select-one-fn :status :model/RemoteSyncObject :model_type model-type :model_id model-id))
+
+(defn- transforms-rso-count
+ "The number of RemoteSyncObjects of the models that sync with transforms."
+ []
+ (t2/count :model/RemoteSyncObject :model_type [:in ["Transform" "TransformTag" "TransformTest" "PythonLibrary"]]))
+
+(defn- reset-default-python-library!
+ "Replaces the built-in Python library with the default one the migrations create."
+ []
+ (t2/delete! :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)
+ (t2/insert! :model/PythonLibrary {:path "common.py"
+ :source ""
+ :entity_id transforms-python/builtin-entity-id
+ :is_default true}))
+
+(defn- edit-default-python-library!
+ "Replaces the built-in Python library with an edited one, returning its ID."
+ []
+ (reset-default-python-library!)
+ (t2/update! :model/PythonLibrary :entity_id transforms-python/builtin-entity-id {:source "def f(): pass"})
+ (t2/select-one-pk :model/PythonLibrary :entity_id transforms-python/builtin-entity-id))
+
+(deftest enable-transforms-tracking-creates-rows-test
+ (testing "Enabling transforms tracking records a 'create' row for each untracked piece of transforms content"
(mt/with-premium-features #{:transforms-basic}
- (mt/with-temporary-setting-values [remote-sync-enabled true
- remote-sync-type :read-write]
- ;; Clean up any existing Transforms root RSO first
- (t2/delete! :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id)
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}
- :model/Transform _transform {:name "Existing Transform" :collection_id coll-id}
- :model/TransformTag _tag {:name "Existing Tag"}]
- (is (not (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id))
- "Should not have Transforms root RSO initially")
- (settings/sync-transform-tracking! true)
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "create")
- "Transforms root should be created with status 'create'")
- (let [rso (t2/select-one :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id)]
- (is (= "Transforms" (:model_name rso))
- "Transforms root should have name 'Transforms'")))))))
-
-(deftest disable-transform-sync-removes-all-tracking-test
- (testing "Disabling transform sync creates a Transforms RSO with status 'delete'"
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {folder-id :id} {:name "ETL" :namespace collection/transforms-ns}
+ :model/Transform {transform-id :id} {:name "Untracked" :collection_id folder-id}
+ :model/Transform {tracked-id :id} {:name "Tracked" :collection_id folder-id}
+ :model/TransformTag {tag-id :id} {:name "Custom Tag"}
+ :model/TransformTest {test-id :id} {:transform_id transform-id :name "No nulls"}]
+ (let [library-id (edit-default-python-library!)]
+ (insert-rso! "Transform" tracked-id "synced")
+ (settings/remote-sync-transforms! true)
+ (remote-sync.events/enable-transforms-tracking!)
+ (is (= {["Collection" folder-id] "create"
+ ["Transform" transform-id] "create"
+ ["Transform" tracked-id] "synced"
+ ["TransformTag" tag-id] "create"
+ ["TransformTest" test-id] "create"
+ ["PythonLibrary" library-id] "create"}
+ (into {}
+ (map (juxt (juxt :model_type :model_id) :status))
+ (t2/select :model/RemoteSyncObject))))))))))
+
+(deftest enable-transforms-tracking-skips-default-content-test
+ (testing "Enabling transforms tracking with only built-in tags and the default Python library records nothing"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true]
+ (reset-default-python-library!)
+ (is (t2/exists? :model/TransformTag :built_in_type [:not= nil]))
+ (remote-sync.events/enable-transforms-tracking!)
+ (is (not (t2/exists? :model/RemoteSyncObject))))))
+
+(deftest disable-transforms-tracking-test
+ (testing "Disabling transforms tracking drops rows never pushed and marks pushed ones removed"
(mt/with-premium-features #{:transforms-basic}
- (mt/with-temporary-setting-values [remote-sync-transforms false
- remote-sync-enabled true
- remote-sync-type :read-write]
- (mt/with-temp [:model/Collection {coll-id :id} {:name "Transforms Collection" :namespace collection/transforms-ns}
- :model/Transform _transform {:name "Test Transform" :collection_id coll-id}
- :model/TransformTag _tag {:name "Test Tag"}]
- (mt/with-temporary-setting-values [remote-sync-transforms true]
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "create")
- "Should have Transforms root RSO with 'create' status")
- (settings/sync-transform-tracking! false)
- (is (t2/exists? :model/RemoteSyncObject
- :model_type "Collection"
- :model_id settings/transforms-root-id
- :status "delete")
- "Transforms root should now have 'delete' status")))))))
-
-(defn- transforms-root-status
- "The status of the Transforms root RSO, or nil when there is none."
- []
- (t2/select-one-fn :status :model/RemoteSyncObject :model_type "Collection" :model_id settings/transforms-root-id))
-
-(deftest transform-tracking-keeps-tracked-root-test
- (testing "Re-applying an enabled transforms setting, as every settings cache load does, keeps a synced root synced"
- (mt/with-temporary-setting-values [remote-sync-type :read-write]
- (settings/sync-transform-tracking! true)
- (t2/update! :model/RemoteSyncObject {:model_type "Collection" :model_id settings/transforms-root-id}
- {:status "synced"})
- (settings/sync-transform-tracking! true)
- (is (= "synced" (transforms-root-status)))
- (testing "and disabling then re-enabling marks it again"
- (settings/sync-transform-tracking! false)
- (is (= "delete" (transforms-root-status)))
- (settings/sync-transform-tracking! true)
- (is (= "create" (transforms-root-status)))))))
-
-(deftest transform-tracking-read-only-test
- (testing "A read-only instance never tracks the Transforms root, so it never reports it as unsaved"
- (mt/with-temporary-setting-values [remote-sync-type :read-only]
- (settings/sync-transform-tracking! true)
- (is (nil? (transforms-root-status)))
- (is (not (sync-object/dirty?)))
- (testing "and drops one left over from before the fix"
- (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id settings/transforms-root-id
- :model_name "Transforms" :status "create"
- :status_changed_at (t/offset-date-time)})
- (settings/sync-transform-tracking! true)
- (is (nil? (transforms-root-status)))))))
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Collection {folder-id :id} {:name "ETL" :namespace collection/transforms-ns}
+ :model/Collection {coll-id :id} {:name "Library" :is_remote_synced true}
+ :model/Transform {synced-id :id} {:name "Synced" :collection_id folder-id}
+ :model/Transform {update-id :id} {:name "Updated" :collection_id folder-id}
+ :model/Transform {create-id :id} {:name "Created" :collection_id folder-id}
+ :model/TransformTag {tag-id :id} {:name "Custom Tag"}
+ :model/TransformTest {test-id :id} {:transform_id synced-id :name "No nulls"}]
+ (insert-rso! "Collection" folder-id "synced")
+ (insert-rso! "Collection" coll-id "synced")
+ (insert-rso! "Transform" synced-id "synced")
+ (insert-rso! "Transform" update-id "update")
+ (insert-rso! "Transform" create-id "create")
+ (insert-rso! "TransformTag" tag-id "create")
+ (insert-rso! "TransformTest" test-id "delete")
+ (remote-sync.events/disable-transforms-tracking!)
+ (is (= {["Collection" folder-id] "removed"
+ ["Collection" coll-id] "synced"
+ ["Transform" synced-id] "removed"
+ ["Transform" update-id] "removed"
+ ["TransformTest" test-id] "delete"}
+ (into {}
+ (map (juxt (juxt :model_type :model_id) :status))
+ (t2/select :model/RemoteSyncObject)))))))))
+
+(deftest removed-transforms-rows-stay-dirty-when-sync-disabled-test
+ (testing "Pushed transforms rows that disabling marks removed still count as dirty, so their removal gets pushed"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "Synced"}]
+ (insert-rso! "Transform" transform-id "synced")
+ (is (not (sync-object/dirty?)))
+ (remote-sync.events/disable-transforms-tracking!)
+ (is (= "removed" (rso-status "Transform" transform-id)))
+ (is (sync-object/dirty?)))))))
+
+(deftest re-enabling-transforms-tracking-restores-removed-rows-test
+ (testing "Re-enabling transforms tracking before a push turns removed rows back into updates"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true]
+ (mt/with-temp [:model/Transform {synced-id :id} {:name "Synced"}
+ :model/Transform {created-id :id} {:name "Created"}]
+ (insert-rso! "Transform" synced-id "synced")
+ (insert-rso! "Transform" created-id "create")
+ (remote-sync.events/disable-transforms-tracking!)
+ (remote-sync.events/enable-transforms-tracking!)
+ (is (= "update" (rso-status "Transform" synced-id)))
+ (is (= "create" (rso-status "Transform" created-id)))))))
+
+(deftest re-enabling-transforms-tracking-deletes-removed-rows-of-deleted-content-test
+ (testing "Re-enabling transforms tracking turns the removed row of content deleted while sync was off into a delete"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true]
+ (mt/with-temp [:model/Transform {kept-id :id} {:name "Kept"}
+ :model/Transform {deleted-id :id} {:name "Deleted"}]
+ (insert-rso! "Transform" kept-id "synced")
+ (insert-rso! "Transform" deleted-id "synced")
+ (remote-sync.events/disable-transforms-tracking!)
+ (t2/delete! :model/Transform :id deleted-id)
+ (remote-sync.events/enable-transforms-tracking!)
+ (is (= "update" (rso-status "Transform" kept-id)))
+ (is (= "delete" (rso-status "Transform" deleted-id)))))))
+
+(deftest enable-transforms-tracking-records-only-exported-content-test
+ (testing "Enabling transforms tracking records rows only for what an export writes, and folder rows carry their own id"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-type :read-write
+ remote-sync-transforms true]
+ (mt/with-temp [:model/Collection {folder-id :id} {:name "ETL" :namespace collection/transforms-ns}
+ :model/Collection {archived-id :id} {:name "Old" :namespace collection/transforms-ns :archived true}
+ :model/Transform {live-id :id} {:name "Live" :collection_id folder-id}
+ :model/Transform {stale-id :id} {:name "Stale" :collection_id archived-id}]
+ (remote-sync.events/enable-transforms-tracking!)
+ (is (=? {:status "create" :model_collection_id folder-id}
+ (t2/select-one :model/RemoteSyncObject :model_type "Collection" :model_id folder-id)))
+ (is (= "create" (rso-status "Transform" live-id)))
+ (is (nil? (rso-status "Collection" archived-id)))
+ (is (nil? (rso-status "Transform" stale-id))))))))
+
+(deftest disable-enable-disable-keeps-pushed-rows-removed-test
+ (testing "Disabling, re-enabling, and disabling transforms tracking again before a push still removes pushed files"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write remote-sync-transforms true]
+ (mt/with-temp [:model/Transform {synced-id :id} {:name "Synced"}
+ :model/Transform {created-id :id} {:name "Created"}]
+ (insert-rso! "Transform" synced-id "synced")
+ (insert-rso! "Transform" created-id "create")
+ (remote-sync.events/disable-transforms-tracking!)
+ (remote-sync.events/enable-transforms-tracking!)
+ (remote-sync.events/disable-transforms-tracking!)
+ (is (= "removed" (rso-status "Transform" synced-id)))
+ (is (nil? (rso-status "Transform" created-id)))))))
+
+(deftest sync-transforms-tracking-follows-setting-test
+ (testing "Syncing transforms tracking on a read-write instance enables or disables it according to the setting"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "T"}]
+ (settings/remote-sync-transforms! true)
+ (remote-sync.events/sync-transforms-tracking!)
+ (is (= "create" (rso-status "Transform" transform-id)))
+ (settings/remote-sync-transforms! false)
+ (remote-sync.events/sync-transforms-tracking!)
+ (is (nil? (rso-status "Transform" transform-id)))))))
+
+(deftest setting-transforms-directly-tracks-nothing-test
+ (testing "Changing the remote-sync-transforms setting on its own neither tracks nor untracks transforms content"
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Transform {untracked-id :id} {:name "Untracked"}
+ :model/Transform {tracked-id :id} {:name "Tracked"}]
+ (insert-rso! "Transform" tracked-id "synced")
+ (settings/remote-sync-transforms! true)
+ (is (nil? (rso-status "Transform" untracked-id)))
+ (settings/remote-sync-transforms! false)
+ (is (= "synced" (rso-status "Transform" tracked-id)))))))
+
+(deftest sync-transforms-tracking-read-only-test
+ (testing "A read-only instance publishes nothing, so syncing transforms tracking leaves the rows alone"
+ (mt/with-temporary-setting-values [remote-sync-type :read-only
+ remote-sync-transforms false]
+ (mt/with-temp [:model/Transform {untracked-id :id} {:name "Untracked"}
+ :model/Transform {tracked-id :id} {:name "Tracked"}]
+ (insert-rso! "Transform" tracked-id "synced")
+ (settings/remote-sync-transforms! true)
+ (remote-sync.events/sync-transforms-tracking!)
+ (is (nil? (rso-status "Transform" untracked-id)))
+ (settings/remote-sync-transforms! false)
+ (remote-sync.events/sync-transforms-tracking!)
+ (is (= "synced" (rso-status "Transform" tracked-id)))
+ (is (not (sync-object/dirty?)))))))
+
+(deftest settings-api-drives-transforms-tracking-test
+ (testing "Toggling remote-sync-transforms through PUT /api/ee/remote-sync/settings tracks and untracks transforms"
+ (mt/with-premium-features #{:remote-sync}
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-dynamic-fn-redefs [settings/check-git-settings! (constantly nil)
+ impl/finish-remote-config! (constantly nil)]
+ (mt/with-temp [:model/Transform {created-id :id} {:name "Created"}
+ :model/Transform {synced-id :id} {:name "Synced"}]
+ (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings" {:remote-sync-transforms true})
+ (is (= "create" (rso-status "Transform" created-id)))
+ (t2/update! :model/RemoteSyncObject {:model_type "Transform" :model_id synced-id} {:status "synced"})
+ (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings" {:remote-sync-transforms false})
+ (is (nil? (rso-status "Transform" created-id)))
+ (is (= "removed" (rso-status "Transform" synced-id)))))))))
(deftest switching-to-read-only-with-transforms-test
(testing "Saving read-only settings with transforms on, as the setup guide does, leaves nothing unsaved"
(mt/with-premium-features #{:remote-sync}
- (mt/with-temporary-setting-values [remote-sync-type :read-write
+ (mt/with-temporary-setting-values [remote-sync-type :read-write
remote-sync-transforms false]
(mt/with-dynamic-fn-redefs [settings/check-git-settings! (constantly nil)
- impl/finish-remote-config! (constantly nil)]
- (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
- {:remote-sync-type :read-only :remote-sync-transforms true})
- (is (true? (settings/remote-sync-transforms)))
- (is (nil? (transforms-root-status)))
- (is (not (sync-object/dirty?))))))))
+ impl/finish-remote-config! (constantly nil)]
+ (mt/with-temp [:model/Transform _ {:name "T"}]
+ (mt/user-http-request :crowberto :put 200 "ee/remote-sync/settings"
+ {:remote-sync-type :read-only :remote-sync-transforms true})
+ (is (true? (settings/remote-sync-transforms)))
+ (is (zero? (transforms-rso-count)))
+ (is (not (sync-object/dirty?)))))))))
+
+(defn- insert-legacy-transforms-root!
+ "Inserts the legacy Transforms root RemoteSyncObject in `status`."
+ [status]
+ (t2/insert! :model/RemoteSyncObject {:model_type "Collection" :model_id -1 :model_name "Transforms"
+ :status status :status_changed_at (t/offset-date-time)}))
+
+(defn- legacy-transforms-root-exists?
+ "Whether the legacy Transforms root RemoteSyncObject exists."
+ []
+ (t2/exists? :model/RemoteSyncObject :model_type "Collection" :model_id -1))
+
+(deftest convert-legacy-transforms-root-pending-publish-test
+ (testing "A legacy Transforms root pending publish becomes a 'create' row for each untracked piece of content"
+ (mt/with-temporary-setting-values [remote-sync-transforms true]
+ (doseq [status ["create" "update"]]
+ (testing status
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "T"}]
+ (insert-legacy-transforms-root! status)
+ (remote-sync.events/convert-legacy-transforms-root!)
+ (is (= "create" (rso-status "Transform" transform-id)))
+ (is (not (legacy-transforms-root-exists?)))
+ (t2/delete! :model/RemoteSyncObject)))))))
+
+(deftest convert-legacy-transforms-root-concurrently-test
+ (testing "Nodes converting the legacy Transforms root at the same time record one row per piece of content"
+ (mt/with-temporary-setting-values [remote-sync-transforms true]
+ (mt/with-temp [:model/Transform {transform-id :id} {:name "T"}]
+ (insert-legacy-transforms-root! "create")
+ (let [start (promise)
+ nodes (doall (repeatedly 4 #(future @start (remote-sync.events/convert-legacy-transforms-root!))))]
+ (deliver start true)
+ (run! deref nodes))
+ (is (= 1 (t2/count :model/RemoteSyncObject :model_type "Transform" :model_id transform-id)))
+ (is (not (legacy-transforms-root-exists?)))))))
+
+(deftest convert-legacy-transforms-root-pending-delete-test
+ (testing "A legacy Transforms root pending deletion marks the pushed transforms removed and drops the others"
+ (mt/with-temp [:model/Transform {synced-id :id} {:name "Synced"}
+ :model/Transform {created-id :id} {:name "Created"}]
+ (insert-rso! "Transform" synced-id "synced")
+ (insert-rso! "Transform" created-id "create")
+ (insert-legacy-transforms-root! "delete")
+ (remote-sync.events/convert-legacy-transforms-root!)
+ (is (= "removed" (rso-status "Transform" synced-id)))
+ (is (nil? (rso-status "Transform" created-id)))
+ (is (not (legacy-transforms-root-exists?))))))
+
+(deftest convert-legacy-transforms-root-synced-test
+ (testing "A synced legacy Transforms root is dropped and leaves the transforms tracking as it was"
+ (mt/with-temp [:model/Transform {synced-id :id} {:name "Synced"}
+ :model/Transform {untracked-id :id} {:name "Untracked"}]
+ (insert-rso! "Transform" synced-id "synced")
+ (insert-legacy-transforms-root! "synced")
+ (remote-sync.events/convert-legacy-transforms-root!)
+ (is (= "synced" (rso-status "Transform" synced-id)))
+ (is (nil? (rso-status "Transform" untracked-id)))
+ (is (not (legacy-transforms-root-exists?))))))
(deftest transforms-included-in-dirty-check-when-enabled-test
(testing "Transforms are included in dirty check when setting is enabled"
@@ -1213,3 +1409,72 @@ serdes/meta:
(is (= :success (:status (impl/export! (source.p/snapshot mock) (new-export-task!) "rename"))))
(is (str/ends-with? (test-path) "/etl/orders_report/no_nulls.yaml"))
(is (not (contains? (get @(:files-atom mock) "main") old-path)))))))))))
+
+(deftest enabling-transforms-sync-with-default-content-test
+ (testing "Enabling transforms sync with only built-in tags and the default Python library leaves nothing to push, and a
+ pull of a branch with transforms then has no conflicts"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-type :read-write
+ remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Transform :model/Collection]
+ (reset-default-python-library!)
+ (is (t2/exists? :model/TransformTag :built_in_type [:not= nil]))
+ (settings/remote-sync-transforms! true)
+ (remote-sync.events/sync-transforms-tracking!)
+ (is (not (t2/exists? :model/RemoteSyncObject)))
+ (is (not (sync-object/dirty?)))
+ (let [coll-eid (u/generate-nano-id)
+ files {"main" {"collections/transforms/remote_transforms/remote_transforms.yaml"
+ (generate-transforms-namespace-collection-yaml coll-eid "Remote Transforms")
+ "collections/transforms/remote_transforms/remote_transform.yaml"
+ (test-helpers/generate-transform-yaml (u/generate-nano-id) "Remote Transform"
+ :collection-id coll-eid)}}
+ task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import"
+ :initiated_by (mt/user->id :rasta)})]
+ (is (=? {:status :success}
+ (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files files))
+ task-id)))))))))
+
+(deftest saving-default-python-library-unchanged-keeps-it-default-test
+ (testing "Saving the default Python library unchanged keeps it the default one"
+ (reset-default-python-library!)
+ (t2/update! :model/PythonLibrary :entity_id transforms-python/builtin-entity-id {:source ""})
+ (is (true? (t2/select-one-fn :is_default :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)))))
+
+(deftest edited-python-library-is-not-default-test
+ (testing "Editing the default Python library makes it content to publish"
+ (edit-default-python-library!)
+ (is (false? (t2/select-one-fn :is_default :model/PythonLibrary :entity_id transforms-python/builtin-entity-id)))))
+
+(defn- import-files!
+ "Imports `files` on the main branch from a mock source, returning the result."
+ [files]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import"
+ :initiated_by (mt/user->id :rasta)})]
+ (impl/import! (source.p/snapshot (test-helpers/create-mock-source :initial-files {"main" files})) task-id)))
+
+(deftest import-ignores-default-builtin-python-library-test
+ (testing "A pull of a branch whose only transforms content is the blank built-in common.py leaves transforms sync off"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]
+ (reset-default-python-library!)
+ (is (=? {:status :success}
+ (import-files! {"python_libraries/common.py.yaml"
+ (generate-python-library-yaml transforms-python/builtin-entity-id "common.py" "")})))
+ (is (false? (settings/remote-sync-transforms))))))))
+
+(deftest import-enables-transforms-sync-for-edited-builtin-python-library-test
+ (testing "A pull of a branch whose built-in common.py has code turns transforms sync on"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [remote-sync-enabled true
+ remote-sync-transforms false]
+ (mt/with-model-cleanup [:model/RemoteSyncTask :model/Collection]
+ (reset-default-python-library!)
+ (is (=? {:status :success}
+ (import-files! {"python_libraries/common.py.yaml"
+ (generate-python-library-yaml transforms-python/builtin-entity-id "common.py"
+ "def shared_func():\n return 42")})))
+ (is (true? (settings/remote-sync-transforms))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/models/python_library_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/models/python_library_test.clj
index 5fae0000ae0..a7ed5f0bdd8 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/models/python_library_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/models/python_library_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase-enterprise.transforms-python.models.python-library :as python-library]
[metabase.test.fixtures :as fixtures]
+ [metabase.util :as u]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -74,3 +75,22 @@
(is (=? {:source "def test(): pass"
:path "common.py"}
(python-library/get-python-library-by-path "common"))))))
+
+(deftest is-default-cleared-by-a-source-change-test
+ (testing "Saving the default library with its source unchanged keeps it the default one"
+ (t2/delete! :model/PythonLibrary)
+ (t2/insert! :model/PythonLibrary {:path "common.py" :source "" :entity_id (u/generate-nano-id) :is_default true})
+ (python-library/update-python-library-source! "common" "")
+ (is (true? (t2/select-one-fn :is_default :model/PythonLibrary :path "common.py"))))
+ (testing "while changing its source makes it no longer the default one"
+ (python-library/update-python-library-source! "common" "def f(): pass")
+ (is (false? (t2/select-one-fn :is_default :model/PythonLibrary :path "common.py")))))
+
+(deftest recreated-library-is-the-built-in-one-test
+ (testing "A recreated common.py takes the built-in entity_id, and is the default one only with an empty source"
+ (doseq [[source default?] [["" true] ["def f(): pass" false]]]
+ (testing (pr-str source)
+ (t2/delete! :model/PythonLibrary)
+ (python-library/update-python-library-source! "common" source)
+ (is (=? {:entity_id python-library/builtin-entity-id :is_default default?}
+ (t2/select-one :model/PythonLibrary :path "common.py")))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
index 2023e610071..b1c562a2e4a 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.tsx
@@ -39,13 +39,12 @@ interface AllChangesViewProps {
export const AllChangesView = ({ entities, title }: AllChangesViewProps) => {
const isUsingTenants = useSetting("use-tenants");
- const isTransformsSyncEnabled = useSetting("remote-sync-transforms");
const { data: collectionTree = [] } = useListCollectionsTreeQuery({
namespaces: [
"",
"analytics",
...(isUsingTenants ? ["shared-tenant-collection"] : []),
- ...(isTransformsSyncEnabled ? ["transforms"] : []),
+ "transforms",
],
"include-library": true,
});
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
index e92a82d34b8..38563bdb7db 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/components/ChangesLists/AllChangesView.unit.spec.tsx
@@ -1,8 +1,10 @@
+import fetchMock from "fetch-mock";
+
import { setupEnterpriseOnlyPlugin } from "__support__/enterprise";
import { setupCollectionTreeEndpoint } from "__support__/server-mocks";
import { mockSettings } from "__support__/settings";
import { createMockState } from "__support__/state";
-import { renderWithProviders, screen } from "__support__/ui";
+import { renderWithProviders, screen, waitFor } from "__support__/ui";
import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/displayGroups";
import type { Collection, RemoteSyncEntity } from "metabase-types/api";
import {
@@ -187,6 +189,31 @@ describe("AllChangesView", () => {
expect(screen.getByText("Transform in Collection")).toBeInTheDocument();
});
+ it("should load transforms collections while remote-sync-transforms is off, so removed transforms keep their folders", async () => {
+ const removedTransform = createMockRemoteSyncEntity({
+ id: 200,
+ name: "Removed Transform",
+ model: "transform",
+ collection_id: 100,
+ sync_status: "removed",
+ });
+
+ setup({
+ entities: [removedTransform],
+ collections: [],
+ isTransformsSyncEnabled: false,
+ });
+
+ await waitFor(() => {
+ const namespaces = fetchMock.callHistory
+ .calls("collection-tree-simple")
+ .map((call) => new URL(call.url).searchParams.getAll("namespaces"));
+ expect(namespaces).toContainEqual(
+ expect.arrayContaining(["transforms"]),
+ );
+ });
+ });
+
it("should display transforms collection hierarchy when remote-sync-transforms is enabled", async () => {
const childTransformsCollection = createMockCollection({
id: 101,
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
index 3c504f88597..dab66a78ea2 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.ts
@@ -3,46 +3,31 @@ import { useMemo } from "react";
import { useListCollectionsTreeQuery } from "metabase/api";
import { buildCollectionTree } from "metabase/common/collections/utils";
import { getAllDescendantIds } from "metabase/common/components/tree/utils";
-import { useSetting } from "metabase/settings";
-import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/utils";
import { useGitSyncVisible } from "./use-git-sync-visible";
import { useRemoteSyncDirtyState } from "./use-remote-sync-dirty-state";
+const TRANSFORM_MODELS = new Set([
+ "transform",
+ "transformtag",
+ "transformtest",
+ "pythonlibrary",
+]);
+
export function useHasTransformDirtyChanges(): boolean {
const { isVisible: isGitSyncVisible } = useGitSyncVisible();
- const transformsSetting = useSetting("remote-sync-transforms");
const { dirty, isDirty } = useRemoteSyncDirtyState();
- // Fetch transforms-namespace collections to check for dirty collections
const { data: transformsCollections = [] } = useListCollectionsTreeQuery(
{ namespace: "transforms" },
- { skip: !isGitSyncVisible || !transformsSetting },
+ { skip: !isGitSyncVisible },
);
return useMemo(() => {
- // Only show if git sync is visible and there are dirty entities
if (!isGitSyncVisible || !isDirty) {
return false;
}
- // Check for Transforms root collection (sentinel value id=-1)
- // This needs to show even when transforms setting is off (to show pending delete)
- const hasTransformsRootChange = dirty.some(
- (entity) =>
- entity.model === "collection" && entity.id === TRANSFORMS_ROOT_ID,
- );
-
- if (hasTransformsRootChange) {
- return true;
- }
-
- // For other transform-related changes, only show if transforms setting is enabled
- if (!transformsSetting) {
- return false;
- }
-
- // Build set of transforms-namespace collection IDs
const transformsTree = buildCollectionTree(transformsCollections);
const transformsCollectionIds = getAllDescendantIds(transformsTree);
const numericCollectionIds = new Set(
@@ -51,31 +36,10 @@ export function useHasTransformDirtyChanges(): boolean {
),
);
- // Check if any dirty entity is:
- // 1. A transform, transform tag, transform test, or Python library
- // 2. A collection in the transforms namespace
- return dirty.some((entity) => {
- if (
- entity.model === "transform" ||
- entity.model === "transformtag" ||
- entity.model === "transformtest" ||
- entity.model === "pythonlibrary"
- ) {
- return true;
- }
- if (
- entity.model === "collection" &&
- numericCollectionIds.has(entity.id)
- ) {
- return true;
- }
- return false;
- });
- }, [
- isGitSyncVisible,
- transformsSetting,
- dirty,
- isDirty,
- transformsCollections,
- ]);
+ return dirty.some(
+ (entity) =>
+ TRANSFORM_MODELS.has(entity.model) ||
+ (entity.model === "collection" && numericCollectionIds.has(entity.id)),
+ );
+ }, [isGitSyncVisible, dirty, isDirty, transformsCollections]);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
index 92cb090f9fe..679b23a8879 100644
--- a/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
+++ b/enterprise/frontend/src/metabase-enterprise/remote_sync/hooks/use-has-transform-dirty-changes.unit.spec.ts
@@ -8,7 +8,6 @@ import {
import { mockSettings } from "__support__/settings";
import { createMockState } from "__support__/state";
import { renderHookWithProviders, waitFor } from "__support__/ui";
-import { TRANSFORMS_ROOT_ID } from "metabase-enterprise/remote_sync/utils";
import type { Collection, RemoteSyncEntity } from "metabase-types/api";
import {
createMockCollection,
@@ -108,15 +107,20 @@ describe("useHasTransformDirtyChanges", () => {
});
});
- it("returns false when remote-sync-transforms setting is disabled", async () => {
+ it("returns true for a pending removal of a transform while transforms sync is disabled", async () => {
const { result } = setup({
isTransformsSyncEnabled: false,
collections: [createMockTransformsCollection()],
- dirty: [createMockRemoteSyncEntity({ model: "transform" })],
+ dirty: [
+ createMockRemoteSyncEntity({
+ model: "transform",
+ sync_status: "removed",
+ }),
+ ],
});
await waitFor(() => {
- expect(result.current).toBe(false);
+ expect(result.current).toBe(true);
});
});
@@ -203,56 +207,4 @@ describe("useHasTransformDirtyChanges", () => {
expect(result.current).toBe(false);
});
});
-
- it("returns true when Transforms root collection (id=-1) is dirty with create status", async () => {
- const transformsRootEntity = createMockRemoteSyncEntity({
- id: TRANSFORMS_ROOT_ID,
- name: "Transforms",
- model: "collection",
- sync_status: "create",
- });
- const { result } = setup({
- collections: [createMockTransformsCollection()],
- dirty: [transformsRootEntity],
- });
-
- await waitFor(() => {
- expect(result.current).toBe(true);
- });
- });
-
- it("returns true when Transforms root collection (id=-1) is dirty with delete status", async () => {
- const transformsRootEntity = createMockRemoteSyncEntity({
- id: TRANSFORMS_ROOT_ID,
- name: "Transforms",
- model: "collection",
- sync_status: "delete",
- });
- const { result } = setup({
- collections: [createMockTransformsCollection()],
- dirty: [transformsRootEntity],
- });
-
- await waitFor(() => {
- expect(result.current).toBe(true);
- });
- });
-
- it("returns true when Transforms root collection (id=-1) is dirty even when transforms setting is disabled", async () => {
- const transformsRootEntity = createMockRemoteSyncEntity({
- id: TRANSFORMS_ROOT_ID,
- name: "Transforms",
- model: "collection",
- sync_status: "delete",
- });
- const { result } = setup({
- isTransformsSyncEnabled: false,
- collections: [],
- dirty: [transformsRootEntity],
- });
-
- await waitFor(() => {
- expect(result.current).toBe(true);
- });
- });
});
diff --git a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/tests/enterprise.unit.spec.tsx b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/tests/enterprise.unit.spec.tsx
index ea7c5561880..b8774f0174e 100644
--- a/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/tests/enterprise.unit.spec.tsx
+++ b/frontend/src/metabase/data-studio/app/pages/DataStudioLayout/tests/enterprise.unit.spec.tsx
@@ -242,12 +242,12 @@ describe("DataStudioLayout", () => {
).not.toBeInTheDocument();
});
- it("should not show dirty indicator when remote-sync-transforms setting is disabled", async () => {
+ it("should show dirty indicator for a pending transform removal when remote-sync-transforms setting is disabled", async () => {
setup({
...DEFAULT_EE_SETTINGS,
remoteSyncBranch: "main",
isNavbarOpened: true,
- dirty: [createMockDirtyTransformEntity()],
+ dirty: [createMockDirtyTransformEntity({ sync_status: "removed" })],
remoteSyncTransforms: false,
});
@@ -257,8 +257,8 @@ describe("DataStudioLayout", () => {
const transformsTab = screen.getByLabelText("Data transformation");
expect(
- within(transformsTab).queryByTestId("remote-sync-status"),
- ).not.toBeInTheDocument();
+ await within(transformsTab).findByTestId("remote-sync-status"),
+ ).toBeInTheDocument();
});
});
diff --git a/resources/migrations/065/20261016_python_library_is_default.yaml b/resources/migrations/065/20261016_python_library_is_default.yaml
new file mode 100644
index 00000000000..2893ce802fd
--- /dev/null
+++ b/resources/migrations/065/20261016_python_library_is_default.yaml
@@ -0,0 +1,32 @@
+## A Python library is the default one, as created, until it is edited. Remote sync only publishes transforms content
+## when there is some, and the built-in common.py the migrations create with an empty source isn't.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.pyld7a
+ author: ranquild
+ comment: Add is_default to python_library
+ changes:
+ - addColumn:
+ tableName: python_library
+ columns:
+ - column:
+ name: is_default
+ type: ${boolean.type}
+ remarks: Whether the library is the default one, as created, never edited
+ defaultValueBoolean: false
+ constraints:
+ nullable: false
+ rollback:
+ - dropColumn:
+ tableName: python_library
+ columnName: is_default
+
+ - changeSet:
+ id: v65.pyld7b
+ author: ranquild
+ comment: Mark the Python libraries with an empty source as the default ones
+ changes:
+ - sql:
+ sql: UPDATE python_library SET is_default = TRUE WHERE source = ''
+ rollback: []
diff --git a/src/metabase/transforms/models/transform_tag.clj b/src/metabase/transforms/models/transform_tag.clj
index c95ab239ae2..b69da3737f4 100644
--- a/src/metabase/transforms/models/transform_tag.clj
+++ b/src/metabase/transforms/models/transform_tag.clj
@@ -63,7 +63,8 @@
(t2/define-before-update :model/TransformTag [tag]
(if (or (nil? (:built_in_type tag))
- mi/*deserializing?*)
+ mi/*deserializing?*
+ (empty? (dissoc (t2/changes tag) :updated_at)))
tag
(-> (merge (translate-name tag) ;; translated default names
{:built_in_type nil} ;; never translate this again
diff --git a/test/metabase/app_db/schema_migrations_test.clj b/test/metabase/app_db/schema_migrations_test.clj
index 51b5f59a8c9..3012987572e 100644
--- a/test/metabase/app_db/schema_migrations_test.clj
+++ b/test/metabase/app_db/schema_migrations_test.clj
@@ -4015,3 +4015,19 @@
(set (concat (t2/select-pks-vec :remote_sync_object :model_type "TableUserSettings" :model_id tracked)
(t2/select-pks-vec :remote_sync_object :model_type "FieldUserSettings" :model_id tracked-fld)))))
(is (t2/exists? :remote_sync_object :id card-rso)))))))
+
+(deftest python-library-is-default-test
+ (testing "v65.pyld7a-b: python_library.is_default is added, true for a library with an empty source"
+ (impl/test-migrations ["v65.pyld7a" "v65.pyld7b"] [migrate!]
+ (t2/delete! :python_library)
+ (let [library! (fn [path source]
+ (t2/insert-returning-pk! :python_library {:path path
+ :source source
+ :entity_id (u/generate-nano-id)
+ :created_at :%now
+ :updated_at :%now}))
+ unedited (library! "common.py" "")
+ edited (library! "other.py" "def f(): pass")]
+ (migrate!)
+ (is (true? (t2/select-one-fn :is_default :python_library :id unedited)))
+ (is (false? (t2/select-one-fn :is_default :python_library :id edited)))))))
diff --git a/test/metabase/transforms/models/transform_tag_test.clj b/test/metabase/transforms/models/transform_tag_test.clj
index 5d275da38a1..693603b382c 100644
--- a/test/metabase/transforms/models/transform_tag_test.clj
+++ b/test/metabase/transforms/models/transform_tag_test.clj
@@ -25,7 +25,15 @@
(t2/update! :model/TransformTag :id (:id tag)
{:name "default2"})
(is (= "default2"
- (:name (t2/select-one :model/TransformTag (:id tag))))))))
+ (:name (t2/select-one :model/TransformTag (:id tag)))))
+ (is (nil? (t2/select-one-fn :built_in_type :model/TransformTag (:id tag)))))))
+
+(deftest initial-tags-kept-on-noop-update-test
+ (testing "An update that only touches the timestamp leaves a tag built in"
+ (doseq [type ["hourly" "daily" "weekly" "monthly"]]
+ (mt/with-temp [:model/TransformTag tag {:name "default" :built_in_type type}]
+ (t2/update! :model/TransformTag :id (:id tag) {:updated_at (mi/now)})
+ (is (= type (t2/select-one-fn :built_in_type :model/TransformTag (:id tag))))))))
(deftest data-analyst-tag-access-follows-the-advanced-permissions-feature-test
(testing "a data analyst reads, writes and creates transform tags only while advanced-permissions is available"