Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: better conversions of v57 remote-sync files (#66894)

Edward Paget672f62f9ee14
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
index 12e541573d7..8b133bc0cfd 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/impl_test.clj
@@ -548,3 +548,46 @@
"Should return nil when remote sync is disabled")
(is @clear-called?
"Should call clear-remote-synced-collection!")))))))))
+
+(deftest import!-v57-type-remote-synced-migration-test
+ (testing "importing v57 export with type=remote-synced sets is_remote_synced=true and clears type"
+ (let [v57-entity-id "v57-collection-xxxx"
+ v57-files {"main" {(str "collections/" v57-entity-id "_v57_collection/" v57-entity-id "_v57_collection.yaml")
+ (test-helpers/generate-v57-collection-yaml v57-entity-id "V57 Collection" :type "remote-synced")}}
+ mock-source (test-helpers/create-mock-source :initial-files v57-files)]
+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ (let [imported-collection (t2/select-one :model/Collection :entity_id v57-entity-id)]
+ (is (some? imported-collection)
+ "Collection should be imported")
+ (is (true? (:is_remote_synced imported-collection))
+ "is_remote_synced should be true after import")
+ (is (nil? (:type imported-collection))
+ "type should be nil (not 'remote-synced') after migration")))))))
+
+(deftest import!-v57-nested-remote-synced-collections-test
+ (testing "importing v57 export with nested type=remote-synced collections migrates all correctly"
+ (let [parent-entity-id "v57-parent-collxxxxxx"
+ child-entity-id "v57-child-collxxxxxxx"
+ v57-files {"main" {(str "collections/" parent-entity-id "_v57_parent/" parent-entity-id "_v57_parent.yaml")
+ (test-helpers/generate-v57-collection-yaml parent-entity-id "V57 Parent" :type "remote-synced")
+
+ (str "collections/" parent-entity-id "_v57_parent/" child-entity-id "_v57_child/" child-entity-id "_v57_child.yaml")
+ (test-helpers/generate-v57-collection-yaml child-entity-id "V57 Child" :parent-id parent-entity-id :type "remote-synced")}}
+ mock-source (test-helpers/create-mock-source :initial-files v57-files)]
+ (mt/with-model-cleanup [:model/Collection :model/RemoteSyncTask]
+ (let [task-id (t2/insert-returning-pk! :model/RemoteSyncTask {:sync_task_type "import" :initiated_by (mt/user->id :rasta)})
+ result (impl/import! (source.p/snapshot mock-source) task-id)]
+ (is (= :success (:status result)))
+ (let [parent-collection (t2/select-one :model/Collection :entity_id parent-entity-id)
+ child-collection (t2/select-one :model/Collection :entity_id child-entity-id)]
+ (testing "parent collection"
+ (is (some? parent-collection) "Parent collection should be imported")
+ (is (true? (:is_remote_synced parent-collection)) "Parent is_remote_synced should be true")
+ (is (nil? (:type parent-collection)) "Parent type should be nil"))
+ (testing "child collection"
+ (is (some? child-collection) "Child collection should be imported")
+ (is (true? (:is_remote_synced child-collection)) "Child is_remote_synced should be true")
+ (is (nil? (:type child-collection)) "Child type should be nil"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 6b65f850269..6ce75131275 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -40,6 +40,44 @@ is_sample: false
name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")
(or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
+(defn generate-v57-collection-yaml
+ "Generates YAML content for a collection in v57 format.
+
+ In v57, remote-synced collections used `type: remote-synced` instead of
+ `is_remote_synced: true`. This helper is used to test backward compatibility
+ when importing exports from v57 Metabase instances.
+
+ Args:
+ entity-id: The unique identifier for the collection.
+ name: The name of the collection.
+ parent-id: Optional parent collection ID.
+ type: Optional type value (e.g., \"remote-synced\" for v57 remote-synced collections).
+
+ Returns:
+ A string containing the v57-format YAML representation of the collection."
+ [entity-id name & {:keys [parent-id type]}]
+ (format "name: %s
+description: null
+entity_id: %s
+slug: %s
+created_at: '2024-08-28T09:46:18.671622Z'
+archived: false
+type: %s
+parent_id: %s
+personal_owner_id: null
+namespace: null
+authority_level: null
+serdes/meta:
+- id: %s
+ label: %s
+ model: Collection
+archive_operation_id: null
+archived_directly: null
+is_sample: false
+"
+ name entity-id (str/replace (u/lower-case-en name) #"\s+" "_")
+ (or type "null") (or parent-id "null") entity-id (str/replace (u/lower-case-en name) #"\s+" "_")))
+
(defn generate-card-yaml
"Generates YAML content for a card.
diff --git a/src/metabase/collections/models/collection.clj b/src/metabase/collections/models/collection.clj
index 59009c1da65..94588ec05d3 100644
--- a/src/metabase/collections/models/collection.clj
+++ b/src/metabase/collections/models/collection.clj
@@ -1565,22 +1565,16 @@
(throw (ex-info "Can't create a personal collection for an API key" {:user user-id}))))))
(t2/define-before-insert :model/Collection
- [{collection-name :name :keys [location type] :as collection}]
+ [{collection-name :name :keys [type] :as collection}]
(assert-valid-location collection)
(assert-not-personal-collection-for-api-key collection)
(assert-valid-namespace (merge {:namespace nil} collection))
- (assert-valid-remote-synced-parent collection)
(check-allowed-content (:type collection) (when-let [location (:location (t2/changes collection))] (location-path->parent-id location)))
- ;; Inherit is_remote_synced from parent if not explicitly set
- (let [parent-is-remote-synced? (when-let [parent-id (and location (location-path->parent-id location))]
- (t2/select-one-fn :is_remote_synced :model/Collection :id parent-id))]
- (-> collection
- (assoc :slug (slugify collection-name))
- (cond->
- (= type "remote-synced") (-> (assoc :is_remote_synced true) (dissoc :type))
- (and (not (contains? collection :is_remote_synced))
- parent-is-remote-synced?)
- (assoc :is_remote_synced true)))))
+ (u/prog1 (-> collection
+ (assoc :slug (slugify collection-name))
+ (cond->
+ (= type "remote-synced") (-> (assoc :is_remote_synced true) (dissoc :type))))
+ (assert-valid-remote-synced-parent <>)))
(defn- copy-collection-permissions!
"Grant read permissions to destination Collections for every Group with read permissions for a source Collection,
@@ -1777,10 +1771,7 @@
(let [msg (tru "You cannot move a Collection to a different namespace once it has been created.")]
(throw (ex-info msg {:status-code 400, :errors {:namespace msg}})))))
(assert-valid-namespace (merge (select-keys collection-before-updates [:namespace]) collection-updates))
- ;; (3.5) Check remote-synced validation rules
- (when-not *clearing-remote-sync*
- (let [merged-collection (merge collection-before-updates collection-updates)]
- (assert-valid-remote-synced-parent merged-collection)))
+
;; (3.6) Check that the parent collection allows this collection to be there
(check-allowed-content (:type collection) (when-let [location (:location collection)] (location-path->parent-id location)))
;; (3.7) Check if it's a semantic-library collection that can't be updated
@@ -1795,9 +1786,11 @@
;; OK, AT THIS POINT THE CHANGES ARE VALIDATED. NOW START ISSUING UPDATES
;; slugify the collection name in case it's changed in the output; the results of this will get passed along
;; to Toucan's `update!` impl
- (cond-> collection-updates
- (= type "remote-synced") (-> (assoc :is_remote_synced true) (dissoc :type))
- collection-name (assoc :slug (slugify collection-name)))))
+ (u/prog1 (cond-> collection-updates
+ (= type "remote-synced") (-> (assoc :is_remote_synced true) (dissoc :type))
+ collection-name (assoc :slug (slugify collection-name)))
+ (when-not *clearing-remote-sync*
+ (assert-valid-remote-synced-parent (merge collection-before-updates <>))))))
;;; ----------------------------------------------------- DELETE -----------------------------------------------------