Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix escape logic in serdes (#71940)

Alexander Polyankin940c3e343fff
diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.yml
index 417f9645663..6cb2063ce34 100644
--- a/.github/workflows/serialization.yml
+++ b/.github/workflows/serialization.yml
@@ -36,4 +36,4 @@ jobs:
- name: Prepare front-end environment
uses: ./.github/actions/prepare-frontend
- name: Validate serialization baseline schema
- run: bunx @metabase/[email protected] validate-schema --folder test_resources/serialization_baseline
+ run: bunx @metabase/representations validate-schema --folder test_resources/serialization_baseline
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
index 9207a100e32..c8fffef2ae5 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj
@@ -168,14 +168,18 @@
`opts` are passed down to [[serdes/extract-all]] for each model."
[{:keys [targets user-id] :as opts}]
(log/tracef "Extracting subtrees with options: %s" (pr-str opts))
- (let [nodes (resolve-targets opts user-id)
+ (let [models (model-set opts)
+ has-content? (seq (set/intersection (set serdes.models/content) models))
+ nodes (resolve-targets opts user-id)
;; by model is a map of `{model-name [ids ...]}`
by-model (u/group-by first second (keys nodes))
- {:keys [reportable-escaped analytics-card-ids]} (escape-analysis by-model nodes)]
+ ;; Escape analysis only matters when exporting collection content — it checks whether
+ ;; cards referenced by dashboards live outside the target collections.
+ {:keys [reportable-escaped analytics-card-ids]} (when has-content?
+ (escape-analysis by-model nodes))]
(if (seq reportable-escaped)
(log-escape-report! reportable-escaped)
- (let [models (model-set opts)
- coll-set (get by-model "Collection")
+ (let [coll-set (get by-model "Collection")
;; When targets are specified, also include Tables found via descendants
;; (published tables in target collections). These are extracted by ID, not all.
targeted-data-model (when (seq targets)
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
index 96064c8c294..48aa5a0caa5 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj
@@ -82,7 +82,7 @@
(def legal-top-level-paths
"Known top-level paths for directory with serialization output.
We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."
- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
+ #{"actions" "channels" "collections" "databases" "glossary" "metabots" "python_libraries" "python-libraries" "snippets" "transforms"})
(defn- ingest-all
"Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
index 486310ceeb4..81cca3cbdff 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj
@@ -5,6 +5,7 @@
[clojure.test :refer :all]
[metabase-enterprise.remote-sync.source.git :as git]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase.test :as mt]
[metabase.util :as u])
(:import (java.io File)
@@ -80,10 +81,6 @@
remote))
-(def ^:private test-managed-dirs
- "Managed dirs for test sources — matches v2.ingest/legal-top-level-paths."
- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
-
(defn- ->source!
"Creates a (local) 'remote' repo and initializes a git source that uses it"
[branch {:keys [^Git git] :as _remote-repo}]
@@ -93,7 +90,7 @@
(.toURL)
(.toExternalForm))
local-repo (#'git/get-jgit (#'git/repo-path {:remote-url remote-url}) {:remote-url remote-url})]
- (git/->GitSource local-repo remote-url branch nil test-managed-dirs)))
+ (git/->GitSource local-repo remote-url branch nil ingest/legal-top-level-paths)))
(defn- init-source!
[branch dir & config]
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
index 590ae6a0928..2980dee718e 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj
@@ -4,6 +4,7 @@
[clojure.string :as str]
[clojure.test :as t]
[metabase-enterprise.remote-sync.source.protocol :as source.p]
+ [metabase-enterprise.serialization.v2.ingest :as ingest]
[metabase-enterprise.transforms-python.core :as transforms-python]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -233,17 +234,13 @@ width: fixed
(snapshot [_this]
(->MockSourceSnapshot source-id base-url branch fail-mode files-atom managed-dirs)))
-(def ^:private default-managed-dirs
- "Default managed dirs for mock sources, matching v2.ingest/legal-top-level-paths."
- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})
-
(defn create-mock-source
"Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, `:initial-files`, and `:managed-dirs`."
[& {:keys [branch fail-mode initial-files managed-dirs]
:or {branch "main"
fail-mode nil
initial-files nil
- managed-dirs default-managed-dirs}}]
+ managed-dirs ingest/legal-top-level-paths}}]
(let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"
(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index dca780cb2a1..bac1421d691 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1729,12 +1729,17 @@
(deftest escape-report-test
(mt/with-empty-h2-app-db!
- (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}
- :model/Collection {coll2-id :id} {:name "Other Collection"}
- :model/Collection {coll3-id :id} {:name "Third Collection"}
- :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}
- :model/Database {db-id :id} {}
- :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}
+ (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}
+ :model/Collection {coll2-id :id} {:name "Other Collection"}
+ :model/Collection {coll3-id :id} {:name "Third Collection"}
+ :model/Collection {clean-coll-id :id
+ clean-coll-eid :entity_id} {:name "Clean Collection"}
+ :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}
+ :model/Database {db-id :id} {}
+ :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}
+ :model/Card {clean-card-eid :entity_id} {:name "Clean Card"
+ :collection_id clean-coll-id
+ :database_id db-id}
:model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}
:model/Card _ {:name "Dependent Card"
:collection_id coll2-id
@@ -1792,7 +1797,20 @@
(is (some #(str/starts-with? % "Failed to export Cards")
(into #{}
(map :message)
- (messages))))))))))
+ (messages)))))))
+ (testing "exporting a clean collection works even when other collections have escape issues"
+ (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]
+ :no-settings true
+ :no-data-model true
+ :no-transforms true}))]
+ (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))
+ (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))
+ (testing "data-model-only export is not blocked by escape analysis"
+ (let [extracted (into [] (extract/extract {:no-collections true
+ :no-settings true
+ :no-transforms true}))]
+ (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))
+ "Databases should be exported even when cards reference personal collections"))))))
(deftest recursive-colls-test
(mt/with-empty-h2-app-db!
@@ -2031,7 +2049,11 @@
(testing "metabot depends on its model entities"
(is (= #{[{:model "Card" :id model-eid}]}
- (set (serdes/dependencies ser))))))))))
+ (set (serdes/dependencies ser)))))
+
+ (testing "metabot storage-path uses top-level metabots directory"
+ (is (= [{:label "metabots"} {:label "Test Metabot" :key metabot-eid}]
+ (serdes/storage-path ser {})))))))))
(deftest metabot-collection-test
(mt/with-empty-h2-app-db!
@@ -2687,3 +2709,48 @@
{:id "alpha" :name "A param" :type :category}]
imported (serdes/import-parameters params)]
(is (= ["zebra" "alpha"] (map :id imported))))))
+
+(deftest channel-test
+ (mt/with-empty-h2-app-db!
+ (ts/with-temp-dpc
+ [:model/Channel {email-id :id} {:name "Email Channel"
+ :type :channel/email
+ :details {:host "smtp.example.com" :port 587}
+ :description "An email channel"}
+ :model/Channel {http-id :id} {:name "HTTP Channel"
+ :type :channel/http
+ :details {:url "https://example.com/webhook"
+ :method "POST"
+ :auth-method "none"}
+ :description "An HTTP channel"}]
+ (testing "email channel extraction"
+ (let [ser (ts/extract-one "Channel" email-id)]
+ (is (=? {:serdes/meta [{:model "Channel" :id "Email Channel"}]
+ :name "Email Channel"
+ :type :channel/email
+ :description "An email channel"
+ :details {:host "smtp.example.com" :port 587}
+ :created_at string?}
+ ser))
+ (is (not (contains? ser :id)))
+ (is (not (contains? ser :active)))))
+
+ (testing "http channel extraction"
+ (let [ser (ts/extract-one "Channel" http-id)]
+ (is (=? {:serdes/meta [{:model "Channel" :id "HTTP Channel"}]
+ :name "HTTP Channel"
+ :type :channel/http
+ :description "An HTTP channel"
+ :details {:url "https://example.com/webhook"
+ :method "POST"
+ :auth-method "none"}
+ :created_at string?}
+ ser))))
+
+ (testing "channel storage-path uses top-level channels directory"
+ (let [ser (ts/extract-one "Channel" email-id)]
+ (is (= [{:label "channels"} {:label "Email Channel" :key "Email Channel"}]
+ (serdes/storage-path ser {}))))
+ (let [ser (ts/extract-one "Channel" http-id)]
+ (is (= [{:label "channels"} {:label "HTTP Channel" :key "HTTP Channel"}]
+ (serdes/storage-path ser {}))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index fb28e2d064f..7c74a78f990 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -2056,3 +2056,104 @@
(is (some? dashboard))
(is (= 1 (count dashcards)))
(is (= 1 (count series))))))))))
+
+(deftest channel-roundtrip-test
+ (testing "a channel is serialized and deserialized correctly"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (testing "extraction succeeds"
+ (ts/with-db source-db
+ (ts/create! :model/Channel :name "Test Email Channel"
+ :type :channel/email
+ :details {:host "smtp.example.com" :port 587}
+ :description "A test email channel")
+ (reset! serialized (into [] (serdes.extract/extract {})))
+ (is (some (fn [{[{:keys [model id]}] :serdes/meta}]
+ (and (= model "Channel") (= id "Test Email Channel")))
+ @serialized))))
+
+ (testing "loading into an empty database succeeds"
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (let [channels (t2/select :model/Channel :name "Test Email Channel")]
+ (is (= 1 (count channels)))
+ (is (= "Test Email Channel" (:name (first channels))))
+ (is (= :channel/email (:type (first channels))))
+ (is (= "A test email channel" (:description (first channels)))))))
+
+ (testing "loading again does not duplicate"
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (is (= 1 (t2/count :model/Channel :name "Test Email Channel")))))))))
+
+(deftest metabot-roundtrip-test
+ (testing "a metabot is serialized and deserialized correctly"
+ (let [serialized (atom nil)
+ metabot-eid (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (testing "extraction succeeds"
+ (ts/with-db source-db
+ (let [mb (ts/create! :model/Metabot :name "Test Bot"
+ :description "A test metabot"
+ :use_verified_content false)]
+ (reset! metabot-eid (:entity_id mb)))
+ (reset! serialized (into [] (serdes.extract/extract {:include-metabot true})))
+ (is (some (fn [{[{:keys [model id]}] :serdes/meta}]
+ (and (= model "Metabot") (= id @metabot-eid)))
+ @serialized))))
+
+ (testing "loading into an empty database succeeds"
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (let [metabots (t2/select :model/Metabot :entity_id @metabot-eid)]
+ (is (= 1 (count metabots)))
+ (is (= "Test Bot" (:name (first metabots))))
+ (is (= "A test metabot" (:description (first metabots)))))))
+
+ (testing "loading again does not duplicate"
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (is (= 1 (t2/count :model/Metabot :entity_id @metabot-eid)))))))))
+
+(deftest channel-minimal-required-properties-test
+ (testing "Channel deserialized with only: name, type, details"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (ts/create! :model/Channel :name "Minimal Channel"
+ :type :channel/email
+ :details {:host "smtp.example.com" :port 587}
+ :description "Some description")
+ (reset! serialized (into [] (serdes.extract/extract {}))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Channel" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :name :type :details])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [channel (t2/select-one :model/Channel :name "Minimal Channel")]
+ (is (some? channel))
+ (is (= :channel/email (:type channel))))))))))
+
+(deftest metabot-minimal-required-properties-test
+ (testing "Metabot deserialized with only: entity_id, name"
+ (let [serialized (atom nil)]
+ (ts/with-dbs [source-db dest-db]
+ (ts/with-db source-db
+ (ts/create! :model/Metabot :name "Minimal Bot"
+ :description "Some description"
+ :use_verified_content false)
+ (reset! serialized (into [] (serdes.extract/extract {:include-metabot true}))))
+
+ (let [minimal (mapv (fn [entity]
+ (if (= "Metabot" (-> entity :serdes/meta last :model))
+ (select-keys entity [:serdes/meta :entity_id :name])
+ entity))
+ @serialized)]
+ (ts/with-db dest-db
+ (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (let [metabot (t2/select-one :model/Metabot :name "Minimal Bot")]
+ (is (some? metabot))
+ (is (= "Minimal Bot" (:name metabot))))))))))
diff --git a/src/metabase/channel/models/channel.clj b/src/metabase/channel/models/channel.clj
index 0ed3d95c5fa..a505e954c8e 100644
--- a/src/metabase/channel/models/channel.clj
+++ b/src/metabase/channel/models/channel.clj
@@ -71,6 +71,16 @@
(defmethod serdes/hash-fields :model/Channel [_instance] [:name :type])
+(defmethod serdes/load-find-local "Channel"
+ [path]
+ (t2/select-one :model/Channel :name (:id (last path))))
+
+(defmethod serdes/generate-path "Channel" [_ channel]
+ [(serdes/infer-self-path "Channel" channel)])
+
+(defmethod serdes/storage-path "Channel" [channel _ctx]
+ [{:label "channels"} {:label (:name channel) :key (serdes/entity-id "Channel" channel)}])
+
(defmethod serdes/make-spec "Channel"
[_model-name _opts]
{:copy [:name :description :type :details :active]
diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.clj
index fa76134cfb0..f6c8bfbf6a8 100644
--- a/src/metabase/metabot/models/metabot.clj
+++ b/src/metabase/metabot/models/metabot.clj
@@ -41,6 +41,9 @@
(defmethod serdes/generate-path "Metabot" [_ metabot]
[(serdes/infer-self-path "Metabot" metabot)])
+(defmethod serdes/storage-path "Metabot" [metabot _ctx]
+ [{:label "metabots"} {:label (:name metabot) :key (:entity_id metabot)}])
+
(defmethod serdes/make-spec "Metabot" [_model-name opts]
{:copy [:name :description :entity_id :use_verified_content]
:transform {:created_at (serdes/date)
diff --git a/test_resources/serialization_baseline/channels/email_channel.yaml b/test_resources/serialization_baseline/channels/email_channel.yaml
new file mode 100644
index 00000000000..88101db67cc
--- /dev/null
+++ b/test_resources/serialization_baseline/channels/email_channel.yaml
@@ -0,0 +1,10 @@
+created_at: '2024-09-11T15:36:01.845352Z'
+description: Email notification channel
+details:
+ host: smtp.example.com
+ port: 587
+name: Email Channel
+type: channel/email
+serdes/meta:
+- id: Email Channel
+ model: Channel
diff --git a/test_resources/serialization_baseline/collections/main/my_http_channel.yaml b/test_resources/serialization_baseline/channels/http_channel.yaml
similarity index 67%
rename from test_resources/serialization_baseline/collections/main/my_http_channel.yaml
rename to test_resources/serialization_baseline/channels/http_channel.yaml
index 8010bd08ce2..8b9f0db6a99 100644
--- a/test_resources/serialization_baseline/collections/main/my_http_channel.yaml
+++ b/test_resources/serialization_baseline/channels/http_channel.yaml
@@ -2,9 +2,8 @@ created_at: '2024-09-11T15:36:01.845352Z'
details:
auth-method: none
url: http://example.com
-name: My HTTP channel
+name: HTTP Channel
type: channel/http
serdes/meta:
-- id: My HTTP channel
- label: my_http_channel
+- id: HTTP Channel
model: Channel
diff --git a/test_resources/serialization_baseline/collections/main/embedded_metabot.yaml b/test_resources/serialization_baseline/metabots/embedded_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/main/embedded_metabot.yaml
rename to test_resources/serialization_baseline/metabots/embedded_metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/main/metabot.yaml b/test_resources/serialization_baseline/metabots/metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/main/metabot.yaml
rename to test_resources/serialization_baseline/metabots/metabot.yaml
diff --git a/test_resources/serialization_baseline/collections/main/slack_metabot.yaml b/test_resources/serialization_baseline/metabots/slack_metabot.yaml
similarity index 100%
rename from test_resources/serialization_baseline/collections/main/slack_metabot.yaml
rename to test_resources/serialization_baseline/metabots/slack_metabot.yaml