The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix escape logic in serdes (#71940)
Alexander Polyankin940c3e343fff
diff --git a/.github/workflows/serialization.yml b/.github/workflows/serialization.ymlindex 417f9645663..6cb2063ce34 100644--- a/.github/workflows/serialization.yml+++ b/.github/workflows/serialization.yml@@ -36,4 +36,4 @@ jobs:- name: Prepare front-end environmentuses: ./.github/actions/prepare-frontend- name: Validate serialization baseline schema- run: bunx @metabase/[email protected] validate-schema --folder test_resources/serialization_baseline+ run: bunx @metabase/representations validate-schema --folder test_resources/serialization_baselinediff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex 9207a100e32..c8fffef2ae5 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -168,14 +168,18 @@`opts` are passed down to [[serdes/extract-all]] for each model."[{:keys [targets user-id] :as opts}](log/tracef "Extracting subtrees with options: %s" (pr-str opts))- (let [nodes (resolve-targets opts user-id)+ (let [models (model-set opts)+ has-content? (seq (set/intersection (set serdes.models/content) models))+ nodes (resolve-targets opts user-id);; by model is a map of `{model-name [ids ...]}`by-model (u/group-by first second (keys nodes))- {:keys [reportable-escaped analytics-card-ids]} (escape-analysis by-model nodes)]+ ;; Escape analysis only matters when exporting collection content — it checks whether+ ;; cards referenced by dashboards live outside the target collections.+ {:keys [reportable-escaped analytics-card-ids]} (when has-content?+ (escape-analysis by-model nodes))](if (seq reportable-escaped)(log-escape-report! reportable-escaped)- (let [models (model-set opts)- coll-set (get by-model "Collection")+ (let [coll-set (get by-model "Collection");; When targets are specified, also include Tables found via descendants;; (published tables in target collections). These are extracted by ID, not all.targeted-data-model (when (seq targets)diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.cljindex 96064c8c294..48aa5a0caa5 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/ingest.clj@@ -82,7 +82,7 @@(def legal-top-level-paths"Known top-level paths for directory with serialization output.We support both \"python-libraries\" and \"python_libraries\" for backwards compatibility. The modern name is \"python_libraries\"."- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})+ #{"actions" "channels" "collections" "databases" "glossary" "metabots" "python_libraries" "python-libraries" "snippets" "transforms"})(defn- ingest-all"Returns {:entities {unlabeled-hierarchy [hierarchy File]}, :errors [Exception...]}.diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.cljindex 486310ceeb4..81cca3cbdff 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/source/git_test.clj@@ -5,6 +5,7 @@[clojure.test :refer :all][metabase-enterprise.remote-sync.source.git :as git][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.serialization.v2.ingest :as ingest][metabase.test :as mt][metabase.util :as u])(:import (java.io File)@@ -80,10 +81,6 @@remote))-(def ^:private test-managed-dirs- "Managed dirs for test sources — matches v2.ingest/legal-top-level-paths."- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})-(defn- ->source!"Creates a (local) 'remote' repo and initializes a git source that uses it"[branch {:keys [^Git git] :as _remote-repo}]@@ -93,7 +90,7 @@(.toURL)(.toExternalForm))local-repo (#'git/get-jgit (#'git/repo-path {:remote-url remote-url}) {:remote-url remote-url})]- (git/->GitSource local-repo remote-url branch nil test-managed-dirs)))+ (git/->GitSource local-repo remote-url branch nil ingest/legal-top-level-paths)))(defn- init-source![branch dir & config]diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.cljindex 590ae6a0928..2980dee718e 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj@@ -4,6 +4,7 @@[clojure.string :as str][clojure.test :as t][metabase-enterprise.remote-sync.source.protocol :as source.p]+ [metabase-enterprise.serialization.v2.ingest :as ingest][metabase-enterprise.transforms-python.core :as transforms-python][metabase.util :as u][toucan2.core :as t2]))@@ -233,17 +234,13 @@ width: fixed(snapshot [_this](->MockSourceSnapshot source-id base-url branch fail-mode files-atom managed-dirs)))-(def ^:private default-managed-dirs- "Default managed dirs for mock sources, matching v2.ingest/legal-top-level-paths."- #{"actions" "collections" "databases" "glossary" "python_libraries" "python-libraries" "snippets" "transforms"})-(defn create-mock-source"Create a mock Source for testing. Optionally accepts `:branch`, `:fail-mode`, `:initial-files`, and `:managed-dirs`."[& {:keys [branch fail-mode initial-files managed-dirs]:or {branch "main"fail-mode nilinitial-files nil- managed-dirs default-managed-dirs}}]+ managed-dirs ingest/legal-top-level-paths}}](let [default-files {"main" {"collections/M-Q4pcV0qkiyJ0kiSWECl_some_collection/M-Q4pcV0qkiyJ0kiSWECl_some_collection.yaml"(generate-collection-yaml "M-Q4pcV0qkiyJ0kiSWECl" "Some Collection")diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex dca780cb2a1..bac1421d691 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1729,12 +1729,17 @@(deftest escape-report-test(mt/with-empty-h2-app-db!- (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}- :model/Collection {coll2-id :id} {:name "Other Collection"}- :model/Collection {coll3-id :id} {:name "Third Collection"}- :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}- :model/Database {db-id :id} {}- :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}+ (ts/with-temp-dpc [:model/Collection {coll1-id :id} {:name "Some Collection"}+ :model/Collection {coll2-id :id} {:name "Other Collection"}+ :model/Collection {coll3-id :id} {:name "Third Collection"}+ :model/Collection {clean-coll-id :id+ clean-coll-eid :entity_id} {:name "Clean Collection"}+ :model/Dashboard {dash-id :id} {:name "A Dashboard" :collection_id coll1-id}+ :model/Database {db-id :id} {}+ :model/Card {card1-id :id} {:name "Some Card", :database_id db-id}+ :model/Card {clean-card-eid :entity_id} {:name "Clean Card"+ :collection_id clean-coll-id+ :database_id db-id}:model/DashboardCard _ {:card_id card1-id :dashboard_id dash-id}:model/Card _ {:name "Dependent Card":collection_id coll2-id@@ -1792,7 +1797,20 @@(is (some #(str/starts-with? % "Failed to export Cards")(into #{}(map :message)- (messages))))))))))+ (messages)))))))+ (testing "exporting a clean collection works even when other collections have escape issues"+ (let [extracted (into [] (extract/extract {:targets [["Collection" clean-coll-id]]+ :no-settings true+ :no-data-model true+ :no-transforms true}))]+ (is (= #{clean-coll-eid} (ids-by-model "Collection" extracted)))+ (is (= #{clean-card-eid} (ids-by-model "Card" extracted)))))+ (testing "data-model-only export is not blocked by escape analysis"+ (let [extracted (into [] (extract/extract {:no-collections true+ :no-settings true+ :no-transforms true}))]+ (is (seq (filter #(= "Database" (-> % :serdes/meta last :model)) extracted))+ "Databases should be exported even when cards reference personal collections"))))))(deftest recursive-colls-test(mt/with-empty-h2-app-db!@@ -2031,7 +2049,11 @@(testing "metabot depends on its model entities"(is (= #{[{:model "Card" :id model-eid}]}- (set (serdes/dependencies ser))))))))))+ (set (serdes/dependencies ser)))))++ (testing "metabot storage-path uses top-level metabots directory"+ (is (= [{:label "metabots"} {:label "Test Metabot" :key metabot-eid}]+ (serdes/storage-path ser {})))))))))(deftest metabot-collection-test(mt/with-empty-h2-app-db!@@ -2687,3 +2709,48 @@{:id "alpha" :name "A param" :type :category}]imported (serdes/import-parameters params)](is (= ["zebra" "alpha"] (map :id imported))))))++(deftest channel-test+ (mt/with-empty-h2-app-db!+ (ts/with-temp-dpc+ [:model/Channel {email-id :id} {:name "Email Channel"+ :type :channel/email+ :details {:host "smtp.example.com" :port 587}+ :description "An email channel"}+ :model/Channel {http-id :id} {:name "HTTP Channel"+ :type :channel/http+ :details {:url "https://example.com/webhook"+ :method "POST"+ :auth-method "none"}+ :description "An HTTP channel"}]+ (testing "email channel extraction"+ (let [ser (ts/extract-one "Channel" email-id)]+ (is (=? {:serdes/meta [{:model "Channel" :id "Email Channel"}]+ :name "Email Channel"+ :type :channel/email+ :description "An email channel"+ :details {:host "smtp.example.com" :port 587}+ :created_at string?}+ ser))+ (is (not (contains? ser :id)))+ (is (not (contains? ser :active)))))++ (testing "http channel extraction"+ (let [ser (ts/extract-one "Channel" http-id)]+ (is (=? {:serdes/meta [{:model "Channel" :id "HTTP Channel"}]+ :name "HTTP Channel"+ :type :channel/http+ :description "An HTTP channel"+ :details {:url "https://example.com/webhook"+ :method "POST"+ :auth-method "none"}+ :created_at string?}+ ser))))++ (testing "channel storage-path uses top-level channels directory"+ (let [ser (ts/extract-one "Channel" email-id)]+ (is (= [{:label "channels"} {:label "Email Channel" :key "Email Channel"}]+ (serdes/storage-path ser {}))))+ (let [ser (ts/extract-one "Channel" http-id)]+ (is (= [{:label "channels"} {:label "HTTP Channel" :key "HTTP Channel"}]+ (serdes/storage-path ser {}))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex fb28e2d064f..7c74a78f990 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -2056,3 +2056,104 @@(is (some? dashboard))(is (= 1 (count dashcards)))(is (= 1 (count series))))))))))++(deftest channel-roundtrip-test+ (testing "a channel is serialized and deserialized correctly"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (testing "extraction succeeds"+ (ts/with-db source-db+ (ts/create! :model/Channel :name "Test Email Channel"+ :type :channel/email+ :details {:host "smtp.example.com" :port 587}+ :description "A test email channel")+ (reset! serialized (into [] (serdes.extract/extract {})))+ (is (some (fn [{[{:keys [model id]}] :serdes/meta}]+ (and (= model "Channel") (= id "Test Email Channel")))+ @serialized))))++ (testing "loading into an empty database succeeds"+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (let [channels (t2/select :model/Channel :name "Test Email Channel")]+ (is (= 1 (count channels)))+ (is (= "Test Email Channel" (:name (first channels))))+ (is (= :channel/email (:type (first channels))))+ (is (= "A test email channel" (:description (first channels)))))))++ (testing "loading again does not duplicate"+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (is (= 1 (t2/count :model/Channel :name "Test Email Channel")))))))))++(deftest metabot-roundtrip-test+ (testing "a metabot is serialized and deserialized correctly"+ (let [serialized (atom nil)+ metabot-eid (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (testing "extraction succeeds"+ (ts/with-db source-db+ (let [mb (ts/create! :model/Metabot :name "Test Bot"+ :description "A test metabot"+ :use_verified_content false)]+ (reset! metabot-eid (:entity_id mb)))+ (reset! serialized (into [] (serdes.extract/extract {:include-metabot true})))+ (is (some (fn [{[{:keys [model id]}] :serdes/meta}]+ (and (= model "Metabot") (= id @metabot-eid)))+ @serialized))))++ (testing "loading into an empty database succeeds"+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (let [metabots (t2/select :model/Metabot :entity_id @metabot-eid)]+ (is (= 1 (count metabots)))+ (is (= "Test Bot" (:name (first metabots))))+ (is (= "A test metabot" (:description (first metabots)))))))++ (testing "loading again does not duplicate"+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (is (= 1 (t2/count :model/Metabot :entity_id @metabot-eid)))))))))++(deftest channel-minimal-required-properties-test+ (testing "Channel deserialized with only: name, type, details"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (ts/create! :model/Channel :name "Minimal Channel"+ :type :channel/email+ :details {:host "smtp.example.com" :port 587}+ :description "Some description")+ (reset! serialized (into [] (serdes.extract/extract {}))))++ (let [minimal (mapv (fn [entity]+ (if (= "Channel" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :name :type :details])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [channel (t2/select-one :model/Channel :name "Minimal Channel")]+ (is (some? channel))+ (is (= :channel/email (:type channel))))))))))++(deftest metabot-minimal-required-properties-test+ (testing "Metabot deserialized with only: entity_id, name"+ (let [serialized (atom nil)]+ (ts/with-dbs [source-db dest-db]+ (ts/with-db source-db+ (ts/create! :model/Metabot :name "Minimal Bot"+ :description "Some description"+ :use_verified_content false)+ (reset! serialized (into [] (serdes.extract/extract {:include-metabot true}))))++ (let [minimal (mapv (fn [entity]+ (if (= "Metabot" (-> entity :serdes/meta last :model))+ (select-keys entity [:serdes/meta :entity_id :name])+ entity))+ @serialized)]+ (ts/with-db dest-db+ (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (let [metabot (t2/select-one :model/Metabot :name "Minimal Bot")]+ (is (some? metabot))+ (is (= "Minimal Bot" (:name metabot))))))))))diff --git a/src/metabase/channel/models/channel.clj b/src/metabase/channel/models/channel.cljindex 0ed3d95c5fa..a505e954c8e 100644--- a/src/metabase/channel/models/channel.clj+++ b/src/metabase/channel/models/channel.clj@@ -71,6 +71,16 @@(defmethod serdes/hash-fields :model/Channel [_instance] [:name :type])+(defmethod serdes/load-find-local "Channel"+ [path]+ (t2/select-one :model/Channel :name (:id (last path))))++(defmethod serdes/generate-path "Channel" [_ channel]+ [(serdes/infer-self-path "Channel" channel)])++(defmethod serdes/storage-path "Channel" [channel _ctx]+ [{:label "channels"} {:label (:name channel) :key (serdes/entity-id "Channel" channel)}])+(defmethod serdes/make-spec "Channel"[_model-name _opts]{:copy [:name :description :type :details :active]diff --git a/src/metabase/metabot/models/metabot.clj b/src/metabase/metabot/models/metabot.cljindex fa76134cfb0..f6c8bfbf6a8 100644--- a/src/metabase/metabot/models/metabot.clj+++ b/src/metabase/metabot/models/metabot.clj@@ -41,6 +41,9 @@(defmethod serdes/generate-path "Metabot" [_ metabot][(serdes/infer-self-path "Metabot" metabot)])+(defmethod serdes/storage-path "Metabot" [metabot _ctx]+ [{:label "metabots"} {:label (:name metabot) :key (:entity_id metabot)}])+(defmethod serdes/make-spec "Metabot" [_model-name opts]{:copy [:name :description :entity_id :use_verified_content]:transform {:created_at (serdes/date)diff --git a/test_resources/serialization_baseline/channels/email_channel.yaml b/test_resources/serialization_baseline/channels/email_channel.yamlnew file mode 100644index 00000000000..88101db67cc--- /dev/null+++ b/test_resources/serialization_baseline/channels/email_channel.yaml@@ -0,0 +1,10 @@+created_at: '2024-09-11T15:36:01.845352Z'+description: Email notification channel+details:+ host: smtp.example.com+ port: 587+name: Email Channel+type: channel/email+serdes/meta:+- id: Email Channel+ model: Channeldiff --git a/test_resources/serialization_baseline/collections/main/my_http_channel.yaml b/test_resources/serialization_baseline/channels/http_channel.yamlsimilarity index 67%rename from test_resources/serialization_baseline/collections/main/my_http_channel.yamlrename to test_resources/serialization_baseline/channels/http_channel.yamlindex 8010bd08ce2..8b9f0db6a99 100644--- a/test_resources/serialization_baseline/collections/main/my_http_channel.yaml+++ b/test_resources/serialization_baseline/channels/http_channel.yaml@@ -2,9 +2,8 @@ created_at: '2024-09-11T15:36:01.845352Z'details:auth-method: noneurl: http://example.com-name: My HTTP channel+name: HTTP Channeltype: channel/httpserdes/meta:-- id: My HTTP channel- label: my_http_channel+- id: HTTP Channelmodel: Channeldiff --git a/test_resources/serialization_baseline/collections/main/embedded_metabot.yaml b/test_resources/serialization_baseline/metabots/embedded_metabot.yamlsimilarity index 100%rename from test_resources/serialization_baseline/collections/main/embedded_metabot.yamlrename to test_resources/serialization_baseline/metabots/embedded_metabot.yamldiff --git a/test_resources/serialization_baseline/collections/main/metabot.yaml b/test_resources/serialization_baseline/metabots/metabot.yamlsimilarity index 100%rename from test_resources/serialization_baseline/collections/main/metabot.yamlrename to test_resources/serialization_baseline/metabots/metabot.yamldiff --git a/test_resources/serialization_baseline/collections/main/slack_metabot.yaml b/test_resources/serialization_baseline/metabots/slack_metabot.yamlsimilarity index 100%rename from test_resources/serialization_baseline/collections/main/slack_metabot.yamlrename to test_resources/serialization_baseline/metabots/slack_metabot.yaml