The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Strictly limit generated data app schemas to library-published content (#83548)
Phoomparin Mano9deb406fa0cd
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.cljindex 35d50f66103..84055b183d1 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj@@ -321,6 +321,14 @@[:= :model_id nil][:= :type "query"][:= :archived false]+ [:or+ [:= :collection_id nil]+ [:exists ^:allow-subquery {:select [1]+ :from [[(t2/table-name :model/Collection) :c]]+ :where [:and+ [:= :c.id :action.collection_id]+ [:= :c.archived false]+ [:= :c.namespace "data-actions"]]}]](when (seq excluded-collection-ids)[:or [:= :collection_id nil] [:not-in :collection_id excluded-collection-ids]])]:order-by [[:name :asc] [:id :asc]]}))@@ -363,7 +371,11 @@[measure-ids :- [:sequential ::lib.schema.id/measure]](t2/select [:model/Measure :id :definition] :id [:in measure-ids]))-(mu/defn collections-with-entity-ids- "The Collections with `entity-ids`."- [entity-ids :- [:set :string]]- (t2/select :model/Collection :entity_id [:in entity-ids]))+(mu/defn unarchived-collections+ "The unarchived Collections among `collection-ids`, with their Library types."+ [collection-ids :- [:set ::lib.schema.id/collection]]+ (if (seq collection-ids)+ (t2/select [:model/Collection :id :type]+ :id [:in collection-ids]+ :archived false)+ []))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.cljindex e06b497ac4d..fc602a9ac4c 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj@@ -30,8 +30,7 @@[:metrics [:sequential :map]]])(defn fetch-items- "Fetches the library's metrics, its published tables and the tables those metrics map, and the query actions that- belong to no model."+ "Fetches Library metrics, published tables and metric backing tables, and eligible query actions without a model."([](fetch-items schemas.source/app-db-source))([source]diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.cljindex 23881a7e315..6c61ff17ecf 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj@@ -78,8 +78,8 @@:entityId entity_id)))(defn action-schemas- "Returns schema entries for the query actions without a model, leaving out the copies data apps own and those a- routing destination backs."+ "Returns schemas for standalone actions, excluding copies owned by data app and+ those a destination backs."[](let [ids (data-apps.db/model-less-query-action-ids (set (perms/data-app-collection-ids)))details-by-id (when (seq ids) (u/index-by :id (actions/select-actions-for-ids nil ids)))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.cljindex b496aad6737..cb3f13b054f 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj@@ -7,18 +7,16 @@`metabase-enterprise.data-apps.generate.schemas.*` builders, except [[library-tables]], which returns raw table rows(only `:id` is consumed)."(:require+ [clojure.set :as set][metabase-enterprise.data-apps.db :as data-apps.db][metabase-enterprise.data-apps.generate.schemas.action :as schemas.action][metabase-enterprise.data-apps.generate.schemas.metric :as schemas.metric][metabase-enterprise.data-apps.generate.schemas.table :as schemas.table]+ [metabase.collections.core :as collections][metabase.collections.models.collection :as collection]))(set! *warn-on-reflection* true)-(def ^:private library-root-entity-ids- "Entity ids of the root data library and root metrics library collections."- #{"librarylibrarydatadat" "librarylibrarymetrics"})-(def LibraryScope"The library collection tree, classified by collection type: `:data-collection-ids` hold the published tables,`:metric-collection-ids` the metrics."@@ -29,9 +27,10 @@(defn- app-db-library-scope"The [[LibraryScope]] of the root libraries and their descendants, empty where the instance has no library."[]- (let [roots (filter #(contains? collection/library-collection-types (:type %))- (data-apps.db/collections-with-entity-ids library-root-entity-ids))- collections (concat roots (when (seq roots) (collection/descendants-flat-for roots)))+ (let [library (collections/library-collection)+ descendants (when (and library (not (:archived library)))+ (collection/descendants-flat-for [library]))+ collections (data-apps.db/unarchived-collections (into #{} (map :id) descendants))ids-of-type (fn [collection-type](into #{} (comp (filter #(= (:type %) collection-type)) (map :id)) collections))]{:data-collection-ids (ids-of-type collection/library-data-collection-type)@@ -50,6 +49,10 @@(library-tables [source collection-ids]"Published table rows in `collection-ids`."))+(defn- curated-library-ids+ [scope-key requested-ids]+ (set/intersection (scope-key (app-db-library-scope)) requested-ids))+(def app-db-source"The production [[SchemaSource]], backed by the application database."(reify SchemaSource@@ -58,8 +61,10 @@(actions [_](schemas.action/action-schemas))(metrics [_ collection-ids]- (vec (schemas.metric/metric-schemas collection-ids)))+ (vec (schemas.metric/metric-schemas+ (curated-library-ids :metric-collection-ids collection-ids))))(tables [_ table-ids](vec (schemas.table/table-schemas (schemas.table/select-tables table-ids))))(library-tables [_ collection-ids]- (vec (schemas.table/select-library-tables collection-ids)))))+ (vec (schemas.table/select-library-tables+ (curated-library-ids :data-collection-ids collection-ids))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.cljnew file mode 100644index 00000000000..638774e7efa--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.clj@@ -0,0 +1,143 @@+(ns metabase-enterprise.data-apps.generate.schemas.library-test+ (:require+ [clojure.set :as set]+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.data-apps.generate.schemas :as schemas]+ [metabase-enterprise.data-apps.generate.schemas.source :as source]+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase.collections.models.collection :as collection]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(use-fixtures :once (fixtures/initialize :db :web-server :test-users))++(deftest library-content-and-metric-backing-tables-are-exported-test+ (mt/dataset test-data+ (data-apps.tu/do-with-library!+ (fn [{:keys [data-id metrics-id]}]+ (let [mp (mt/metadata-provider)+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ query (lib/aggregate orders-query+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]+ (mt/with-temp [:model/Collection nested-data {:type "library-data"+ :location (collection/children-location+ (t2/select-one :model/Collection :id data-id))}+ :model/Collection nested-metrics {:type "library-metrics"+ :location (collection/children-location+ (t2/select-one :model/Collection :id metrics-id))}+ :model/Collection outside {}+ :model/Collection lookalike {:type "library-data"}+ :model/Table published {:db_id (mt/id), :name "library_widgets"+ :is_published true, :collection_id data-id}+ :model/Field _ {:table_id (:id published), :name "price", :base_type :type/Float}+ :model/Table nested {:db_id (mt/id), :name "nested_widgets"+ :is_published true, :collection_id (:id nested-data)}+ :model/Table _unpublished {:db_id (mt/id), :name "unpublished_widgets"+ :is_published false, :collection_id data-id}+ :model/Table _elsewhere {:db_id (mt/id), :name "outside_widgets"+ :is_published true, :collection_id (:id outside)}+ :model/Table _lookalike {:db_id (mt/id), :name "lookalike_widgets"+ :is_published true, :collection_id (:id lookalike)}+ :model/Card metric {:name "Library revenue", :type :metric, :display :scalar+ :database_id (mt/id), :collection_id metrics-id, :dataset_query query}+ :model/Card nested-metric {:name "Nested revenue", :type :metric, :display :scalar+ :database_id (mt/id), :collection_id (:id nested-metrics)+ :dataset_query query}+ :model/Card _outside-metric {:name "Outside revenue", :type :metric, :display :scalar+ :database_id (mt/id), :collection_id (:id outside)+ :dataset_query query}+ :model/Card _archived-metric {:name "Archived revenue", :type :metric, :display :scalar+ :database_id (mt/id), :collection_id metrics-id+ :dataset_query query, :archived true}+ :model/Card _model {:name "Outside model", :type :model, :database_id (mt/id)+ :collection_id (:id outside), :dataset_query orders-query}]+ (mt/with-test-user :crowberto+ (testing "items published to libraries can be exported"+ (let [items (schemas/fetch-items)]+ (is (= #{(:id published) (:id nested) (mt/id :orders)} (into #{} (map :id) (:tables items))))+ (is (= #{(:id metric) (:id nested-metric)} (into #{} (map :id) (:metrics items))))+ (is (= #{(mt/id :orders)} (into #{} (mapcat :mappedTableIds) (:metrics items))))))+ (testing "only collection IDs from the library can be exported"+ (is (= [] (source/library-tables source/app-db-source #{(:id outside) (:id lookalike)})))+ (is (= [] (source/metrics source/app-db-source #{(:id outside)}))))+ ; we pick the metric dimensions from table fields, therefore tables+ ; that backed metrics are included in the export, even if they are not published to the library+ (testing "the response includes metric backing tables and excludes unrelated content"+ (mt/with-premium-features #{:data-apps}+ (let [body (:body (mt/user-http-request-full-response+ :crowberto :get 200 "apps/generate/schemas"))]+ (doseq [included ["libraryWidgets: {" "nestedWidgets: {" "libraryRevenue: {"+ "nestedRevenue: {" "orders: {"]]+ (is (str/includes? body included) included))+ (doseq [excluded ["unpublishedWidgets" "outsideWidgets" "lookalikeWidgets" "outsideRevenue"+ "archivedRevenue" "outsideModel" "models:"]]+ (is (not (str/includes? body excluded)) excluded)))))+ (testing "moving a metric out of the library immediately removes it"+ (mt/with-temp-vals-in-db :model/Card (:id metric) {:collection_id (:id outside)}+ (is (= [(:id nested-metric)]+ (map :id (source/metrics source/app-db-source #{metrics-id (:id nested-metrics)})))))))))))))++(deftest table-publication-lifecycle-test+ (data-apps.tu/do-with-library!+ (fn [{:keys [data-id]}]+ (mt/with-temp [:model/Database db {}+ :model/Table table {:db_id (:id db), :name "widgets", :is_published true, :collection_id data-id}+ :model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]+ (mt/with-test-user :crowberto+ (is (= [(:id table)] (map :id (source/library-tables source/app-db-source #{data-id}))))+ (doseq [changes [{:is_published false} {:collection_id nil} {:active false}]]+ (testing (str "table eligibility after " changes)+ (mt/with-temp-vals-in-db :model/Table (:id table) changes+ (is (= [] (source/library-tables source/app-db-source #{data-id}))))))+ (testing "an archived Library collection is excluded"+ (mt/with-temp-vals-in-db :model/Collection data-id {:archived true}+ (is (= [] (source/library-tables source/app-db-source #{data-id}))))))+ (testing "Library publication does not bypass read permissions"+ (mt/with-no-data-perms-for-all-users!+ (mt/with-test-user :rasta+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"permissions"+ (source/tables source/app-db-source #{(:id table)}))))))))))++(deftest only-published-data-actions-are-exported-test+ (mt/with-actions-enabled+ (let [mp (mt/metadata-provider)+ query (lib/native-query mp "UPDATE categories SET name = name")+ model-query (lib/query mp (lib.metadata/table mp (mt/id :categories)))]+ (mt/with-temp [:model/Collection data-actions {:namespace "data-actions"}+ :model/Collection archived-collection {:namespace "data-actions", :archived true}+ :model/Collection app-collection {:namespace "data-apps"}+ :model/Card model {:type :model, :dataset_query model-query}+ :model/Action root {:name "Root data action", :type :query}+ :model/QueryAction _ {:action_id (:id root), :dataset_query query}+ :model/Action nested {:name "Nested data action", :type :query, :collection_id (:id data-actions)}+ :model/QueryAction _ {:action_id (:id nested), :dataset_query query}+ :model/Action attached {:name "Model action", :type :query, :model_id (:id model)}+ :model/QueryAction _ {:action_id (:id attached), :dataset_query query}+ :model/Action copy {:name "App copy action", :type :query, :collection_id (:id app-collection)}+ :model/QueryAction _ {:action_id (:id copy), :dataset_query query}+ :model/Action archived {:name "Archived data action", :type :query, :archived true}+ :model/QueryAction _ {:action_id (:id archived), :dataset_query query}+ :model/Action in-archived {:name "Archived collection action", :type :query+ :collection_id (:id archived-collection)}+ :model/QueryAction _ {:action_id (:id in-archived), :dataset_query query}]+ (let [ours (into #{} (map :id) [root nested attached copy archived in-archived])]+ (mt/with-test-user :crowberto+ (testing "only standalone actions can be exported"+ (is (= #{(:id root) (:id nested)}+ (set/intersection ours (into #{} (map :id) (source/actions source/app-db-source))))))+ (testing "the response includes root and nested actions and excludes other actions"+ (mt/with-premium-features #{:data-apps}+ (let [body (:body (mt/user-http-request-full-response+ :crowberto :get 200 "apps/generate/schemas"))]+ (doseq [included ["rootDataAction: {" "nestedDataAction: {"]]+ (is (str/includes? body included) included))+ (doseq [excluded ["modelAction" "appCopyAction" "archivedDataAction" "archivedCollectionAction"]]+ (is (not (str/includes? body excluded)) excluded)))))+ (testing "moving a published action into an app collection immediately removes it"+ (mt/with-temp-vals-in-db :model/Action (:id nested) {:collection_id (:id app-collection)}+ (is (= #{(:id root)}+ (set/intersection ours (into #{} (map :id) (source/actions source/app-db-source)))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.cljindex 939104b350d..13ed3f7e937 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj@@ -13,7 +13,6 @@[metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.permissions.core :as perms][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util.malli.registry :as mr]@@ -71,14 +70,16 @@:model/Table table {:db_id (:id db), :name "widgets", :display_name "Widgets", :active true}:model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}](mt/with-current-user (mt/user->id :crowberto)- (testing "returns shaped table entities for the table ids"- (is (=? [{:type "table"- :key "widgets"- :fields {"price" {:jsType "number"}}}]- (schemas.source/tables schemas.source/app-db-source #{(:id table)}))))+ (testing "explicit table ids include unpublished metric backing tables"+ (is (= [(:id table)] (map :id (schemas.source/tables schemas.source/app-db-source #{(:id table)})))))(testing "an empty table-id set matches nothing"(is (= [] (schemas.source/tables schemas.source/app-db-source #{})))))))+(deftest empty-table-request-does-not-query-test+ (t2/with-call-count [call-count]+ (is (= [] (schemas.source/tables schemas.source/app-db-source #{})))+ (is (zero? (call-count)))))+(defn- touch-category-query [](let [query (lib/native-query (mt/metadata-provider) "UPDATE categories SET name = name WHERE id = {{id}}")](lib/with-template-tags query (mapv #(assoc % :type :number) (lib/template-tags query)))))@@ -106,11 +107,8 @@(let [action-ids (fn [] (into #{} (map :id) (schemas.source/actions schemas.source/app-db-source)))ours #{standalone archived in-collection on-model app-copy}](mt/with-current-user (mt/user->id :crowberto)- (testing "returns the unarchived actions without a model"- (is (= #{standalone in-collection app-copy} (set/intersection ours (action-ids)))))- (testing "leaves out the copies a data app owns"- (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{app-collection-id})]- (is (= #{standalone in-collection} (set/intersection ours (action-ids))))))+ (testing "returns unarchived actions without a model and excludes data app copies"+ (is (= #{standalone in-collection} (set/intersection ours (action-ids)))))(testing "an action renders with its parameters typed from the template tag"(is (=? [{:kind "action":key "touchCategory"diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.cljindex b8ae8d965d3..cbc26c482d7 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj@@ -5,7 +5,6 @@[metabase-enterprise.data-apps.test-util :as data-apps.tu][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.permissions.core :as perms][metabase.test :as mt][metabase.test.fixtures :as fixtures][toucan2.core :as t2]))@@ -79,6 +78,4 @@(is (str/includes? body "discountOrder"))(is (not (str/includes? body "updateOrder"))))(testing "a copy a data app owns stays out"- (is (str/includes? body "copiedOrder"))- (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{(:id copies)})]- (is (not (str/includes? (:body (get-schemas :crowberto 200)) "copiedOrder"))))))))))))))+ (is (not (str/includes? body "copiedOrder")))))))))))))