Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Strictly limit generated data app schemas to library-published content (#83548)

Phoomparin Mano9deb406fa0cd
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
index 35d50f66103..84055b183d1 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
@@ -321,6 +321,14 @@
[:= :model_id nil]
[:= :type "query"]
[:= :archived false]
+ [:or
+ [:= :collection_id nil]
+ [:exists ^:allow-subquery {:select [1]
+ :from [[(t2/table-name :model/Collection) :c]]
+ :where [:and
+ [:= :c.id :action.collection_id]
+ [:= :c.archived false]
+ [:= :c.namespace "data-actions"]]}]]
(when (seq excluded-collection-ids)
[:or [:= :collection_id nil] [:not-in :collection_id excluded-collection-ids]])]
:order-by [[:name :asc] [:id :asc]]}))
@@ -363,7 +371,11 @@
[measure-ids :- [:sequential ::lib.schema.id/measure]]
(t2/select [:model/Measure :id :definition] :id [:in measure-ids]))
-(mu/defn collections-with-entity-ids
- "The Collections with `entity-ids`."
- [entity-ids :- [:set :string]]
- (t2/select :model/Collection :entity_id [:in entity-ids]))
+(mu/defn unarchived-collections
+ "The unarchived Collections among `collection-ids`, with their Library types."
+ [collection-ids :- [:set ::lib.schema.id/collection]]
+ (if (seq collection-ids)
+ (t2/select [:model/Collection :id :type]
+ :id [:in collection-ids]
+ :archived false)
+ []))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj
index e06b497ac4d..fc602a9ac4c 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas.clj
@@ -30,8 +30,7 @@
[:metrics [:sequential :map]]])
(defn fetch-items
- "Fetches the library's metrics, its published tables and the tables those metrics map, and the query actions that
- belong to no model."
+ "Fetches Library metrics, published tables and metric backing tables, and eligible query actions without a model."
([]
(fetch-items schemas.source/app-db-source))
([source]
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj
index 23881a7e315..6c61ff17ecf 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/action.clj
@@ -78,8 +78,8 @@
:entityId entity_id)))
(defn action-schemas
- "Returns schema entries for the query actions without a model, leaving out the copies data apps own and those a
- routing destination backs."
+ "Returns schemas for standalone actions, excluding copies owned by data app and
+ those a destination backs."
[]
(let [ids (data-apps.db/model-less-query-action-ids (set (perms/data-app-collection-ids)))
details-by-id (when (seq ids) (u/index-by :id (actions/select-actions-for-ids nil ids)))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj
index b496aad6737..cb3f13b054f 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/generate/schemas/source.clj
@@ -7,18 +7,16 @@
`metabase-enterprise.data-apps.generate.schemas.*` builders, except [[library-tables]], which returns raw table rows
(only `:id` is consumed)."
(:require
+ [clojure.set :as set]
[metabase-enterprise.data-apps.db :as data-apps.db]
[metabase-enterprise.data-apps.generate.schemas.action :as schemas.action]
[metabase-enterprise.data-apps.generate.schemas.metric :as schemas.metric]
[metabase-enterprise.data-apps.generate.schemas.table :as schemas.table]
+ [metabase.collections.core :as collections]
[metabase.collections.models.collection :as collection]))
(set! *warn-on-reflection* true)
-(def ^:private library-root-entity-ids
- "Entity ids of the root data library and root metrics library collections."
- #{"librarylibrarydatadat" "librarylibrarymetrics"})
-
(def LibraryScope
"The library collection tree, classified by collection type: `:data-collection-ids` hold the published tables,
`:metric-collection-ids` the metrics."
@@ -29,9 +27,10 @@
(defn- app-db-library-scope
"The [[LibraryScope]] of the root libraries and their descendants, empty where the instance has no library."
[]
- (let [roots (filter #(contains? collection/library-collection-types (:type %))
- (data-apps.db/collections-with-entity-ids library-root-entity-ids))
- collections (concat roots (when (seq roots) (collection/descendants-flat-for roots)))
+ (let [library (collections/library-collection)
+ descendants (when (and library (not (:archived library)))
+ (collection/descendants-flat-for [library]))
+ collections (data-apps.db/unarchived-collections (into #{} (map :id) descendants))
ids-of-type (fn [collection-type]
(into #{} (comp (filter #(= (:type %) collection-type)) (map :id)) collections))]
{:data-collection-ids (ids-of-type collection/library-data-collection-type)
@@ -50,6 +49,10 @@
(library-tables [source collection-ids]
"Published table rows in `collection-ids`."))
+(defn- curated-library-ids
+ [scope-key requested-ids]
+ (set/intersection (scope-key (app-db-library-scope)) requested-ids))
+
(def app-db-source
"The production [[SchemaSource]], backed by the application database."
(reify SchemaSource
@@ -58,8 +61,10 @@
(actions [_]
(schemas.action/action-schemas))
(metrics [_ collection-ids]
- (vec (schemas.metric/metric-schemas collection-ids)))
+ (vec (schemas.metric/metric-schemas
+ (curated-library-ids :metric-collection-ids collection-ids))))
(tables [_ table-ids]
(vec (schemas.table/table-schemas (schemas.table/select-tables table-ids))))
(library-tables [_ collection-ids]
- (vec (schemas.table/select-library-tables collection-ids)))))
+ (vec (schemas.table/select-library-tables
+ (curated-library-ids :data-collection-ids collection-ids))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.clj
new file mode 100644
index 00000000000..638774e7efa
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/library_test.clj
@@ -0,0 +1,143 @@
+(ns metabase-enterprise.data-apps.generate.schemas.library-test
+ (:require
+ [clojure.set :as set]
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.data-apps.generate.schemas :as schemas]
+ [metabase-enterprise.data-apps.generate.schemas.source :as source]
+ [metabase-enterprise.data-apps.test-util :as data-apps.tu]
+ [metabase.collections.models.collection :as collection]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(use-fixtures :once (fixtures/initialize :db :web-server :test-users))
+
+(deftest library-content-and-metric-backing-tables-are-exported-test
+ (mt/dataset test-data
+ (data-apps.tu/do-with-library!
+ (fn [{:keys [data-id metrics-id]}]
+ (let [mp (mt/metadata-provider)
+ orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ query (lib/aggregate orders-query
+ (lib/sum (lib.metadata/field mp (mt/id :orders :total))))]
+ (mt/with-temp [:model/Collection nested-data {:type "library-data"
+ :location (collection/children-location
+ (t2/select-one :model/Collection :id data-id))}
+ :model/Collection nested-metrics {:type "library-metrics"
+ :location (collection/children-location
+ (t2/select-one :model/Collection :id metrics-id))}
+ :model/Collection outside {}
+ :model/Collection lookalike {:type "library-data"}
+ :model/Table published {:db_id (mt/id), :name "library_widgets"
+ :is_published true, :collection_id data-id}
+ :model/Field _ {:table_id (:id published), :name "price", :base_type :type/Float}
+ :model/Table nested {:db_id (mt/id), :name "nested_widgets"
+ :is_published true, :collection_id (:id nested-data)}
+ :model/Table _unpublished {:db_id (mt/id), :name "unpublished_widgets"
+ :is_published false, :collection_id data-id}
+ :model/Table _elsewhere {:db_id (mt/id), :name "outside_widgets"
+ :is_published true, :collection_id (:id outside)}
+ :model/Table _lookalike {:db_id (mt/id), :name "lookalike_widgets"
+ :is_published true, :collection_id (:id lookalike)}
+ :model/Card metric {:name "Library revenue", :type :metric, :display :scalar
+ :database_id (mt/id), :collection_id metrics-id, :dataset_query query}
+ :model/Card nested-metric {:name "Nested revenue", :type :metric, :display :scalar
+ :database_id (mt/id), :collection_id (:id nested-metrics)
+ :dataset_query query}
+ :model/Card _outside-metric {:name "Outside revenue", :type :metric, :display :scalar
+ :database_id (mt/id), :collection_id (:id outside)
+ :dataset_query query}
+ :model/Card _archived-metric {:name "Archived revenue", :type :metric, :display :scalar
+ :database_id (mt/id), :collection_id metrics-id
+ :dataset_query query, :archived true}
+ :model/Card _model {:name "Outside model", :type :model, :database_id (mt/id)
+ :collection_id (:id outside), :dataset_query orders-query}]
+ (mt/with-test-user :crowberto
+ (testing "items published to libraries can be exported"
+ (let [items (schemas/fetch-items)]
+ (is (= #{(:id published) (:id nested) (mt/id :orders)} (into #{} (map :id) (:tables items))))
+ (is (= #{(:id metric) (:id nested-metric)} (into #{} (map :id) (:metrics items))))
+ (is (= #{(mt/id :orders)} (into #{} (mapcat :mappedTableIds) (:metrics items))))))
+ (testing "only collection IDs from the library can be exported"
+ (is (= [] (source/library-tables source/app-db-source #{(:id outside) (:id lookalike)})))
+ (is (= [] (source/metrics source/app-db-source #{(:id outside)}))))
+ ; we pick the metric dimensions from table fields, therefore tables
+ ; that backed metrics are included in the export, even if they are not published to the library
+ (testing "the response includes metric backing tables and excludes unrelated content"
+ (mt/with-premium-features #{:data-apps}
+ (let [body (:body (mt/user-http-request-full-response
+ :crowberto :get 200 "apps/generate/schemas"))]
+ (doseq [included ["libraryWidgets: {" "nestedWidgets: {" "libraryRevenue: {"
+ "nestedRevenue: {" "orders: {"]]
+ (is (str/includes? body included) included))
+ (doseq [excluded ["unpublishedWidgets" "outsideWidgets" "lookalikeWidgets" "outsideRevenue"
+ "archivedRevenue" "outsideModel" "models:"]]
+ (is (not (str/includes? body excluded)) excluded)))))
+ (testing "moving a metric out of the library immediately removes it"
+ (mt/with-temp-vals-in-db :model/Card (:id metric) {:collection_id (:id outside)}
+ (is (= [(:id nested-metric)]
+ (map :id (source/metrics source/app-db-source #{metrics-id (:id nested-metrics)})))))))))))))
+
+(deftest table-publication-lifecycle-test
+ (data-apps.tu/do-with-library!
+ (fn [{:keys [data-id]}]
+ (mt/with-temp [:model/Database db {}
+ :model/Table table {:db_id (:id db), :name "widgets", :is_published true, :collection_id data-id}
+ :model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]
+ (mt/with-test-user :crowberto
+ (is (= [(:id table)] (map :id (source/library-tables source/app-db-source #{data-id}))))
+ (doseq [changes [{:is_published false} {:collection_id nil} {:active false}]]
+ (testing (str "table eligibility after " changes)
+ (mt/with-temp-vals-in-db :model/Table (:id table) changes
+ (is (= [] (source/library-tables source/app-db-source #{data-id}))))))
+ (testing "an archived Library collection is excluded"
+ (mt/with-temp-vals-in-db :model/Collection data-id {:archived true}
+ (is (= [] (source/library-tables source/app-db-source #{data-id}))))))
+ (testing "Library publication does not bypass read permissions"
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-test-user :rasta
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"permissions"
+ (source/tables source/app-db-source #{(:id table)}))))))))))
+
+(deftest only-published-data-actions-are-exported-test
+ (mt/with-actions-enabled
+ (let [mp (mt/metadata-provider)
+ query (lib/native-query mp "UPDATE categories SET name = name")
+ model-query (lib/query mp (lib.metadata/table mp (mt/id :categories)))]
+ (mt/with-temp [:model/Collection data-actions {:namespace "data-actions"}
+ :model/Collection archived-collection {:namespace "data-actions", :archived true}
+ :model/Collection app-collection {:namespace "data-apps"}
+ :model/Card model {:type :model, :dataset_query model-query}
+ :model/Action root {:name "Root data action", :type :query}
+ :model/QueryAction _ {:action_id (:id root), :dataset_query query}
+ :model/Action nested {:name "Nested data action", :type :query, :collection_id (:id data-actions)}
+ :model/QueryAction _ {:action_id (:id nested), :dataset_query query}
+ :model/Action attached {:name "Model action", :type :query, :model_id (:id model)}
+ :model/QueryAction _ {:action_id (:id attached), :dataset_query query}
+ :model/Action copy {:name "App copy action", :type :query, :collection_id (:id app-collection)}
+ :model/QueryAction _ {:action_id (:id copy), :dataset_query query}
+ :model/Action archived {:name "Archived data action", :type :query, :archived true}
+ :model/QueryAction _ {:action_id (:id archived), :dataset_query query}
+ :model/Action in-archived {:name "Archived collection action", :type :query
+ :collection_id (:id archived-collection)}
+ :model/QueryAction _ {:action_id (:id in-archived), :dataset_query query}]
+ (let [ours (into #{} (map :id) [root nested attached copy archived in-archived])]
+ (mt/with-test-user :crowberto
+ (testing "only standalone actions can be exported"
+ (is (= #{(:id root) (:id nested)}
+ (set/intersection ours (into #{} (map :id) (source/actions source/app-db-source))))))
+ (testing "the response includes root and nested actions and excludes other actions"
+ (mt/with-premium-features #{:data-apps}
+ (let [body (:body (mt/user-http-request-full-response
+ :crowberto :get 200 "apps/generate/schemas"))]
+ (doseq [included ["rootDataAction: {" "nestedDataAction: {"]]
+ (is (str/includes? body included) included))
+ (doseq [excluded ["modelAction" "appCopyAction" "archivedDataAction" "archivedCollectionAction"]]
+ (is (not (str/includes? body excluded)) excluded)))))
+ (testing "moving a published action into an app collection immediately removes it"
+ (mt/with-temp-vals-in-db :model/Action (:id nested) {:collection_id (:id app-collection)}
+ (is (= #{(:id root)}
+ (set/intersection ours (into #{} (map :id) (source/actions source/app-db-source)))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj
index 939104b350d..13ed3f7e937 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas/source_test.clj
@@ -13,7 +13,6 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.permissions.core :as perms]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util.malli.registry :as mr]
@@ -71,14 +70,16 @@
:model/Table table {:db_id (:id db), :name "widgets", :display_name "Widgets", :active true}
:model/Field _ {:table_id (:id table), :name "price", :base_type :type/Float}]
(mt/with-current-user (mt/user->id :crowberto)
- (testing "returns shaped table entities for the table ids"
- (is (=? [{:type "table"
- :key "widgets"
- :fields {"price" {:jsType "number"}}}]
- (schemas.source/tables schemas.source/app-db-source #{(:id table)}))))
+ (testing "explicit table ids include unpublished metric backing tables"
+ (is (= [(:id table)] (map :id (schemas.source/tables schemas.source/app-db-source #{(:id table)})))))
(testing "an empty table-id set matches nothing"
(is (= [] (schemas.source/tables schemas.source/app-db-source #{})))))))
+(deftest empty-table-request-does-not-query-test
+ (t2/with-call-count [call-count]
+ (is (= [] (schemas.source/tables schemas.source/app-db-source #{})))
+ (is (zero? (call-count)))))
+
(defn- touch-category-query []
(let [query (lib/native-query (mt/metadata-provider) "UPDATE categories SET name = name WHERE id = {{id}}")]
(lib/with-template-tags query (mapv #(assoc % :type :number) (lib/template-tags query)))))
@@ -106,11 +107,8 @@
(let [action-ids (fn [] (into #{} (map :id) (schemas.source/actions schemas.source/app-db-source)))
ours #{standalone archived in-collection on-model app-copy}]
(mt/with-current-user (mt/user->id :crowberto)
- (testing "returns the unarchived actions without a model"
- (is (= #{standalone in-collection app-copy} (set/intersection ours (action-ids)))))
- (testing "leaves out the copies a data app owns"
- (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{app-collection-id})]
- (is (= #{standalone in-collection} (set/intersection ours (action-ids))))))
+ (testing "returns unarchived actions without a model and excludes data app copies"
+ (is (= #{standalone in-collection} (set/intersection ours (action-ids)))))
(testing "an action renders with its parameters typed from the template tag"
(is (=? [{:kind "action"
:key "touchCategory"
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj
index b8ae8d965d3..cbc26c482d7 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/schemas_api_test.clj
@@ -5,7 +5,6 @@
[metabase-enterprise.data-apps.test-util :as data-apps.tu]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
- [metabase.permissions.core :as perms]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
@@ -79,6 +78,4 @@
(is (str/includes? body "discountOrder"))
(is (not (str/includes? body "updateOrder"))))
(testing "a copy a data app owns stays out"
- (is (str/includes? body "copiedOrder"))
- (mt/with-dynamic-fn-redefs [perms/data-app-collection-ids (constantly #{(:id copies)})]
- (is (not (str/includes? (:body (get-schemas :crowberto 200)) "copiedOrder"))))))))))))))
+ (is (not (str/includes? body "copiedOrder")))))))))))))