The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't use persisted Model cache for non-standard requests (#82579)
Nathan Voxland16aff535a6d0
diff --git a/.clj-kondo/ratchets-test.edn b/.clj-kondo/ratchets-test.ednindex b354b24a5fd..0e9a87cb674 100644--- a/.clj-kondo/ratchets-test.edn+++ b/.clj-kondo/ratchets-test.edn@@ -49,7 +49,7 @@:entity-retrieval.core/search-unfiltered 1:entity-retrieval.mirror/search-unfiltered 1:lib.equality/find-column-indexes-for-refs 2- :test/with-temp 28}+ :test/with-temp 30}:comment-exempt #{:deprecated-namespace:discouraged-namespace:discouraged-vardiff --git a/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.cljnew file mode 100644index 00000000000..0c26f67cc94--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.clj@@ -0,0 +1,58 @@+(ns ^:mb/driver-tests metabase-enterprise.database-routing.persistence-test+ "A Model's persisted cache table holds the results as seen through the Router Database's own connection. A user+ routed to a Destination Database must never read from it, whether the Model is referenced as a source table or via+ a native `{{#N}}` template tag."+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.database-routing.persistence-test]}}}}}}+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase-enterprise.test :as met]+ [metabase.driver :as driver]+ [metabase.lib.core :as lib]+ [metabase.test :as mt]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(deftest persistence-disabled-when-db-routed-test+ (mt/test-drivers (mt/normal-drivers-with-feature :persist-models :database-routing)+ (mt/with-premium-features #{:database-routing}+ (mt/dataset test-data+ (mt/with-persistence-enabled! [persist-models!]+ (mt/with-temp [:model/Card model {:type :model+ :dataset_query (mt/mbql-query products)}]+ (mt/as-admin (persist-models!))+ (let [persisted-info (t2/select-one :model/PersistedInfo :database_id (mt/id) :card_id (:id model))]+ (is (= "persisted" (:state persisted-info)))+ (met/with-user-attributes! :rasta {"db_name" "destination-db"}+ ;; The Destination Database points at the very same physical database as the Router, so the persisted+ ;; cache table is reachable through it: exactly the setup where reading the cache would bypass whatever+ ;; the destination connection is meant to restrict.+ (mt/with-temp [:model/DatabaseRouter _ {:database_id (mt/id)+ :user_attribute "db_name"}+ :model/Database _ {:engine driver/*driver*+ :name "destination-db"+ :details (:details (mt/db))+ :router_database_id (mt/id)}+ :model/Card source-card {:dataset_query (mt/mbql-query nil+ {:aggregation [:count]+ :source-table (str "card__" (:id model))})}+ :model/Card native-card {:dataset_query+ (lib/native-query (mt/metadata-provider)+ (format "SELECT count(*) FROM {{#%d}} AS m"+ (:id model)))}]+ (letfn [(run-card [user card]+ (mt/user-http-request user :post 202 (format "card/%d/query" (u/the-id card))))]+ (doseq [[description card] [["Model as a source table" source-card]+ ["Model referenced via a native {{#N}} template tag" native-card]]]+ (testing description+ (let [routed-result (run-card :rasta card)+ admin-result (run-card :crowberto card)]+ (testing "routed user (rasta) does not hit the model cache"+ (is (= 200 (-> routed-result mt/rows ffirst)))+ (is (not (str/includes? (-> routed-result :data :native_form :query)+ (:table_name persisted-info)))+ "Erroneously used the persisted model cache"))+ (testing "admin (resolves to the Router Database) hits the model cache"+ (is (str/includes? (-> admin-result :data :native_form :query)+ (:table_name persisted-info))+ "Did not use the persisted model cache")))))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.cljindex 193021344e5..3b719fed9b3 100644--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj@@ -845,7 +845,27 @@(testing "Query from admin hits the model cache"(is (str/includes? (-> admin-result :data :native_form :query)(:table_name persisted-info))- "Did not use the persisted model cache")))+ "Did not use the persisted model cache"))+ (testing "Native query referencing the model via a {{#N}} template tag"+ ;; an impersonated user only needs to be able to *read* a saved native Card that references the+ ;; persisted Model to hit this code path+ (mt/with-temp [:model/Card card {:dataset_query+ (lib/native-query (mt/metadata-provider)+ (format "SELECT count(*) FROM {{#%d}} AS m"+ (:id model)))}]+ (let [run-card #(mt/user-http-request % :post 202 (format "card/%d/query" (:id card)))+ impersonated-result (run-card :rasta)+ ;; Make sure we run admin query second to reset the DB role on the connection!+ admin-result (run-card :crowberto)]+ (testing "Impersonated user (rasta) does not hit the model cache"+ (is (= 200 (-> impersonated-result mt/rows ffirst)))+ (is (not (str/includes? (-> impersonated-result :data :native_form :query)+ (:table_name persisted-info)))+ "Erroneously used the persisted model cache"))+ (testing "Query from admin hits the model cache"+ (is (str/includes? (-> admin-result :data :native_form :query)+ (:table_name persisted-info))+ "Did not use the persisted model cache"))))))(finally(doseq [statement ["REVOKE ALL PRIVILEGES ON TABLE \"products\" FROM \"impersonation_role\";""DROP ROLE IF EXISTS \"impersonation_role\";"]]diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 9275ce39a02..1c57363f5b3 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -1363,6 +1363,58 @@(:table_name persisted-info)))"Erroneously used the persisted model cache")))))))))))+(deftest persistence-disabled-for-native-card-template-tag-when-sandboxed-test+ (testing "A native query referencing a persisted Model via a {{#N}} template tag must not use the persisted cache for a sandboxed user"+ (mt/test-drivers (mt/normal-drivers-with-feature :persist-models)+ (mt/dataset test-data+ ;; with-gtaps! creates a new copy of the database. So make sure to do that before anything else.+ (met/with-gtaps! {:gtaps {:products+ {:remappings {:category+ ["dimension"+ [:field (mt/id :products :category)+ nil]]}}}}+ (mt/with-persistence-enabled! [persist-models!]+ ;; with temp ok here since `persist-models!` doesn't use Metadata Providers+ #_{:clj-kondo/ignore [:discouraged-var]}+ (mt/with-temp [:model/Card model {:type :model+ :dataset_query (mt/mbql-query+ products+ {:fields [$id $price]})}]+ ;; persist model (as admin, so sandboxing is not applied to the persisted query)+ (mt/with-test-user :crowberto+ (persist-models!))+ (let [persisted-info (t2/select-one :model/PersistedInfo+ :database_id (mt/id)+ :card_id (:id model))]+ (is (= "persisted" (:state persisted-info))+ "Model failed to persist")+ ;; a sandboxed user has no native-query permissions, but only needs to be able to *read* a saved+ ;; native Card that references the persisted Model to hit this code path.+ #_{:clj-kondo/ignore [:discouraged-var]}+ (mt/with-temp [:model/Card card {:dataset_query+ (lib/native-query (mt/metadata-provider)+ (format "SELECT count(*) FROM {{#%d}} AS m"+ (:id model)))}]+ (let [run-card #(mt/user-http-request % :post 202 (format "card/%d/query" (:id card)))+ regular-result (run-card :crowberto)+ sandboxed-result (met/with-user-attributes! :rasta {"category" "Gizmo"}+ (run-card :rasta))]+ (testing "Unsandboxed"+ (testing "Sees full result set"+ (is (= 200 (-> regular-result mt/rows ffirst))))+ (testing "Uses the cache table"+ (is (str/includes? (-> regular-result :data :native_form :query)+ (:table_name persisted-info))+ "Did not use the persisted model cache")))+ (testing "Sandboxed"+ (testing "sees partial result"+ (is (= 51 (-> sandboxed-result mt/rows ffirst))+ "Sandboxed user got whole results instead of filtered"))+ (testing "Does not use the cache table"+ (is (not (str/includes? (-> sandboxed-result :data :native_form :query)+ (:table_name persisted-info)))+ "Erroneously used the persisted model cache")))))))))))))+(deftest model-metadata-overrides-preserved-for-sandboxed-users-test(testing (str "Column metadata overrides on a Model (custom display_name, semantic_type set in the Edit Metadata ""screen) should apply to queries sourced from that Model regardless of whether the user has a "diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.cljindex e2b891270c2..df729a10217 100644--- a/src/metabase/permissions/core.clj+++ b/src/metabase/permissions/core.clj@@ -168,7 +168,8 @@[metabase.permissions.data-access-tokendata-access-tokendata-access-compatible?- data-access-token-transform])+ data-access-token-transform+ routing-token-for-db]);;; import these vars with different names to make their purpose more obvious.(p/import-def metabase.permissions.models.permissions-group/all-users all-users-group)diff --git a/src/metabase/query_processor/middleware/persistence.clj b/src/metabase/query_processor/middleware/persistence.cljindex a02e4059474..9d19e98ef7a 100644--- a/src/metabase/query_processor/middleware/persistence.clj+++ b/src/metabase/query_processor/middleware/persistence.clj@@ -5,17 +5,16 @@[metabase.util.match :as match]))(defn substitute-persisted-query- "Removes persisted information if user is sandboxed or uses connection impersonation. `:persisted-info/native` is set- in [[metabase.query-processor.middleware.fetch-source-query]].-- Sandboxing is detected by the presence of a :query-permissions/sandboxed-table key anywhere in the provided query+ "Removes persisted information if user is sandboxed, uses connection impersonation, or is routed to a Destination+ Database. `:persisted-info/native` is set in [[metabase.query-processor.middleware.fetch-source-query]].It may be be possible to use the persistence cache with sandboxing and/or impersonation at a later date with furtherwork, but for now we skip the cache in these cases."[query](if (and api/*current-user-id*(or (match/match-one query {:query-permissions/sandboxed-table &truthy} true)- (perms/impersonation-enforced-for-db? (:database query))))+ (perms/impersonation-enforced-for-db? (:database query))+ (:destination-database/id query)))(match/replace query{:persisted-info/native &truthy};; Signal to the SQL QP's independent persisted-cache lookup that it should not use the cache for thisdiff --git a/src/metabase/query_processor/parameters/values.clj b/src/metabase/query_processor/parameters/values.cljindex d6f5f133b9b..844da775f04 100644--- a/src/metabase/query_processor/parameters/values.clj+++ b/src/metabase/query_processor/parameters/values.clj@@ -11,6 +11,7 @@(:refer-clojure :exclude [every? some mapv not-empty get-in])(:require[clojure.string :as str]+ [metabase.api.common :as api][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.parameters.parse.types :as params.types]@@ -20,6 +21,7 @@[metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.schema.template-tag :as lib.schema.template-tag]+ [metabase.permissions.core :as perms][metabase.query-processor.compile :as qp.compile][metabase.query-processor.core :as qp][metabase.query-processor.error-type :as qp.error-type]@@ -219,6 +221,19 @@value (field-filter-value tag params)](lib/parsed-field-filter-param field value alias)))+(defn- persisted-cache-allowed-for-current-user?+ "Whether the current user may read directly from a persisted Model cache table. The cache holds the Model's full,+ unfiltered results as seen through the Database's own connection, so it must never be substituted for a user whose+ queries would otherwise be rewritten by sandboxing or connection impersonation, or routed to a Destination Database.+ Substituting the cache here skips [[qp.compile/compile]] (and thus preprocessing) for the referenced Model entirely,+ so we cannot rely on the [[metabase.query-processor.middleware.persistence/substitute-persisted-query]] middleware+ to disable it for us."+ [database-id]+ (not (and api/*current-user-id*+ (or (perms/sandboxed-user-for-db? database-id)+ (perms/impersonation-enforced-for-db? database-id)+ (perms/routing-token-for-db database-id)))))+(mu/defmethod parse-tag :card :- ::params.types/referenced-card-query[metadata-providerable :- ::lib.schema.metadata/metadata-providerable{:keys [card-id], :as tag} :- ::lib.schema.template-tag/template-tag@@ -235,10 +250,10 @@(try(log/tracef "Compiling referenced query for Card %d" card-id)(let [query (assoc query :info {:card-id card-id})- {:keys [query params]} (or (when (qp.persistence/can-substitute? card persisted-info)- {:query (qp.persistence/persisted-info-native-query- (u/the-id (lib.metadata/database metadata-providerable))- persisted-info)})+ database-id (u/the-id (lib.metadata/database metadata-providerable))+ {:keys [query params]} (or (when (and (qp.persistence/can-substitute? card persisted-info)+ (persisted-cache-allowed-for-current-user? database-id))+ {:query (qp.persistence/persisted-info-native-query database-id persisted-info)})(qp.compile/compile (qp/disable-max-results query)))](lib/parsed-referenced-card-query-param card-id query params))(catch ExceptionInfo e