Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Don't use persisted Model cache for non-standard requests (#82579)

Nathan Voxland16aff535a6d0
diff --git a/.clj-kondo/ratchets-test.edn b/.clj-kondo/ratchets-test.edn
index b354b24a5fd..0e9a87cb674 100644
--- a/.clj-kondo/ratchets-test.edn
+++ b/.clj-kondo/ratchets-test.edn
@@ -49,7 +49,7 @@
:entity-retrieval.core/search-unfiltered 1
:entity-retrieval.mirror/search-unfiltered 1
:lib.equality/find-column-indexes-for-refs 2
- :test/with-temp 28}
+ :test/with-temp 30}
:comment-exempt #{:deprecated-namespace
:discouraged-namespace
:discouraged-var
diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.clj
new file mode 100644
index 00000000000..0c26f67cc94
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/database_routing/persistence_test.clj
@@ -0,0 +1,58 @@
+(ns ^:mb/driver-tests metabase-enterprise.database-routing.persistence-test
+ "A Model's persisted cache table holds the results as seen through the Router Database's own connection. A user
+ routed to a Destination Database must never read from it, whether the Model is referenced as a source table or via
+ a native `{{#N}}` template tag."
+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.database-routing.persistence-test]}}}}}}
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase-enterprise.test :as met]
+ [metabase.driver :as driver]
+ [metabase.lib.core :as lib]
+ [metabase.test :as mt]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(deftest persistence-disabled-when-db-routed-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :persist-models :database-routing)
+ (mt/with-premium-features #{:database-routing}
+ (mt/dataset test-data
+ (mt/with-persistence-enabled! [persist-models!]
+ (mt/with-temp [:model/Card model {:type :model
+ :dataset_query (mt/mbql-query products)}]
+ (mt/as-admin (persist-models!))
+ (let [persisted-info (t2/select-one :model/PersistedInfo :database_id (mt/id) :card_id (:id model))]
+ (is (= "persisted" (:state persisted-info)))
+ (met/with-user-attributes! :rasta {"db_name" "destination-db"}
+ ;; The Destination Database points at the very same physical database as the Router, so the persisted
+ ;; cache table is reachable through it: exactly the setup where reading the cache would bypass whatever
+ ;; the destination connection is meant to restrict.
+ (mt/with-temp [:model/DatabaseRouter _ {:database_id (mt/id)
+ :user_attribute "db_name"}
+ :model/Database _ {:engine driver/*driver*
+ :name "destination-db"
+ :details (:details (mt/db))
+ :router_database_id (mt/id)}
+ :model/Card source-card {:dataset_query (mt/mbql-query nil
+ {:aggregation [:count]
+ :source-table (str "card__" (:id model))})}
+ :model/Card native-card {:dataset_query
+ (lib/native-query (mt/metadata-provider)
+ (format "SELECT count(*) FROM {{#%d}} AS m"
+ (:id model)))}]
+ (letfn [(run-card [user card]
+ (mt/user-http-request user :post 202 (format "card/%d/query" (u/the-id card))))]
+ (doseq [[description card] [["Model as a source table" source-card]
+ ["Model referenced via a native {{#N}} template tag" native-card]]]
+ (testing description
+ (let [routed-result (run-card :rasta card)
+ admin-result (run-card :crowberto card)]
+ (testing "routed user (rasta) does not hit the model cache"
+ (is (= 200 (-> routed-result mt/rows ffirst)))
+ (is (not (str/includes? (-> routed-result :data :native_form :query)
+ (:table_name persisted-info)))
+ "Erroneously used the persisted model cache"))
+ (testing "admin (resolves to the Router Database) hits the model cache"
+ (is (str/includes? (-> admin-result :data :native_form :query)
+ (:table_name persisted-info))
+ "Did not use the persisted model cache")))))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
index 193021344e5..3b719fed9b3 100644
--- a/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/impersonation/driver_test.clj
@@ -845,7 +845,27 @@
(testing "Query from admin hits the model cache"
(is (str/includes? (-> admin-result :data :native_form :query)
(:table_name persisted-info))
- "Did not use the persisted model cache")))
+ "Did not use the persisted model cache"))
+ (testing "Native query referencing the model via a {{#N}} template tag"
+ ;; an impersonated user only needs to be able to *read* a saved native Card that references the
+ ;; persisted Model to hit this code path
+ (mt/with-temp [:model/Card card {:dataset_query
+ (lib/native-query (mt/metadata-provider)
+ (format "SELECT count(*) FROM {{#%d}} AS m"
+ (:id model)))}]
+ (let [run-card #(mt/user-http-request % :post 202 (format "card/%d/query" (:id card)))
+ impersonated-result (run-card :rasta)
+ ;; Make sure we run admin query second to reset the DB role on the connection!
+ admin-result (run-card :crowberto)]
+ (testing "Impersonated user (rasta) does not hit the model cache"
+ (is (= 200 (-> impersonated-result mt/rows ffirst)))
+ (is (not (str/includes? (-> impersonated-result :data :native_form :query)
+ (:table_name persisted-info)))
+ "Erroneously used the persisted model cache"))
+ (testing "Query from admin hits the model cache"
+ (is (str/includes? (-> admin-result :data :native_form :query)
+ (:table_name persisted-info))
+ "Did not use the persisted model cache"))))))
(finally
(doseq [statement ["REVOKE ALL PRIVILEGES ON TABLE \"products\" FROM \"impersonation_role\";"
"DROP ROLE IF EXISTS \"impersonation_role\";"]]
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 9275ce39a02..1c57363f5b3 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -1363,6 +1363,58 @@
(:table_name persisted-info)))
"Erroneously used the persisted model cache")))))))))))
+(deftest persistence-disabled-for-native-card-template-tag-when-sandboxed-test
+ (testing "A native query referencing a persisted Model via a {{#N}} template tag must not use the persisted cache for a sandboxed user"
+ (mt/test-drivers (mt/normal-drivers-with-feature :persist-models)
+ (mt/dataset test-data
+ ;; with-gtaps! creates a new copy of the database. So make sure to do that before anything else.
+ (met/with-gtaps! {:gtaps {:products
+ {:remappings {:category
+ ["dimension"
+ [:field (mt/id :products :category)
+ nil]]}}}}
+ (mt/with-persistence-enabled! [persist-models!]
+ ;; with temp ok here since `persist-models!` doesn't use Metadata Providers
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/Card model {:type :model
+ :dataset_query (mt/mbql-query
+ products
+ {:fields [$id $price]})}]
+ ;; persist model (as admin, so sandboxing is not applied to the persisted query)
+ (mt/with-test-user :crowberto
+ (persist-models!))
+ (let [persisted-info (t2/select-one :model/PersistedInfo
+ :database_id (mt/id)
+ :card_id (:id model))]
+ (is (= "persisted" (:state persisted-info))
+ "Model failed to persist")
+ ;; a sandboxed user has no native-query permissions, but only needs to be able to *read* a saved
+ ;; native Card that references the persisted Model to hit this code path.
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/Card card {:dataset_query
+ (lib/native-query (mt/metadata-provider)
+ (format "SELECT count(*) FROM {{#%d}} AS m"
+ (:id model)))}]
+ (let [run-card #(mt/user-http-request % :post 202 (format "card/%d/query" (:id card)))
+ regular-result (run-card :crowberto)
+ sandboxed-result (met/with-user-attributes! :rasta {"category" "Gizmo"}
+ (run-card :rasta))]
+ (testing "Unsandboxed"
+ (testing "Sees full result set"
+ (is (= 200 (-> regular-result mt/rows ffirst))))
+ (testing "Uses the cache table"
+ (is (str/includes? (-> regular-result :data :native_form :query)
+ (:table_name persisted-info))
+ "Did not use the persisted model cache")))
+ (testing "Sandboxed"
+ (testing "sees partial result"
+ (is (= 51 (-> sandboxed-result mt/rows ffirst))
+ "Sandboxed user got whole results instead of filtered"))
+ (testing "Does not use the cache table"
+ (is (not (str/includes? (-> sandboxed-result :data :native_form :query)
+ (:table_name persisted-info)))
+ "Erroneously used the persisted model cache")))))))))))))
+
(deftest model-metadata-overrides-preserved-for-sandboxed-users-test
(testing (str "Column metadata overrides on a Model (custom display_name, semantic_type set in the Edit Metadata "
"screen) should apply to queries sourced from that Model regardless of whether the user has a "
diff --git a/src/metabase/permissions/core.clj b/src/metabase/permissions/core.clj
index e2b891270c2..df729a10217 100644
--- a/src/metabase/permissions/core.clj
+++ b/src/metabase/permissions/core.clj
@@ -168,7 +168,8 @@
[metabase.permissions.data-access-token
data-access-token
data-access-compatible?
- data-access-token-transform])
+ data-access-token-transform
+ routing-token-for-db])
;;; import these vars with different names to make their purpose more obvious.
(p/import-def metabase.permissions.models.permissions-group/all-users all-users-group)
diff --git a/src/metabase/query_processor/middleware/persistence.clj b/src/metabase/query_processor/middleware/persistence.clj
index a02e4059474..9d19e98ef7a 100644
--- a/src/metabase/query_processor/middleware/persistence.clj
+++ b/src/metabase/query_processor/middleware/persistence.clj
@@ -5,17 +5,16 @@
[metabase.util.match :as match]))
(defn substitute-persisted-query
- "Removes persisted information if user is sandboxed or uses connection impersonation. `:persisted-info/native` is set
- in [[metabase.query-processor.middleware.fetch-source-query]].
-
- Sandboxing is detected by the presence of a :query-permissions/sandboxed-table key anywhere in the provided query
+ "Removes persisted information if user is sandboxed, uses connection impersonation, or is routed to a Destination
+ Database. `:persisted-info/native` is set in [[metabase.query-processor.middleware.fetch-source-query]].
It may be be possible to use the persistence cache with sandboxing and/or impersonation at a later date with further
work, but for now we skip the cache in these cases."
[query]
(if (and api/*current-user-id*
(or (match/match-one query {:query-permissions/sandboxed-table &truthy} true)
- (perms/impersonation-enforced-for-db? (:database query))))
+ (perms/impersonation-enforced-for-db? (:database query))
+ (:destination-database/id query)))
(match/replace query
{:persisted-info/native &truthy}
;; Signal to the SQL QP's independent persisted-cache lookup that it should not use the cache for this
diff --git a/src/metabase/query_processor/parameters/values.clj b/src/metabase/query_processor/parameters/values.clj
index d6f5f133b9b..844da775f04 100644
--- a/src/metabase/query_processor/parameters/values.clj
+++ b/src/metabase/query_processor/parameters/values.clj
@@ -11,6 +11,7 @@
(:refer-clojure :exclude [every? some mapv not-empty get-in])
(:require
[clojure.string :as str]
+ [metabase.api.common :as api]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.parameters.parse.types :as params.types]
@@ -20,6 +21,7 @@
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.parameter :as lib.schema.parameter]
[metabase.lib.schema.template-tag :as lib.schema.template-tag]
+ [metabase.permissions.core :as perms]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.core :as qp]
[metabase.query-processor.error-type :as qp.error-type]
@@ -219,6 +221,19 @@
value (field-filter-value tag params)]
(lib/parsed-field-filter-param field value alias)))
+(defn- persisted-cache-allowed-for-current-user?
+ "Whether the current user may read directly from a persisted Model cache table. The cache holds the Model's full,
+ unfiltered results as seen through the Database's own connection, so it must never be substituted for a user whose
+ queries would otherwise be rewritten by sandboxing or connection impersonation, or routed to a Destination Database.
+ Substituting the cache here skips [[qp.compile/compile]] (and thus preprocessing) for the referenced Model entirely,
+ so we cannot rely on the [[metabase.query-processor.middleware.persistence/substitute-persisted-query]] middleware
+ to disable it for us."
+ [database-id]
+ (not (and api/*current-user-id*
+ (or (perms/sandboxed-user-for-db? database-id)
+ (perms/impersonation-enforced-for-db? database-id)
+ (perms/routing-token-for-db database-id)))))
+
(mu/defmethod parse-tag :card :- ::params.types/referenced-card-query
[metadata-providerable :- ::lib.schema.metadata/metadata-providerable
{:keys [card-id], :as tag} :- ::lib.schema.template-tag/template-tag
@@ -235,10 +250,10 @@
(try
(log/tracef "Compiling referenced query for Card %d" card-id)
(let [query (assoc query :info {:card-id card-id})
- {:keys [query params]} (or (when (qp.persistence/can-substitute? card persisted-info)
- {:query (qp.persistence/persisted-info-native-query
- (u/the-id (lib.metadata/database metadata-providerable))
- persisted-info)})
+ database-id (u/the-id (lib.metadata/database metadata-providerable))
+ {:keys [query params]} (or (when (and (qp.persistence/can-substitute? card persisted-info)
+ (persisted-cache-allowed-for-current-user? database-id))
+ {:query (qp.persistence/persisted-info-native-query database-id persisted-info)})
(qp.compile/compile (qp/disable-max-results query)))]
(lib/parsed-referenced-card-query-param card-id query params))
(catch ExceptionInfo e