The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Tighten FieldValues caching for restricted users (#83483)
Andrei Simionescubd4edf4b7f7b
diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/models.clj b/enterprise/backend/src/metabase_enterprise/database_routing/models.cljindex 33e59adff1e..d0eff929fdb 100644--- a/enterprise/backend/src/metabase_enterprise/database_routing/models.clj+++ b/enterprise/backend/src/metabase_enterprise/database_routing/models.clj@@ -3,7 +3,7 @@[metabase-enterprise.database-routing.common :refer [router-db-or-id->destination-db-id]][metabase-enterprise.database-routing.db :as database-routing.db][metabase.models.interface :as mi]- [metabase.premium-features.core :refer [defenterprise]]+ [metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.util :as u][metabase.warehouse-schema.models.field :as field][methodical.core :as methodical]@@ -25,11 +25,12 @@{:default nil}))(defenterprise hash-input-for-database-routing- "Enterprise version. Returns a hash input that will be used for fields subject to database routing."- :feature :database-routing+ "Enterprise version. Returns a hash input that will be used for fields subject to database routing.+ The destination is nil while the `:database-routing` feature is unavailable."+ :feature :none[field](when-let [destination-db-id (some->> field u/the-id field/field-id->database-id router-db-or-id->destination-db-id)]- {:destination-db-id destination-db-id}))+ {:destination-db-id (when (premium-features/has-feature? :database-routing) destination-db-id)}))(defenterprise delete-associated-database-router!"Deletes the Database Router associated with this router database."diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/driver.clj b/enterprise/backend/src/metabase_enterprise/impersonation/driver.cljindex af607a3d992..b862be4c36a 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/driver.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/driver.clj@@ -107,13 +107,16 @@role)))))))(defenterprise hash-input-for-impersonation- "Returns a hash-key for FieldValues if the current user uses impersonation for the database."- :feature :advanced-permissions+ "Returns a hash-key for FieldValues if the current user uses impersonation for the database.+ The role is nil while the `:advanced-permissions` feature is unavailable."+ :feature :none[field];; Include the role in the hash key, so that we can cache the results of the query for each role.(let [db-id (field/field-id->database-id (u/the-id field))]- (when-let [role (and api/*current-user-id* (connection-impersonation-role db-id))]- {:impersonation-role role})))+ (when-let [role (and api/*current-user-id*+ (impersonation.db/impersonation-exists-for-database? db-id)+ (connection-impersonation-role db-id))]+ {:impersonation-role (when (premium-features/has-feature? :advanced-permissions) role)})))(defenterprise impersonation-token-for-db"Connection-impersonation fingerprint for the current user on `db-id` (the resolved databasediff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/params/field_values.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/params/field_values.cljindex b76f6a7e05d..1f85a19d5a4 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/params/field_values.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/params/field_values.clj@@ -4,7 +4,7 @@[metabase-enterprise.sandbox.db :as sandbox.db][metabase-enterprise.sandbox.query-processor.middleware.sandboxing :as sandboxing][metabase.api.common :as api]- [metabase.premium-features.core :refer [defenterprise]]+ [metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.warehouse-schema.models.field :as field][toucan2.core :as t2]))@@ -75,11 +75,13 @@[k (get login-attributes k)])))])))(defenterprise hash-input-for-sandbox- "Returns a hash-input for FieldValues if the field is sandboxed."- :feature :sandboxes+ "Returns a hash-input for FieldValues if the field is sandboxed.+ The attributes are nil when they can't be resolved, and while the `:sandboxes` feature is unavailable."+ :feature :none[field](when (field-is-sandboxed? field)- {:sandbox-attributes (field->sandbox-attributes-for-current-user field)}))+ {:sandbox-attributes (when (premium-features/has-feature? :sandboxes)+ (field->sandbox-attributes-for-current-user field))}))(defenterprise sandbox-token-for-table"Sandbox fingerprint for the current user on `table-id` (GTAP card, its version, and resolveddiff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.cljindex 96061a2042d..c75e7caef4d 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/models/field_values_test.clj@@ -56,3 +56,11 @@(is (= 2 (t2/count :model/FieldValues :field_id (u/the-id field) :type :advanced)))))))))(finally(t2/delete! :model/FieldValues :field_id (u/the-id field) :type :advanced))))))++(deftest hash-input-without-advanced-permissions-test+ (testing "an impersonated user gets no FieldValues cache key while advanced permissions are unavailable"+ (impersonation.util-test/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]+ :attributes {"impersonation_attr" "impersonation_role"}}+ (mt/with-premium-features #{}+ (is (nil? (params.field-values/hash-input-for-field-values+ (t2/select-one :model/Field :id (mt/id :categories :id)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.cljindex 47ba58f2a6d..5b18116159b 100644--- a/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/database_routing/e2e_test.clj@@ -329,7 +329,11 @@response)))(let [response (mt/user-http-request :lucky :get 200 (str "field/" field-id "/values"))](is (= {:values [["destination-2"]] :field_id field-id :has_more_values false}- response))))))))))))+ response)))+ (testing "a routed user gets no values while database routing is unavailable"+ (mt/with-premium-features #{}+ (is (= [] (:values (mt/user-http-request :rasta :get 200+ (str "field/" field-id "/values"))))))))))))))))(defmulti router-dataset-name"Name for router dataset"diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/field_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/field_test.cljindex 656f1bf0133..2e8eba8470a 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/field_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/field_test.clj@@ -119,6 +119,22 @@(is (= [[1 "$"] [3 "$$$"]](:values (mt/user-http-request :rasta :get 200 (format "field/%d/values" (mt/id :venues :price)))))))))))+(deftest field-values-without-sandboxes-feature-test+ (testing (str "GET /api/field/:id/values gives a sandboxed user no values "+ "while sandboxing is unavailable")+ (mt/with-temp-copy-of-db+ (field-values/get-or-create-full-field-values! (t2/select-one :model/Field :id (mt/id :venues :price)))+ (met/with-gtaps! {:gtaps {:venues+ {:remappings {:cat [:variable [:field (mt/id :venues :category_id) nil]]}}}+ :attributes {:cat 4}}+ (letfn [(fetch-values []+ (:values (mt/user-http-request :rasta :get 200 (format "field/%d/values" (mt/id :venues :price)))))]+ (is (= [[1] [3]] (fetch-values)))+ (mt/with-premium-features #{}+ (is (= [] (fetch-values))))+ (testing "and their own values once it's back"+ (is (= [[1] [3]] (fetch-values)))))))))+(deftest search-test(testing "GET /api/field/:id/search/:search-id"(met/with-gtaps! {:gtaps {:venuesdiff --git a/enterprise/backend/test/metabase_enterprise/sandbox/models/params/field_values_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/models/params/field_values_test.cljindex eec8add1a89..f6b8eccc4db 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/models/params/field_values_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/models/params/field_values_test.clj@@ -8,6 +8,8 @@[metabase.parameters.field-values :as params.field-values][metabase.request.core :as request][metabase.test :as mt]+ [metabase.util :as u]+ [metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -254,6 +256,55 @@(is (not= (hash-input user-id-1 {"state" "CA"})(hash-input user-id-2 {"state" "TX"})))))))))))+(deftest unresolved-sandbox-attributes-test+ (testing "a sandboxed user whose sandbox can't be resolved gets no values"+ (met/with-gtaps! {:gtaps {:venues {:remappings {:cat [:variable [:field (mt/id :venues :category_id) nil]]}}}}+ (let [field (t2/select-one :model/Field :id (mt/id :venues :price))]+ (is (ee-params.field-values/field-is-sandboxed? field))+ ;; the request's permissions cache keeps the sandbox, while the attribute lookup reads the app DB+ (t2/delete! :model/Sandbox :group_id (u/the-id &group))+ (t2/insert! :model/FieldValues {:field_id (u/the-id field)+ :type :advanced+ :hash_key (str (hash {:field-id (u/the-id field), :sandbox-attributes nil}))+ :values [1 2 3 4]})+ (is (nil? (params.field-values/get-or-create-field-values! field)))+ (is (= {(u/the-id field) {}}+ (params.field-values/field-id->field-values-for-current-user [(u/the-id field)])))))))++(deftest batched-field-values-without-sandboxes-feature-test+ (testing "the batched fetches give a sandboxed user no values while sandboxing is unavailable"+ (mt/with-temp-copy-of-db+ (field-values/get-or-create-full-field-values! (t2/select-one :model/Field :id (mt/id :venues :price)))+ (met/with-gtaps! {:gtaps {:venues {:remappings {:cat [:variable [:field (mt/id :venues :category_id) nil]]}}}+ :attributes {:cat 4}}+ (mt/with-premium-features #{}+ (is (= {(mt/id :venues :price) {}}+ (params.field-values/field-id->field-values-for-current-user [(mt/id :venues :price)])))+ (is (= {(mt/id :venues :price) {}}+ (params.field-values/get-or-create-field-values-by-field-id!+ [(t2/select-one :model/Field :id (mt/id :venues :price))]))))))))++(deftest batched-field-values-without-table-id-test+ (testing "a field with no :table_id throws"+ (is (thrown? AssertionError+ (params.field-values/get-or-create-field-values-by-field-id!+ [(dissoc (t2/select-one :model/Field :id (mt/id :venues :price)) :table_id)])))))++(deftest batched-field-values-query-count-test+ (testing "the batched fetch resolves a user's restrictions once per table, not once per field"+ (doseq [features [#{} #{:sandboxes :advanced-permissions :database-routing}]]+ (mt/with-premium-features features+ (mt/with-test-user :rasta+ (let [field-ids (t2/select-pks-vec :model/Field :table_id (mt/id :venues))+ query-count (fn [ids]+ ;; warms the memoized field and table lookups+ (params.field-values/field-id->field-values-for-current-user ids)+ (t2/with-call-count [call-count]+ (params.field-values/field-id->field-values-for-current-user ids)+ (call-count)))]+ (is (= (query-count (take 1 field-ids))+ (query-count field-ids)))))))))+(deftest cross-table-remapping-get-or-create-test(testing "two tenants sandboxed via a joined-table remapping must not share one FieldValues row (SEC-874)"(met/with-gtaps-for-all-users!diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.cljindex 07369a8cd9b..1910031e4fb 100644--- a/src/metabase/metabot/db.clj+++ b/src/metabase/metabot/db.clj@@ -663,6 +663,11 @@[field-id :- ::lib.schema.id/field](t2/select-one :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query)]}))+(mu/defn fields+ "The Fields with `field-ids`."+ [field-ids :- [:set ::lib.schema.id/field]]+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))+(mu/defn field-fingerprint"The fingerprint of the Field with `field-id`."[field-id :- ::lib.schema.id/field]diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.cljindex b25f3fa44e4..060dadeda45 100644--- a/src/metabase/metabot/tools/entity_details.clj+++ b/src/metabase/metabot/tools/entity_details.clj@@ -129,20 +129,15 @@{:output "dashboard not found"}))(defn- get-field-values- "Get field values for a field, creating them if they don't exist.- Uses the user-aware API that respects sandboxing/impersonation."[id->values id]- (if-some [field-values (get id->values id)]- (:values field-values)- (let [field (metabot.db/field id)]- (when (and field- (params.field-values/current-user-can-fetch-field-values? field))- (:values (params.field-values/get-or-create-field-values! field))))))+ (:values (get id->values id)))(defn- add-field-values[cols](if-let [field-ids (seq (keep :id cols))]- (let [id->values (params.field-values/field-id->field-values-for-current-user field-ids)]+ (let [id->values (params.field-values/get-or-create-field-values-by-field-id!+ (filter params.field-values/current-user-can-fetch-field-values?+ (t2/hydrate (metabot.db/fields (set field-ids)) :table)))](map #(m/assoc-some % :field-values (some->> % :id (get-field-values id->values))) cols))cols))diff --git a/src/metabase/parameters/field_values.clj b/src/metabase/parameters/field_values.cljindex e1036c15368..4f1a97d06cf 100644--- a/src/metabase/parameters/field_values.clj+++ b/src/metabase/parameters/field_values.clj@@ -10,7 +10,7 @@[metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))-(declare get-or-create-field-values!)+(declare get-or-create-field-values! get-or-create-field-values-with-hash-input!)(defn get-or-create-field-values-for-current-user!*"Fetch cached FieldValues for a `field`, creating them if needed if the Field should have FieldValues."@@ -61,30 +61,39 @@This ensures that any difference in these elements results in a distinct cache key.- Returns a map suitable for hashing into a cache key."+ Returns a map suitable for hashing into a cache key, or nil when a restriction applies to the current user but can't+ be keyed, because its premium feature is unavailable or it can't be resolved. No cached FieldValues may be served to+ or created for such a user."[field & [constraints]]- (merge- {:field-id (u/the-id field)}- (field-values/hash-input-for-sandbox field)- (field-values/hash-input-for-impersonation field)- (field-values/hash-input-for-linked-filters field constraints)- (field-values/hash-input-for-database-routing field)))--(defn- requires-advanced-field-value?- "Given a field, returns falsey if this field should use the normal batched implementation to get field values."- [field]- (not= (hash-input-for-field-values field)- {:field-id (u/the-id field)}))+ (let [hash-input (merge+ {:field-id (u/the-id field)}+ (field-values/hash-input-for-sandbox field)+ (field-values/hash-input-for-impersonation field)+ (field-values/hash-input-for-linked-filters field constraints)+ (field-values/hash-input-for-database-routing field))]+ (when (every? some? (vals hash-input))+ hash-input)))++(defn- hash-inputs-by-table+ "Returns a function from each of `fields` to its [[hash-input-for-field-values]], computed once per table.+ The hash inputs of one table's fields differ only in `:field-id`."+ [fields]+ {:pre [(every? :table_id fields)]}+ (let [table-id->hash-input (update-vals (group-by :table_id fields) (comp hash-input-for-field-values first))]+ (fn [{:keys [id table_id]}]+ (some-> (table-id->hash-input table_id) (assoc :field-id id)))))(defn field-id->field-values-for-current-user"Fetch *existing* FieldValues for a sequence of `field-ids` for the current User. Values are returned as a map of{field-id FieldValues-instance}- Returns `nil` if `field-ids` is empty of no matching FieldValues exist."+ Returns `nil` if `field-ids` is empty or no matching FieldValues exist. A field whose restrictions can't be keyed+ for the current User maps to an empty map."[field-ids](let [fields (when (seq field-ids)(t2/hydrate (parameters.db/fields (set field-ids)) :table))+ hash-input (hash-inputs-by-table fields){normal-fields false- advanced-fields true} (group-by requires-advanced-field-value? fields)]+ advanced-fields true} (group-by #(not= (hash-input %) {:field-id (:id %)}) fields)](merge;; use the normal OSS batched implementation for any Fields that aren't subject to sandboxing.(when (seq normal-fields)@@ -92,7 +101,7 @@(map u/the-id normal-fields)));; for sandboxed (or otherwise advanced) fields, fetch the sandboxed values individually.(into {} (for [{field-id :id, :as field} advanced-fields]- [field-id (select-keys (get-or-create-field-values! field)+ [field-id (select-keys (get-or-create-field-values-with-hash-input! field nil (hash-input field))[:values :human_readable_values :field_id])])))));;; +----------------------------------------------------------------------------------------------------------------+@@ -137,31 +146,56 @@:human_readable_values human-readable-values:values values}))+(defn- get-or-create-advanced-field-values!+ [field constraints hash-input]+ (let [hash-key (str (hash hash-input))+ ;; look first on this thread: a hit is one SELECT, and handing that to a background+ ;; thread costs more than it saves. Only a miss is worth detaching, because only a+ ;; miss scans the warehouse.+ fv (or (parameters.db/advanced-field-values (:id field) hash-key)+ (field-values/detached-fetch!+ [:advanced (:id field) hash-key]+ (fn []+ (parameters.db/find-or-insert-advanced-field-values!+ (:id field) hash-key #(prepare-advanced-field-values field hash-key constraints)))))]+ ;; If it's expired, delete then try to re-create it+ (if (some-> fv field-values/advanced-field-values-expired?)+ (do+ ;; It's possible another process has already recalculated this, but spurious recalculations are OK.+ (parameters.db/delete-field-values! (:id fv))+ (recur field constraints hash-input))+ fv)))++(defn- get-or-create-field-values-with-hash-input!+ [field constraints hash-input]+ (when hash-input+ (if (not= hash-input {:field-id (u/the-id field)})+ (get-or-create-advanced-field-values! field constraints hash-input)+ (field-values/get-or-create-full-field-values! field))))+(defn get-or-create-field-values!- "Gets or creates field values."+ "Gets or creates field values. Returns nil when [[hash-input-for-field-values]] can't key the current user's access."([field] (get-or-create-field-values! field nil))([field constraints]- (let [hash-input (hash-input-for-field-values field constraints)- advanced-field-value? (not= hash-input {:field-id (u/the-id field)})]- (if advanced-field-value?- (let [hash-key (str (hash hash-input))- ;; look first on this thread: a hit is one SELECT, and handing that to a background- ;; thread costs more than it saves. Only a miss is worth detaching, because only a- ;; miss scans the warehouse.- fv (or (parameters.db/advanced-field-values (:id field) hash-key)- (field-values/detached-fetch!- [:advanced (:id field) hash-key]- (fn []- (parameters.db/find-or-insert-advanced-field-values!- (:id field) hash-key #(prepare-advanced-field-values field hash-key constraints)))))]- ;; If it's expired, delete then try to re-create it- (if (some-> fv field-values/advanced-field-values-expired?)- (do- ;; It's possible another process has already recalculated this, but spurious recalculations are OK.- (parameters.db/delete-field-values! (:id fv))- (recur field constraints))- fv))- (field-values/get-or-create-full-field-values! field)))))+ (get-or-create-field-values-with-hash-input! field constraints (hash-input-for-field-values field constraints))))++(defn get-or-create-field-values-by-field-id!+ "Calls [[get-or-create-field-values!]] for each of `fields`, resolving the current user's restrictions once per table.+ Active FieldValues of unrestricted fields are read in one query instead, without updating their `last_used_at`.+ Returns a map of field id to FieldValues, with an empty map for a field that gets none. Throws if a field has no+ `:table_id`."+ [fields]+ (let [hash-input (hash-inputs-by-table fields)+ cached (field-values/batched-get-latest-full-field-values+ (for [field fields+ :when (and (= (hash-input field) {:field-id (:id field)})+ (field-values/field-should-have-field-values? field))]+ (:id field)))]+ (into {} (for [{:keys [id] :as field} fields+ :let [fv (get cached id)]]+ [id (or (when (and fv (not (field-values/inactive? fv))) fv)+ (get-or-create-field-values-with-hash-input! field nil (hash-input field))+ {})]))));;; +----------------------------------------------------------------------------------------------------------------+;;; | Public functions |diff --git a/test/metabase/metabot/tools/entity_details_test.clj b/test/metabase/metabot/tools/entity_details_test.cljindex e3d52ab934c..11a10ca04cb 100644--- a/test/metabase/metabot/tools/entity_details_test.clj+++ b/test/metabase/metabot/tools/entity_details_test.clj@@ -13,6 +13,7 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))(use-fixtures :once (fixtures/initialize :db :test-users))@@ -406,15 +407,26 @@raw-values ["African" "American"]](testing "cache hits return raw values"(is (= raw-values- (#'entity-details/get-field-values {field-id {:values raw-values}} field-id))))- (testing "cache misses return the same raw-value shape"- (mt/with-dynamic-fn-redefs- [params.field-values/current-user-can-fetch-field-values? (constantly true)- params.field-values/get-or-create-field-values! (constantly {:values raw-values})- params.field-values/get-or-create-field-values-for-current-user!- (constantly {:values (mapv vector raw-values)})]- (is (= raw-values- (#'entity-details/get-field-values {} field-id)))))))+ (#'entity-details/get-field-values {field-id {:values raw-values}} field-id))))))++(deftest table-details-read-cached-field-values-in-one-query-test+ (testing "table details read the cached values of unrestricted columns in one query, not one per column"+ (mt/with-temp-copy-of-db+ (mt/with-current-user (mt/user->id :rasta)+ (let [field-ids (t2/select-pks-set :model/Field :table_id (mt/id :venues))+ details #(:structured-output+ (entity-details/get-table-details {:entity-type :table, :entity-id (mt/id :venues)}))+ query-count (fn [field-ids-with-values]+ (mt/with-dynamic-fn-redefs [field-values/field-should-have-field-values?+ #(contains? field-ids-with-values (:id %))]+ ;; creates the FieldValues and warms the memoized field and table lookups+ (is (= field-ids-with-values+ (set (keep #(when (:field_values %) (:field_id %)) (:fields (details))))))+ (t2/with-call-count [call-count]+ (details)+ (call-count))))]+ (is (= (query-count #{(first field-ids)})+ (query-count field-ids))))))));;; ============================================================;;; Base-table surfacing on get-metric-details (regression)@@ -459,7 +471,7 @@(mt/with-no-data-perms-for-all-users!(mt/with-current-user (mt/user->id :rasta)(mt/with-dynamic-fn-redefs- [params.field-values/field-id->field-values-for-current-user+ [params.field-values/get-or-create-field-values-by-field-id!(fn [_](throw (ex-info "field values must not be fetched" {})))](let [output (:structured-output