The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Drop fingerprint from sandboxed responses (#83366)
lbrdnk384e4a357361
diff --git a/.clj-kondo/ratchets-test.edn b/.clj-kondo/ratchets-test.ednindex 830d17821f2..800b3d02963 100644--- a/.clj-kondo/ratchets-test.edn+++ b/.clj-kondo/ratchets-test.edn@@ -46,6 +46,6 @@:lib.equality/find-column-indexes-for-refs 2:search.entity-retrieval.core/search-unfiltered 1:search.entity-retrieval.mirror/search-unfiltered 1- :test/with-temp 29}+ :test/with-temp 30}:comment-exempt #{:deprecated-namespace:discouraged-var}}diff --git a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.jsindex bb929cf0dcd..2b833086d16 100644--- a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js+++ b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js@@ -1067,7 +1067,16 @@ describe("admin > permissions > sandboxes (tested via the API)", () => {database: SAMPLE_DB_ID,},display: "pivot",- visualization_settings: {},+ // The split the query builder persists on save. A sandboxed run's cols+ // carry no fingerprint (BOT-2115), so without a stored split the+ // default layout for the sandboxed user differs from the admin's.+ visualization_settings: {+ "pivot_table.column_split": {+ rows: ["SOURCE"],+ columns: ["CATEGORY"],+ values: ["sum"],+ },+ },}).then(({ body: { id: QUESTION_ID } }) => {cy.signOut();cy.signInAsSandboxedUser();diff --git a/enterprise/backend/test/metabase_enterprise/database_routing/result_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/database_routing/result_metadata_test.cljnew file mode 100644index 00000000000..ab0159683c4--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/database_routing/result_metadata_test.clj@@ -0,0 +1,94 @@+(ns metabase-enterprise.database-routing.result-metadata-test+ "Tests that query results routed to a destination database carry no column fingerprint, since the stored+ fingerprint was synced from the router database and describes rows the routed user never queries (BOT-2115)."+ (:require+ [clojure.core.async :as a]+ [clojure.test :refer :all]+ [metabase-enterprise.database-routing.e2e-test :as e2e]+ [metabase-enterprise.test :as met]+ [metabase.driver.settings :as driver.settings]+ [metabase.query-processor :as qp]+ [metabase.query-processor.middleware.cache-test :as cache-test]+ [metabase.test :as mt]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(defn- col-fingerprints+ "`{col-name fingerprint}` for every col in a query `result`."+ [result]+ (into {} (map (juxt :name :fingerprint)) (mt/cols result)))++(defn- results-metadata-fingerprints+ "`{col-name fingerprint}` for every column of the `:results_metadata` of a query `result`."+ [result]+ (into {} (map (juxt :name :fingerprint)) (-> result :data :results_metadata :columns)))++(def ^:private router-fingerprint+ "Stands in for the fingerprint sync computes over the router database's rows."+ {:global {:distinct-count 3, :nil% 0.0}+ :type {:type/Text {:percent-json 0.0, :percent-url 0.0, :percent-email 0.0, :percent-state 0.0+ :average-length 6.0}}})++(defn- do-with-routed-table!+ "Stand up a router with one destination, routing `:rasta` to the destination and `:crowberto` to the router itself.+ The router table's only field gets a stored fingerprint. Calls `f` with the router DB and the table id."+ [f]+ (mt/with-premium-features #{:database-routing}+ (binding [driver.settings/*allow-testing-h2-connections* true]+ (met/with-user-attributes! :crowberto {"db_name" "__METABASE_ROUTER__"}+ (met/with-user-attributes! :rasta {"db_name" "destination-db"}+ (e2e/with-routing-setup! [router-db [[destination-db "destination-db"]]]+ (mt/with-temp [:model/DatabaseRouter _ {:database_id (u/the-id router-db)+ :user_attribute "db_name"}]+ (e2e/execute-statement! router-db "INSERT INTO \"my_database_name\" (str) VALUES ('router')")+ (e2e/execute-statement! destination-db "INSERT INTO \"my_database_name\" (str) VALUES ('destination')")+ (let [table-id (t2/select-one-pk :model/Table :db_id (u/the-id router-db))]+ (t2/update! :model/Field :table_id table-id {:fingerprint router-fingerprint})+ (f router-db table-id)))))))))++(defn- table-query [router-db table-id]+ {:database (u/the-id router-db)+ :type :query+ :query {:source-table table-id}})++(deftest routed-results-omit-fingerprints-test+ (testing "BOT-2115: result cols for a user routed to a destination database carry no fingerprint"+ (do-with-routed-table!+ (fn [router-db table-id]+ (let [query (table-query router-db table-id)+ result (mt/user-http-request :rasta :post 202 "dataset" query)]+ (testing "sanity check: the query was routed"+ (is (= [["destination"]] (mt/rows result))))+ (is (every? nil? (vals (col-fingerprints result))))+ (testing "results_metadata carries no fingerprint either: the frontend posts it back as the Card's result_metadata"+ (is (seq (-> result :data :results_metadata :columns)))+ (is (every? nil? (vals (results-metadata-fingerprints result)))))+ (testing "a user routed to the router itself still gets the fingerprint, in cols and results_metadata"+ (let [result (mt/user-http-request :crowberto :post 202 "dataset" query)]+ (is (= [["router"]] (mt/rows result)))+ (is (=? {"STR" router-fingerprint} (col-fingerprints result)))+ (is (=? {"STR" router-fingerprint} (results-metadata-fingerprints result))))))))))++(deftest routed-cached-results-omit-fingerprints-test+ (testing "BOT-2115: a cached result replayed to a routed user carries no fingerprint"+ (do-with-routed-table!+ (fn [router-db table-id]+ (cache-test/with-mock-cache! [save-chan]+ (mt/with-test-user :rasta+ (letfn [(run-query []+ (qp/process-query (assoc (table-query router-db table-id)+ :cache-strategy {:type :ttl+ :multiplier 60+ :avg-execution-ms 10+ :min_duration_ms 0})))]+ (let [result (run-query)]+ (is (nil? (:cached (:cache/details result))))+ (is (= [["destination"]] (mt/rows result)))+ (is (every? nil? (vals (col-fingerprints result)))))+ (testing "cache entry should be saved within 5 seconds"+ (let [[_ chan] (a/alts!! [save-chan (a/timeout 5000)])]+ (is (= save-chan chan))))+ (let [result (run-query)]+ (is (true? (:cached (:cache/details result))))+ (is (= [["destination"]] (mt/rows result)))+ (is (every? nil? (vals (col-fingerprints result))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/impersonation/result_metadata_test.clj b/enterprise/backend/test/metabase_enterprise/impersonation/result_metadata_test.cljnew file mode 100644index 00000000000..91eab793fd4--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/impersonation/result_metadata_test.clj@@ -0,0 +1,76 @@+(ns metabase-enterprise.impersonation.result-metadata-test+ "Tests that query results served under a connection-impersonation policy carry no column fingerprint, since the+ stored fingerprint was computed over rows the impersonated role may not see (BOT-2115)."+ {:clj-kondo/config+ '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query+ {:namespaces [metabase-enterprise.impersonation.result-metadata-test]}}}}}}+ (:require+ [clojure.core.async :as a]+ [clojure.test :refer :all]+ [metabase-enterprise.impersonation.util-test :as impersonation.util-test]+ [metabase.query-processor :as qp]+ [metabase.query-processor.middleware.cache-test :as cache-test]+ [metabase.test :as mt]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(defn- col-fingerprints+ "`{col-name fingerprint}` for every col in a query `result`."+ [result]+ (into {} (map (juxt :name :fingerprint)) (mt/cols result)))++(defn- impersonation-def []+ {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}]+ :attributes {"impersonation_attr" "impersonation_role"}})++(deftest impersonated-results-omit-fingerprints-test+ (mt/with-premium-features #{:advanced-permissions}+ (testing "BOT-2115: result cols for a user under an impersonation policy carry no fingerprint"+ (let [query (mt/mbql-query venues)]+ (impersonation.util-test/with-impersonations! (impersonation-def)+ (let [result (mt/user-http-request :rasta :post 202 "dataset" query)]+ (is (seq (mt/rows result)))+ (is (every? nil? (vals (col-fingerprints result)))))+ (testing "an admin is never impersonated and still gets the fingerprint"+ (is (=? {"PRICE" {:global {:distinct-count 4}}}+ (col-fingerprints (mt/user-http-request :crowberto :post 202 "dataset" query))))))+ (testing "the same user without a policy still gets the fingerprint"+ (is (=? {"PRICE" {:global {:distinct-count 4}}}+ (col-fingerprints (mt/user-http-request :rasta :post 202 "dataset" query)))))))))++(deftest impersonated-card-results-omit-fingerprints-test+ (mt/with-premium-features #{:advanced-permissions}+ (testing "BOT-2115: saved card results for a user under an impersonation policy carry no fingerprint"+ (mt/with-temp [:model/Card card {:dataset_query (mt/mbql-query venues)}]+ (mt/user-http-request :crowberto :post 202 (format "card/%d/query" (u/the-id card)))+ (let [persisted (t2/select-one-fn :result_metadata :model/Card :id (u/the-id card))]+ (is (=? {:name "PRICE", :fingerprint {:global {:distinct-count 4}}} (last persisted)))+ (impersonation.util-test/with-impersonations! (impersonation-def)+ (let [result (mt/user-http-request :rasta :post 202 (format "card/%d/query" (u/the-id card)))]+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))+ (testing "the impersonated run does not persist its result_metadata on the Card (BOT-2115)"+ (is (= persisted (t2/select-one-fn :result_metadata :model/Card :id (u/the-id card))))))))))))++(deftest impersonated-cached-results-omit-fingerprints-test+ (mt/with-premium-features #{:advanced-permissions}+ (testing "BOT-2115: a cached result replayed to a user under an impersonation policy carries no fingerprint"+ (cache-test/with-mock-cache! [save-chan]+ (impersonation.util-test/with-impersonations! (impersonation-def)+ (letfn [(run-query []+ (qp/process-query (assoc (mt/mbql-query venues)+ :cache-strategy {:type :ttl+ :multiplier 60+ :avg-execution-ms 10+ :min_duration_ms 0})))]+ (let [result (run-query)]+ (is (nil? (:cached (:cache/details result))))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result)))))+ (testing "cache entry should be saved within 5 seconds"+ (let [[_ chan] (a/alts!! [save-chan (a/timeout 5000)])]+ (is (= save-chan chan))))+ (let [result (run-query)]+ (is (true? (:cached (:cache/details result))))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result)))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 719740a44ff..3b5f0448a90 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -732,6 +732,8 @@;; should return a field for the original field(update :field_ref (fn [[tag _id opts]][tag id opts]))+ ;; except for the fingerprint, which describes rows the sandbox hides (BOT-2115)+ (assoc :fingerprint (symbol "nil #_\"key is not present.\""))(dissoc :fk_target_field_id:lib/original-display-name:lib/transformation-added-base-type))))]@@ -1212,11 +1214,10 @@:attributes {"user_id" 1, "user_cat" "Widget"}}(qp.store/with-metadata-provider (remap-metadata-provider)(mt/run-mbql-query orders)))]- (testing "Sanity check: merged results metadata should not get normalized incorrectly"- (is (=? {:type {:type/Number {}}}- (-> (get-in mbql-sandbox-results [:data :cols])- (nth 3)- :fingerprint))))+ (testing "Sanity check: merged results metadata carries no fingerprint (BOT-2115)"+ (is (nil? (-> (get-in mbql-sandbox-results [:data :cols])+ (nth 3)+ :fingerprint))))(doseq [orders-gtap-card-has-metadata? [true false]products-gtap-card-has-metadata? [true false]](testing (format "\nwith GTAP metadata for Orders? %s Products? %s"@@ -2413,3 +2414,159 @@:let [source-col (col-by-name (:remapped_from col))]](is (= (:name col)(:remapped_to source-col)))))))))++(defn- col-fingerprints+ "`{col-name fingerprint}` for every col in a query `result`."+ [result]+ (into {} (map (juxt :name :fingerprint)) (mt/cols result)))++(defn- results-metadata-fingerprints+ "`{col-name fingerprint}` for every column of the `:results_metadata` of a query `result`. The insights step copies+ the cols' fingerprint when they carry one; a row-restricted run returns none here either, since the frontend posts+ `:results_metadata` back as the Card's `result_metadata` on save."+ [result]+ (into {} (map (juxt :name :fingerprint)) (-> result :data :results_metadata :columns)))++(deftest sandboxed-results-omit-fingerprints-test+ (testing "BOT-2115: result cols for a sandboxed user carry no fingerprint, since it describes rows the sandbox hides"+ (testing "control: the same non-admin without a sandbox gets the fingerprint, in cols and results_metadata"+ (let [result (mt/user-http-request :rasta :post 202 "dataset" (mt/mbql-query venues))]+ (is (=? {"PRICE" {:global {:distinct-count 4}}+ "LATITUDE" {:type {:type/Number {:min number?}}}}+ (col-fingerprints result)))+ (is (=? {"PRICE" {:global {:distinct-count 4}}}+ (results-metadata-fingerprints result)))))+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ ;; built inside the sandbox, which runs against a temp copy of the test DB+ (let [query (mt/mbql-query venues)+ result (mt/user-http-request :rasta :post 202 "dataset" query)]+ (testing "sanity check: the sandbox is in force"+ (is (true? (-> result :data :is_sandboxed)))+ (is (= #{1} (into #{} (map #(nth % 5)) (mt/rows result)))))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))+ (testing "results_metadata carries no fingerprint either: the frontend posts it back as the Card's result_metadata"+ (is (seq (-> result :data :results_metadata :columns)))+ (is (every? nil? (vals (results-metadata-fingerprints result)))))+ (testing "an unsandboxed admin still gets the fingerprint, in cols and results_metadata"+ (let [result (mt/user-http-request :crowberto :post 202 "dataset" query)]+ (is (=? {"PRICE" {:global {:distinct-count 4}}+ "LATITUDE" {:type {:type/Number {:min number?}}}}+ (col-fingerprints result)))+ (is (=? {"PRICE" {:global {:distinct-count 4}}}+ (results-metadata-fingerprints result)))))))))++(deftest sandboxed-card-and-dashcard-results-omit-fingerprints-test+ (testing "BOT-2115: saved card and dashcard results for a sandboxed user carry no fingerprint"+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ ;; the HTTP endpoints need real Card, Dashboard and DashboardCard rows; a mock metadata provider cannot stand in+ #_{:clj-kondo/ignore [:discouraged-var]}+ (mt/with-temp [:model/Collection {collection-id :id} {}+ :model/Card {card-id :id} {:dataset_query (mt/mbql-query venues)+ :collection_id collection-id}+ :model/Dashboard {dashboard-id :id} {:collection_id collection-id}+ :model/DashboardCard {dashcard-id :id} {:dashboard_id dashboard-id+ :card_id card-id}]+ (perms/grant-collection-read-permissions! &group collection-id)+ (testing "an admin run persists the card's result_metadata, fingerprints over the whole table included"+ (mt/user-http-request :crowberto :post 202 (format "card/%d/query" card-id))+ (is (=? [{:name "ID"} {:name "NAME"} {:name "CATEGORY_ID"} {:name "LATITUDE"} {:name "LONGITUDE"}+ {:name "PRICE", :fingerprint {:global {:distinct-count 4}}}]+ (t2/select-one-fn :result_metadata :model/Card :id card-id))))+ (let [persisted (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (doseq [[path url] {"card" (format "card/%d/query" card-id)+ "dashcard" (format "dashboard/%d/dashcard/%d/card/%d/query"+ dashboard-id dashcard-id card-id)}]+ (testing path+ (let [result (mt/user-http-request :rasta :post 202 url)]+ (is (true? (-> result :data :is_sandboxed)))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))+ (testing "the sandboxed run's results_metadata carries no fingerprint either"+ (is (=? {:name "PRICE", :fingerprint (symbol "nil #_\"key is not present.\"")}+ (last (-> result :data :results_metadata :columns)))))+ (testing "and is not persisted on the Card, where every reader would see it (BOT-2115)"+ (is (= persisted (t2/select-one-fn :result_metadata :model/Card :id card-id))))))))))))++(deftest sandboxed-join-results-omit-fingerprints-test+ (testing "BOT-2115: when a sandboxed table is joined to an unrestricted one, no result col carries a fingerprint"+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ (data-perms/set-table-permission! &group (mt/id :categories) :perms/create-queries :query-builder)+ (data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)+ (let [result (mt/user-http-request :rasta :post 202 "dataset"+ (mt/mbql-query venues+ {:joins [{:source-table $$categories+ :fields :all+ :condition [:= $category_id &c.categories.id]+ :alias "c"}]}))]+ (is (true? (-> result :data :is_sandboxed)))+ (testing "sanity check: the joined table's cols are in the result"+ (is (contains? (set (map :table_id (mt/cols result))) (mt/id :categories))))+ (is (every? nil? (vals (col-fingerprints result))))))))++(deftest native-sandbox-results-omit-fingerprints-test+ (testing "BOT-2115: result cols for a user sandboxed by a native query carry no fingerprint"+ (met/with-gtaps! {:gtaps {:venues {:query (mt/native-query {:query "SELECT * FROM VENUES WHERE PRICE = 1"})}}}+ (let [result (mt/user-http-request :rasta :post 202 "dataset" (mt/mbql-query venues))]+ (is (true? (-> result :data :is_sandboxed)))+ (is (= 22 (count (mt/rows result))))+ (is (every? nil? (vals (col-fingerprints result))))))))++(deftest sandboxed-binned-query-still-works-test+ (testing "BOT-2115: stripping fingerprints from results must not break binning, which needs them in preprocessing"+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ (let [result (mt/user-http-request :rasta :post 202 "dataset"+ (mt/mbql-query venues+ {:aggregation [[:count]]+ :breakout [[:field %latitude {:binning {:strategy :num-bins+ :num-bins 10}}]]}))]+ (is (=? {:status "completed", :data {:is_sandboxed true}} result))+ (is (= 22 (reduce + (map second (mt/rows result)))))+ (is (=? {:binning_info {:binning_strategy "num-bins"}} (first (mt/cols result))))+ (is (every? nil? (vals (col-fingerprints result))))))))++(deftest sandboxed-cached-results-omit-fingerprints-test+ (testing "BOT-2115: a cached result replayed to a sandboxed user carries no fingerprint"+ (cache-test/with-mock-cache! [save-chan]+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ (letfn [(run-query []+ (qp/process-query (assoc (mt/mbql-query venues)+ :cache-strategy {:type :ttl+ :multiplier 60+ :avg-execution-ms 10+ :min_duration_ms 0})))]+ (let [result (run-query)]+ (is (nil? (:cached (:cache/details result))))+ (is (true? (-> result :data :is_sandboxed)))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))+ (is (every? nil? (vals (results-metadata-fingerprints result)))))+ (testing "cache entry should be saved within 5 seconds"+ (let [[_ chan] (a/alts!! [save-chan (a/timeout 5000)])]+ (is (= save-chan chan))))+ (let [result (run-query)]+ (is (true? (:cached (:cache/details result))))+ (is (true? (-> result :data :is_sandboxed)))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))+ (testing "results_metadata is rebuilt on replay and carries no fingerprint either"+ (is (seq (-> result :data :results_metadata :columns)))+ (is (every? nil? (vals (results-metadata-fingerprints result)))))))))))++(deftest sandboxed-pivot-results-omit-fingerprints-test+ (testing "BOT-2115: pivot query results for a sandboxed user carry no fingerprint"+ (met/with-gtaps! {:gtaps {:venues (venues-price-mbql-gtap-def)}+ :attributes {:price 1}}+ (let [result (qp.pivot/run-pivot-query+ (merge (mt/mbql-query venues+ {:aggregation [[:count]]+ :breakout [$price $category_id]})+ {:pivot-rows [0] :pivot-cols [1]}))]+ (is (=? {:status :completed, :data {:is_sandboxed true}} result))+ (is (seq (mt/cols result)))+ (is (every? nil? (vals (col-fingerprints result))))))))diff --git a/frontend/src/metabase/visualizations/visualizations/PivotTable/settings.ts b/frontend/src/metabase/visualizations/visualizations/PivotTable/settings.tsindex aa48035425a..7b90fe2cb9b 100644--- a/frontend/src/metabase/visualizations/visualizations/PivotTable/settings.ts+++ b/frontend/src/metabase/visualizations/visualizations/PivotTable/settings.ts@@ -114,6 +114,12 @@ export const settings = {columnsToPartition,isDimension,);+ // The cols of a run whose rows are restricted by a sandbox, an+ // impersonation role or database routing carry no fingerprint, since+ // it describes rows the user cannot see. The sort then keeps the+ // breakout order, so a card saved with `pivot_table.column_split`+ // (which the query builder persists on save) keeps its layout for+ // every viewer, while an unsaved split can differ between viewers.const [first, second, ...rest] = _.sortBy(dimensions, (col) =>getIn(col, ["fingerprint", "global", "distinct-count"]),);diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.cljindex 9b833a7c457..c4e7ad57049 100644--- a/src/metabase/query_processor/middleware/results_metadata.clj+++ b/src/metabase/query_processor/middleware/results_metadata.clj@@ -14,6 +14,7 @@[metabase.lib.schema.metadata :as lib.schema.metadata][metabase.queries.schema :as queries.schema][metabase.query-processor.db :as query-processor.db]+ [metabase.query-processor.middleware.row-restricted-metadata :as row-restricted-metadata][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema];; still passes state via the store's miscellaneous-value slot; general-cached-value migration pending@@ -84,6 +85,9 @@;; `qp.card/process-query-for-card` when a `card-transform` rewrites the query) to tell us;; this run's output is not the Card's canonical shape.(not (:qp/skip-result-metadata-persistence query))+ ;; A row-restricted run (sandbox, impersonation, DB routing) returns metadata without fingerprints.+ ;; Persisting it would erase the whole-table fingerprints an unrestricted run stored for every reader.+ (not (row-restricted-metadata/row-restricted? query));; Only update changed metadata; a non-default parameterized run writes only on;; parameter-independent column changes, not fingerprint drift (#83734, see also #72007).(let [parameter-independent? (boolean (:qp/non-default-parameterized? query))]@@ -132,14 +136,16 @@insights-col-metadata))(mu/defn- insights-xform :- fn?- [orig-metadata :- [:maybe ::qp.schema/metadata]- record! :- ifn?- rf :- ifn?]+ [orig-metadata :- [:maybe ::qp.schema/metadata]+ record! :- ifn?+ keep-fingerprints? :- :boolean+ rf :- ifn?](qp.reducible/combine-additional-reducing-fnsrf[(analyze/insights-rf orig-metadata)](fn combine [result {:keys [metadata insights]}]- (let [metadata (merge-final-column-metadata (-> result :data :cols) metadata)]+ (let [metadata (cond-> (merge-final-column-metadata (-> result :data :cols) metadata)+ (not keep-fingerprints?) row-restricted-metadata/remove-fingerprints)](record! metadata)(rf (cond-> result(map? result)@@ -160,9 +166,14 @@(let [query (cond-> queryskip-result-metadata-persistence?(assoc :qp/skip-result-metadata-persistence true))- record! (partial record-metadata! query)]+ record! (partial record-metadata! query)+ ;; With no fingerprint on the cols of a row-restricted run (sandbox, impersonation, DB routing), the insights+ ;; step computes one over the rows the current user sees. The frontend posts `:results_metadata` back as the+ ;; Card's `result_metadata` when the user saves the question, which would hand that subset's statistics to+ ;; every reader of the Card, so such a run returns no fingerprint in `:results_metadata` either.+ keep-fingerprints? (not (row-restricted-metadata/row-restricted? query))](fn record-and-return-metadata!-rff* [metadata]- (insights-xform metadata record! (rff metadata))))))+ (insights-xform metadata record! keep-fingerprints? (rff metadata))))))(mu/defn store-previous-result-metadata!"Store the previous value of a card's result metadata in the qp.store"diff --git a/src/metabase/query_processor/middleware/row_restricted_metadata.clj b/src/metabase/query_processor/middleware/row_restricted_metadata.cljnew file mode 100644index 00000000000..b1aab9ef1d9--- /dev/null+++ b/src/metabase/query_processor/middleware/row_restricted_metadata.clj@@ -0,0 +1,60 @@+(ns metabase.query-processor.middleware.row-restricted-metadata+ "Post-processing middleware that drops column metadata derived from rows the current user is not allowed to see.++ A Field's `:fingerprint` is computed at sync time over every row of the table, so it describes the whole column+ rather than any particular caller's row set. When the current user's row access is restricted, the fingerprint+ reveals the bounds and statistics of the rows that are hidden from them, so it is removed from every result column.++ `:results_metadata` is built after this middleware runs: with no fingerprint left on the cols, the insights step+ computes one from the returned rows. That one describes the rows the user got, but the frontend posts+ `:results_metadata` back as the Card's `result_metadata` when the user saves the question, so the results-metadata+ middleware drops it again with [[remove-fingerprints]] and skips persisting a row-restricted run's metadata on the+ Card; [[row-restricted?]] is the guard for both."+ (:require+ [metabase.util.match :as match]+ [metabase.util.performance :as perf]))++(defn- sandboxed?+ "Whether a sandbox was applied anywhere in `query`. Same marker+ [[metabase-enterprise.sandbox.query-processor.middleware.sandboxing/merge-sandboxing-metadata]] reads to set+ `:is_sandboxed`."+ [query]+ (boolean (match/match-one query {:query-permissions/sandboxed-table &truthy} true)))++(defn- impersonated?+ "Whether the query runs under a connection-impersonation role, which hands the database a role that filters rows.+ Set by [[metabase-enterprise.impersonation.middleware/apply-impersonation]]."+ [query]+ (some? (:impersonation/role query)))++(defn- db-routed?+ "Whether the query runs against a destination database swapped in by DB routing. The fingerprint was synced from the+ router database, so it describes rows of a database the user does not query at all.+ Set by [[metabase-enterprise.database-routing.middleware/attach-destination-db-middleware]]."+ [query]+ (some? (:destination-database/id query)))++(defn row-restricted?+ "Whether the current user sees only a subset of the rows the query's tables hold."+ [query]+ ;; Every signal is read from the preprocessed query rather than from the dynamic vars the execution middleware binds+ ;; (`*impersonation-role*` and friends): a cached result is replayed from outside those bindings.+ (or (sandboxed? query)+ (impersonated? query)+ (db-routed? query)))++(defn remove-fingerprints+ "`cols` without their `:fingerprint`."+ [cols]+ (perf/mapv #(dissoc % :fingerprint) cols))++(defn strip-row-restricted-fingerprints+ "Post-processing middleware. Removes `:fingerprint` from every result col when the current user's row access is+ restricted. Must run after every middleware that can add a fingerprint to the cols, which means it goes directly+ above [[metabase.query-processor.middleware.enterprise/merge-sandboxing-metadata]] in the post-processing list:+ that one merges the unrestricted query's metadata, fingerprint included, into the sandboxed results."+ [query rff]+ (if (row-restricted? query)+ (fn strip-row-restricted-fingerprints-rff* [metadata]+ (rff (update metadata :cols remove-fingerprints)))+ rff))diff --git a/src/metabase/query_processor/postprocess.clj b/src/metabase/query_processor/postprocess.cljindex a9deee94ed0..16764f46fcd 100644--- a/src/metabase/query_processor/postprocess.clj+++ b/src/metabase/query_processor/postprocess.clj@@ -14,6 +14,7 @@[metabase.query-processor.middleware.limit :as limit][metabase.query-processor.middleware.pivot-export :as pivot-export][metabase.query-processor.middleware.results-metadata :as results-metadata]+ [metabase.query-processor.middleware.row-restricted-metadata :as row-restricted-metadata][metabase.query-processor.middleware.visualization-settings :as viz-settings][metabase.query-processor.pivot.middleware :as qp.pivot.middleware][metabase.query-processor.schema :as qp.schema]@@ -40,6 +41,7 @@#'qp.middleware.enterprise/limit-download-result-rows#'qp.add-rows-truncated/add-rows-truncated#'qp.add-timezone-info/add-timezone-info+ #'row-restricted-metadata/strip-row-restricted-fingerprints#'qp.middleware.enterprise/merge-sandboxing-metadata#'qp.add-remaps/remap-results#'pivot-export/add-data-for-pivot-exportdiff --git a/test/metabase/query_processor/middleware/row_restricted_metadata_test.clj b/test/metabase/query_processor/middleware/row_restricted_metadata_test.cljnew file mode 100644index 00000000000..636e8eb972c--- /dev/null+++ b/test/metabase/query_processor/middleware/row_restricted_metadata_test.clj@@ -0,0 +1,57 @@+(ns metabase.query-processor.middleware.row-restricted-metadata-test+ (:require+ [clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.middleware.row-restricted-metadata :as row-restricted-metadata]))++(def ^:private metadata+ {:cols [{:name "PRICE", :fingerprint {:global {:distinct-count 4}, :type {:type/Number {:min 1.0, :max 4.0}}}}+ {:name "NAME", :fingerprint {:global {:distinct-count 100}}}+ {:name "count"}]})++(def ^:private stripped-metadata+ {:cols [{:name "PRICE"} {:name "NAME"} {:name "count"}]})++(defn- venues-query+ "An MBQL 5 query, the shape post-processing receives."+ []+ (lib/query meta/metadata-provider (meta/table-metadata :venues)))++(defn- strip+ "Run `metadata` through the middleware for `query`, returning what the next rff receives."+ [query]+ ((row-restricted-metadata/strip-row-restricted-fingerprints query identity) metadata))++(deftest ^:parallel unrestricted-query-keeps-fingerprints-test+ (testing "with no row restriction in force the cols are passed through untouched"+ (is (= metadata (strip (venues-query))))))++(deftest ^:parallel sandboxed-query-loses-fingerprints-test+ (testing "a sandboxed-table marker on a stage strips the fingerprint from every col, not only its table's (BOT-2115)"+ (is (= stripped-metadata+ (strip (assoc-in (venues-query) [:stages 0 :query-permissions/sandboxed-table] (meta/id :venues))))))+ (testing "including a marker on the stage of a join, where it sits once a sandboxed joined table is inlined"+ (let [query (lib/join (venues-query)+ (lib/join-clause (meta/table-metadata :categories)+ [(lib/= (meta/field-metadata :venues :category-id)+ (meta/field-metadata :categories :id))]))]+ (is (= metadata (strip query)))+ (is (= stripped-metadata+ (strip (assoc-in query [:stages 0 :joins 0 :stages 0 :query-permissions/sandboxed-table]+ (meta/id :categories)))))))+ (testing "including a marker on an inner stage, where it sits once a sandboxed source table is wrapped"+ (is (= stripped-metadata+ (strip (-> (venues-query)+ lib/append-stage+ (assoc-in [:stages 0 :query-permissions/sandboxed-table] (meta/id :venues))))))))++(deftest ^:parallel impersonated-query-loses-fingerprints-test+ (testing "a connection-impersonation role on the query strips the fingerprint from every col (BOT-2115)"+ (is (= stripped-metadata+ (strip (assoc (venues-query) :impersonation/role "restricted_role"))))))++(deftest ^:parallel db-routed-query-loses-fingerprints-test+ (testing "a destination database on the query strips the fingerprint from every col (BOT-2115)"+ (is (= stripped-metadata+ (strip (assoc (venues-query) :destination-database/id 2))))))