The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Strip native_form from query responses when unneeded (#83807)
Nathan Voxland5ac47dc204f3
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/dataset_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/dataset_test.cljindex dd2967dadeb..599619d0698 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/dataset_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/dataset_test.clj@@ -1,6 +1,8 @@(ns metabase-enterprise.sandbox.api.dataset-test- {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.sandbox.api.dataset-test]}}}}}}+ {:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.sandbox.api.dataset-test]}+ metabase.test.data/native-query {:namespaces [metabase-enterprise.sandbox.api.dataset-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all][metabase-enterprise.test :as met][metabase.test :as mt]))@@ -70,3 +72,35 @@(search :crowberto)))(is (=? {:values []}(search :rasta)))))))))++(deftest sandboxed-query-native-form-test+ (testing "POST /api/dataset doesn't return the sandboxed compiled query or its params to a sandboxed user"+ ;; sandbox defs are thunks so `mt/id` resolves against the DB copy `with-gtaps-for-user!` creates+ (doseq [[sandbox-type sandbox-thunk attributes secrets]+ [[:remapping+ (fn []+ {:remappings {"price" ["variable" [:field (mt/id :venues :price) nil]]}})+ {"price" "1"}+ []]+ [:card+ (fn []+ {:query (mt/native-query+ {:query (str "SELECT * FROM VENUES WHERE NAME <> {{secret_attr}} "+ "AND 'SANDBOX-SQL-MARKER' <> ''")+ :template-tags {"secret_attr" {:name "secret_attr"+ :display-name "secret_attr"+ :type "text"+ :required true+ :id "11111111-1111-1111-1111-111111111111"}}})+ :remappings {"secret_attr" ["variable" ["template-tag" "secret_attr"]]}})+ {"secret_attr" "SECRET-ATTRIBUTE-VALUE"}+ ["SANDBOX-SQL-MARKER" "SECRET-ATTRIBUTE-VALUE"]]]]+ (testing sandbox-type+ (met/with-gtaps-for-user! :rasta+ {:gtaps {:venues (sandbox-thunk)}+ :attributes attributes}+ (let [response (mt/user-http-request :rasta :post 202 "dataset" (mt/mbql-query venues {:limit 1}))]+ (is (= "completed" (:status response)))+ (is (nil? (get-in response [:data :native_form])))+ (doseq [secret secrets]+ (is (not (str/includes? (pr-str response) secret))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 1c57363f5b3..011001e17f3 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -23,6 +23,7 @@[metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]+ [metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.cache-test :as cache-test][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]@@ -1343,9 +1344,13 @@:source-table (str "card__" (:id model))})regular-result (mt/with-test-user :crowberto(qp/process-query query))- sandboxed-result (met/with-user-attributes! :rasta {"category" "Gizmo"}+ [sandboxed-result+ sandboxed-sql] (met/with-user-attributes! :rasta {"category" "Gizmo"}(mt/with-test-user :rasta- (qp/process-query query)))]+ ;; results don't include `:native_form` for a sandboxed user, so compile+ ;; the query separately to see which table it uses+ [(qp/process-query query)+ (:query (qp.compile/compile query))]))](testing "Unsandboxed"(testing "Sees full result set"(is (= 200 (-> regular-result mt/rows ffirst))@@ -1359,8 +1364,7 @@(is (= 51 (-> sandboxed-result mt/rows ffirst))"Sandboxed user got whole results instead of filtered"))(testing "Does not use the cache table"- (is (not (str/includes? (-> sandboxed-result :data :native_form :query)- (:table_name persisted-info)))+ (is (not (str/includes? sandboxed-sql (:table_name persisted-info)))"Erroneously used the persisted model cache")))))))))))(deftest persistence-disabled-for-native-card-template-tag-when-sandboxed-testdiff --git a/src/metabase/query_processor/execute.clj b/src/metabase/query_processor/execute.cljindex 60eca1e56ea..9b7e185c63d 100644--- a/src/metabase/query_processor/execute.clj+++ b/src/metabase/query_processor/execute.clj@@ -1,5 +1,6 @@(ns metabase.query-processor.execute(:require+ [medley.core :as m][metabase.lib.core :as lib][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.cache :as cache]@@ -26,6 +27,29 @@(assoc :native_form ((some-fn :qp/compiled-inline :qp/compiled) query)))))](qp query rff*))))+(defn- dissoc-native-form [result]+ (cond-> result+ (map? result) (m/dissoc-in [:data :native_form])))++(mu/defn- remove-native-form-without-native-perms :- ::qp.schema/qp+ "Remove `:native_form` from the results unless the current user has ad-hoc native query perms for the query's+ database. Without them, the compiled query could leak things the user can't otherwise see, e.g. a sandbox's+ injected filter, a sandbox Card's SQL, or the login attribute values bound into it.++ This runs outside the cache middleware, so cached results (which include the `:native_form` of whoever ran the query+ originally) are stripped as well."+ [qp :- ::qp.schema/qp]+ (fn [query rff]+ (if (qp.perms/current-user-has-adhoc-native-query-perms? query)+ (qp query rff)+ (letfn [(rff* [metadata]+ (let [rf (rff (dissoc metadata :native_form))]+ (fn+ ([] (rf))+ ([result] (dissoc-native-form (rf (dissoc-native-form result))))+ ([acc row] (rf acc row)))))]+ (qp query rff*)))))+(def ^:private middleware"Middleware that happens after compilation, AROUND query execution itself. Has the form@@ -45,6 +69,7 @@#'update-used-cards/update-used-cards!#'add-native-form-to-result-metadata#'cache/maybe-return-cached-results+ #'remove-native-form-without-native-perms#'qp.perms/check-query-permissions#'qp.middleware.enterprise/check-download-permissions-middleware])diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 53b740b5a98..e01b6c7ebef 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -1256,3 +1256,25 @@(is (some #(= (:id %) (mt/id :venues :price))(->> result :tables (mapcat :fields)))"Sensitive field SHOULD be included when :settings :include-sensitive-fields is true"))))))++(deftest native-form-requires-native-query-perms-test+ (testing "POST /api/dataset only returns the compiled query in `:native_form` to users with native query perms"+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)+ (doseq [[create-queries expected] {:query-builder-and-native some?+ :query-builder nil?}]+ (testing (format "with %s" create-queries)+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries create-queries)+ (let [response (mt/user-http-request :rasta :post 202 "dataset" (mt/mbql-query venues {:limit 1}))]+ (is (= "completed" (:status response)))+ (is (expected (get-in response [:data :native_form])))))))))++(deftest card-query-native-form-requires-native-query-perms-test+ (testing "POST /api/card/:id/query doesn't return the compiled query to users without native query perms"+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query venues {:limit 1})}]+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)+ (let [response (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id))]+ (is (= "completed" (:status response)))+ (is (nil? (get-in response [:data :native_form]))))))))diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex cc657e6a2f8..43550753212 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -17,6 +17,7 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.lib.metadata :as lib.metadata][metabase.lib.test-util :as lib.tu]+ [metabase.permissions.core :as perms][metabase.queries.models.query :as query][metabase.query-processor.middleware.cache :as cache][metabase.query-processor.middleware.cache-backend.db :as backend.db]@@ -1011,6 +1012,34 @@#"You do not have permissions to run this query"(run-forbidden-query)))))))))))+(deftest cached-results-native-form-requires-native-query-perms-test+ (testing "Cached results don't return the compiled query to a user without native query perms"+ (mt/with-temp-copy-of-db+ (mt/with-no-data-perms-for-all-users!+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)+ (perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :query-builder)+ (with-mock-cache! [save-chan]+ (letfn [(run-query []+ (qp/process-query (assoc (mt/mbql-query checkins {:aggregation [[:count]]})+ :cache-strategy (ttl-strategy))))]+ (testing "Run query as superuser to populate the cache"+ (request/with-current-user (mt/user->id :crowberto)+ (is (=? {:data {:native_form some?}}+ (run-query)))))+ (mt/wait-for-result save-chan)+ (testing "Cached results for the superuser still include the compiled query"+ (request/with-current-user (mt/user->id :crowberto)+ (is (=? {:cache/details {:cached true}+ :data {:native_form some?}}+ (run-query)))))+ (testing "Cached results for a user without native query perms don't include the compiled query"+ (mt/with-test-user :rasta+ (let [result (run-query)]+ (is (=? {:cache/details {:cached true}+ :data {:rows [[1000]]}}+ result))+ (is (nil? (get-in result [:data :native_form]))))))))))))+(deftest ^:parallel cached-results-rff-preserves-fresh-accumulator-test(testing "On cache hit, the rff chain's accumulator (with any modifications from middlewareslike update-viz-settings) must not be clobbered by the replayed cached final-metadata