The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Don't trigger result-metadata write based on non-default parameters (#72007)
Timothy S. Dean8d5bd8ebcb7c
diff --git a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.jsindex 73095cc12ad..dc3f063d6e0 100644--- a/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js+++ b/e2e/test/scenarios/permissions/sandboxing/sandboxing-via-api.cy.spec.js@@ -717,6 +717,7 @@ describe("admin > permissions > sandboxes (tested via the API)", () => {id: "1115dc4f-6b9d-812e-7f72-b87ab885c88a",name: "sandbox",type: "number",+ default: "1",},},},@@ -743,6 +744,7 @@ describe("admin > permissions > sandboxes (tested via the API)", () => {id: "3d69ba99-7076-2252-30bd-0bb8810ba895",name: "sandbox",type: "text",+ default: "Widget",},},},diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.cljindex 8ef0a91b6d7..bd5676c0541 100644--- a/src/metabase/query_processor/middleware/parameters.clj+++ b/src/metabase/query_processor/middleware/parameters.clj@@ -1,5 +1,6 @@(ns metabase.query-processor.middleware.parameters"Middleware for substituting parameters in queries."+ (:refer-clojure :exclude [some])(:require[clojure.data :as data][clojure.set :as set]@@ -14,7 +15,8 @@[metabase.query-processor.middleware.parameters.native :as qp.native][metabase.util :as u][metabase.util.log :as log]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.performance :refer [some]]))(mu/defn- expand-stage :- ::lib.schema/stage"Expand `:parameters` in one stage map that contains them."@@ -82,6 +84,44 @@queryparameters))+(defn- tag-for-param+ "Find the template tag in `template-tags` that matches `param`'s target."+ [template-tags param]+ (when-let [name-or-id (lib.parameters/parameter-target-template-tag-name (:target param))]+ (or (get template-tags name-or-id) ; handle name+ (some (fn [[_ tag]] ; handle id (defensive, may be unnecessary)+ (when (= (:id tag) name-or-id)+ tag))+ template-tags))))++(defn- stage-has-non-default-params?+ "True if a stage has parameters with non-default values.+ For MBQL stages, any parameters count as non-default.+ For native stages, only parameters whose value differs from the template tag default."+ [stage]+ (when-let [params (seq (:parameters stage))]+ (if-not (lib/native-stage? stage)+ true+ ;; Native: check if any parameter value differs from its template tag default+ (let [template-tags (:template-tags stage)+ non-default-param? (fn [param]+ (when-let [value (:value param)]+ (let [tag (tag-for-param template-tags param)+ default (:default tag)]+ (or (nil? default)+ (not= value default)))))]+ (some non-default-param? params)))))++(defn- maybe-skip-result-metadata-persistence+ "Sets :qp/skip-result-metadata-persistence to true/false."+ [query]+ (->> (:stages query)+ (some stage-has-non-default-params?)+ boolean+ ;; `qp/skip-result-metadata-persistence` is used by [[metabase.query-processor.middleware.results-metadata/record-metadata!]]+ ;; to decide whether to record metadata.+ (assoc query :qp/skip-result-metadata-persistence)))+(mu/defn- move-top-level-params-to-stage :- ::lib.schema/query"Move any top-level parameters to the stage they affect."[{:keys [info parameters], :as query} :- ::lib.schema/query]@@ -99,7 +139,8 @@lib/append-stage(seq parameters)- (move-top-level-params-to-stage* parameters)))+ (-> (move-top-level-params-to-stage* parameters)+ maybe-skip-result-metadata-persistence)))(mu/defn- expand-parameters :- ::lib.schema/query"If any parameters were supplied then substitute them into the query."diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.cljindex 476dfcd370b..df890fb173a 100644--- a/src/metabase/query_processor/middleware/results_metadata.clj+++ b/src/metabase/query_processor/middleware/results_metadata.clj@@ -27,7 +27,7 @@;;; | Middleware |;;; +----------------------------------------------------------------------------------------------------------------+-(mu/defn- comparable-metadata :- [:sequential :map]+(mu/defn- comparable-metadata"Smooth out any unimportant differences in metadata so we can do an easy equality check."[metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]](letfn [(remove-underscore-nil-keys@@ -69,6 +69,11 @@driver/*driver*;; pivot queries can run multiple queries, only record metadata for the main query(not= actual-metadata :none)+ ;; Don't persist result_metadata for parameterized queries with non-default values.+ ;; MBQL queries with any parameters are skipped because filters change fingerprints.+ ;; Native queries are only skipped when parameter values differ from template tag defaults.+ ;; See QUE2-502 for details.+ (not (:qp/skip-result-metadata-persistence query))(driver.u/supports? driver/*driver* :nested-queries (lib.metadata/database query))card-id;; don't want to update metadata when we use a Card as a source Card.diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.cljindex a45a26fb8e0..66ffe8c6576 100644--- a/test/metabase/dashboards_rest/api_test.clj+++ b/test/metabase/dashboards_rest/api_test.clj@@ -5528,3 +5528,115 @@})](is (partial= {:name "Test Dashboard":creator_id (mt/user->id :crowberto)} response))))++;;; +--------------------------------------------------------------------------------------------------++;;; | result_metadata persistence with parameters (QUE2-502) |+;;; | See also: metabase.queries-rest.api.card-test (card endpoint tests for native cards) |+;;; +--------------------------------------------------------------------------------------------------+++(deftest parameterized-mbql-dashcard-does-not-persist-result-metadata-test+ ;; QUE2-502+ (testing "POST /api/dashboard/:id/dashcard/:id/card/:id/query — MBQL card with parameters should not update result_metadata"+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query (mt/mbql-query orders+ {:aggregation [[:count] [:sum $total]]+ :breakout [$product_id]})}+ :model/Dashboard {dashboard-id :id}+ {:parameters [{:name "Product ID"+ :slug "product_id"+ :id "_PRODUCT_ID_"+ :type :id}]}+ :model/DashboardCard {dashcard-id :id}+ {:card_id card-id+ :dashboard_id dashboard-id+ :parameter_mappings [{:parameter_id "_PRODUCT_ID_"+ :card_id card-id+ :target [:dimension (mt/$ids orders $product_id)]}]}]+ ;; Run without parameters to establish baseline metadata+ (let [url (dashboard-card-query-url dashboard-id card-id dashcard-id)]+ (mt/user-http-request :rasta :post 202 url)+ (let [baseline-metadata (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (some? baseline-metadata)+ "Baseline metadata should be established by non-parameterized run")+ ;; Run with a parameter value — metadata should NOT be updated+ (mt/user-http-request :rasta :post 202 url+ {:parameters [{:id "_PRODUCT_ID_" :value 1}]})+ (is (= baseline-metadata+ (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should not change when MBQL dashcard is run with parameters")+ ;; Run with a different parameter value — metadata should still not change+ (mt/user-http-request :rasta :post 202 url+ {:parameters [{:id "_PRODUCT_ID_" :value 50}]})+ (is (= baseline-metadata+ (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should remain stable across different parameter values"))))))++(deftest parameterized-native-dashcard-does-not-persist-result-metadata-test+ ;; QUE2-502+ (testing "POST /api/dashboard/:id/dashcard/:id/card/:id/query — native card with non-default parameters should not update result_metadata"+ (let [native-query (mt/native-query+ {:query "SELECT COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{product_id}}"+ :template-tags {"product_id" {:id "_PRODUCT_ID_TAG_"+ :name "product_id"+ :display-name "Product ID"+ :type :number+ :required true+ :default "1"}}})]+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query native-query}+ :model/Dashboard {dashboard-id :id}+ {:parameters [{:name "Product ID"+ :slug "product_id"+ :id "_DASH_PRODUCT_ID_"+ :type :number+ :default "1"}]}+ :model/DashboardCard {dashcard-id :id}+ {:card_id card-id+ :dashboard_id dashboard-id+ :parameter_mappings [{:parameter_id "_DASH_PRODUCT_ID_"+ :card_id card-id+ :target [:variable [:template-tag "product_id"]]}]}]+ (let [url (dashboard-card-query-url dashboard-id card-id dashcard-id)]+ ;; Run without parameters to establish baseline metadata (uses template tag default)+ (mt/user-http-request :rasta :post 202 url)+ (let [baseline-metadata (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (some? baseline-metadata)+ "Baseline metadata should be established by default-value run")+ ;; Run with a non-default parameter value — metadata should NOT be updated+ (mt/user-http-request :rasta :post 202 url+ {:parameters [{:id "_DASH_PRODUCT_ID_" :value "50"}]})+ (is (= baseline-metadata+ (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should not change when native dashcard is run with non-default parameter values")))))))++(deftest native-dashcard-with-default-parameters-persists-result-metadata-test+ ;; QUE2-502+ (testing "POST /api/dashboard/:id/dashcard/:id/card/:id/query — native card with default parameter values SHOULD update result_metadata"+ (let [native-query (mt/native-query+ {:query "SELECT COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{product_id}}"+ :template-tags {"product_id" {:id "_PRODUCT_ID_TAG_"+ :name "product_id"+ :display-name "Product ID"+ :type :number+ :required true+ :default "1"}}})]+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query native-query}+ :model/Dashboard {dashboard-id :id}+ {:parameters [{:name "Product ID"+ :slug "product_id"+ :id "_DASH_PRODUCT_ID_"+ :type :number+ :default "1"}]}+ :model/DashboardCard {dashcard-id :id}+ {:card_id card-id+ :dashboard_id dashboard-id+ :parameter_mappings [{:parameter_id "_DASH_PRODUCT_ID_"+ :card_id card-id+ :target [:variable [:template-tag "product_id"]]}]}]+ ;; Run with the default value passed explicitly (as the frontend does)+ (mt/user-http-request :rasta :post 202+ (dashboard-card-query-url dashboard-id card-id dashcard-id)+ {:parameters [{:id "_DASH_PRODUCT_ID_" :value "1"}]})+ (is (some? (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should be persisted when native dashcard runs with default parameter values")))))diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 542e6012e66..c5e5e2f073b 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -4832,3 +4832,56 @@:aggregation [["count"]]}}(-> (mt/user-http-request :crowberto :get 200 (str "card/" (:id card)) :legacy-mbql true):dataset_query))))))++;;; +--------------------------------------------------------------------------------------------------++;;; | result_metadata persistence with parameters (QUE2-502) |+;;; | See also: metabase.dashboards-rest.api-test (dashboard endpoint tests for MBQL cards) |+;;; +--------------------------------------------------------------------------------------------------+++(deftest parameterized-native-card-does-not-persist-result-metadata-test ; QUE2-502+ (testing "POST /api/card/:id/query — native card with non-default parameter values should not update result_metadata"+ (let [query (mt/native-query+ {:query "SELECT COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{product_id}}"+ :template-tags {"product_id" {:id "_PRODUCT_ID_"+ :name "product_id"+ :display-name "Product ID"+ :type :number+ :required true+ :default "1"}}})]+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query query}]+ ;; First run without parameters to establish baseline metadata (uses default value)+ (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id))+ (let [baseline-metadata (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (some? baseline-metadata)+ "Baseline metadata should be established by default-value run")+ ;; Run with a non-default parameter value — metadata should NOT be updated+ (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id)+ {:parameters [{:id "_PRODUCT_ID_"+ :type :number+ :target [:variable [:template-tag "product_id"]]+ :value "50"}]})+ (is (= baseline-metadata+ (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should not change when native card is run with non-default parameter values"))))))++(deftest native-card-with-default-parameters-persists-result-metadata-test ; QUE2-502+ (testing "POST /api/card/:id/query — native card with default parameter values SHOULD update result_metadata"+ (let [query (mt/native-query+ {:query "SELECT COUNT(*) AS cnt FROM ORDERS WHERE PRODUCT_ID = {{product_id}}"+ :template-tags {"product_id" {:id "_PRODUCT_ID_"+ :name "product_id"+ :display-name "Product ID"+ :type :number+ :required true+ :default "1"}}})]+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query query}]+ ;; Run with the default value passed explicitly (as the frontend does)+ (mt/user-http-request :rasta :post 202 (format "card/%d/query" card-id)+ {:parameters [{:id "_PRODUCT_ID_"+ :type :number+ :target [:variable [:template-tag "product_id"]]+ :value "1"}]})+ (is (some? (t2/select-one-fn :result_metadata :model/Card :id card-id))+ "result_metadata should be persisted when native card runs with default parameter values")))))diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.cljindex a9723840fee..39740e967de 100644--- a/test/metabase/query_processor/middleware/results_metadata_test.clj+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj@@ -12,6 +12,7 @@[metabase.lib.test-util :as lib.tu][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]+ [metabase.query-processor.card :as qp.card][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.results-metadata :as middleware.results-metadata][metabase.query-processor.preprocess :as qp.preprocess]@@ -504,3 +505,73 @@:description "The date and time an order was submitted.":display_name "Created At: Quarter":effective_type :type/DateTime}])))))++(deftest different-filters-update-result-metadata-test+ ;; When the query itself changes (different filter baked in), result_metadata should update.+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query (mt/mbql-query orders+ {:aggregation [[:count] [:sum $total]]+ :breakout [$product_id]})+ :result_metadata nil}]+ (let [run-with-filter!+ (fn [product-id]+ (t2/update! :model/Card card-id+ {:dataset_query (mt/mbql-query orders+ {:aggregation [[:count] [:sum $total]]+ :breakout [$product_id]+ :filter [:= $product_id product-id]})})+ (mt/as-admin+ (qp/process-query-for-card+ card-id :api+ :make-run (constantly+ (fn [query info]+ (qp/process-query (assoc query :info info))))))+ (t2/select-one-fn :result_metadata :model/Card :id card-id))]+ ;; First run establishes baseline metadata+ (run-with-filter! 1)+ ;; Second run with different filter — metadata should update because the query changed+ (let [meta-before (t2/select-one-fn :result_metadata :model/Card :id card-id)+ _ (run-with-filter! 50)+ meta-after (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (not= meta-before meta-after)+ "result_metadata should update when the query changes"))+ ;; Third run with original filter — metadata updates again+ (let [meta-before (t2/select-one-fn :result_metadata :model/Card :id card-id)+ _ (run-with-filter! 1)+ meta-after (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (not= meta-before meta-after)+ "result_metadata should update when the query changes back")))))++(deftest parameterized-queries-do-not-thrash-result-metadata-test+ ;; When parameters are applied via :parameters, result_metadata should not be updated.+ ;; See QUE2-502 for details.+ (mt/with-temp [:model/Card {card-id :id}+ {:dataset_query (mt/mbql-query orders+ {:aggregation [[:count] [:sum $total]]+ :breakout [$product_id]})+ :result_metadata nil}]+ (let [run-with-parameters!+ (fn [product-id]+ ;; Bind *allow-arbitrary-mbql-parameters* as the dashboard code path does.+ (binding [qp.card/*allow-arbitrary-mbql-parameters* true]+ (mt/as-admin+ (qp/process-query-for-card+ card-id :api+ :parameters [{:id "product-id-param"+ :type :id+ :target [:dimension (mt/$ids orders $product_id)]+ :value [product-id]}]+ :make-run (constantly+ (fn [query info]+ (qp/process-query (assoc query :info info)))))))+ (t2/select-one-fn :result_metadata :model/Card :id card-id))+ ;; First run establishes baseline metadata+ _ (run-with-parameters! 1)+ meta-before (t2/select-one-fn :result_metadata :model/Card :id card-id)+ ;; Second run with different parameter+ _ (run-with-parameters! 50)+ meta-after (t2/select-one-fn :result_metadata :model/Card :id card-id)]+ (is (some? meta-before)+ "Baseline metadata should be established by first run")+ (is (= meta-before meta-after)+ "result_metadata should not change for parameterized queries"))))