The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[pivot] Fix restoring pivot query results from cache (#84718)
Oleksandr Yakushevbb00f4cd3ff4
diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljcindex 0f220c76d6b..6759d1f38db 100644--- a/src/metabase/lib/schema/info.cljc+++ b/src/metabase/lib/schema/info.cljc@@ -88,6 +88,11 @@[:metadata/model-metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]][:metadata/own-model-query? {:optional true} :boolean][:pivot/original-query {:optional true} [:maybe [:ref :metabase.lib.schema/query]]]+ ;; Hash of the pivot query as the caller submitted it, before the pivot QP rewrote it into subqueries. Added by+ ;; [[metabase.query-processor.pivot]] to the first subquery only, so that+ ;; [[metabase.query-processor.middleware.process-userland-query]] records the whole pivot's running time under the+ ;; hash that the cache strategy's `:avg-execution-ms` lookup uses.+ [:pivot/query-hash {:optional true} [:maybe [:ref ::hash]]];; Pivot QP runs multiple queries, and in the dataset api, we need to have access to the original query;; so that we can pass it to the pivot.qp for downloads on unsaved questions;; this gets added by [[metabase.query-processor.pivot]] for pivot queries; it's merged in to other metadatadiff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex a2edead9b1a..5d6cdf0068b 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -289,7 +289,8 @@;; catch below. Reading the EE dynamic vars directly from the catch block does NOT work — Clojure pops the;; EE `binding` blocks during stack unwind, so by the time control reaches the catch the values are gone.(binding [*execution-context-ref* (atom nil)]- (let [query (assoc-in query [:info :query-hash] (qp.util/query-hash query))+ (let [query (assoc-in query [:info :query-hash] (or (get-in query [:info :pivot/query-hash])+ (qp.util/query-hash query)))execution-info (query-execution-info query)](letfn [(rff* [metadata](let [result (rff metadata)diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex 887db389328..13af4deeb1b 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -37,6 +37,7 @@[metabase.query-processor.schema :as qp.schema][metabase.query-processor.settings :as qp.settings][metabase.query-processor.setup :as qp.setup]+ [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]@@ -234,7 +235,8 @@;; Single query - use normal QP pipeline to preserve userland metadata(qp/process-query (maybe-userland first-query) rff);; Multiple queries - use custom pivot pipeline- (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns info rff more-queries pivot-limit)+ ;; Only the first query, whose execution spans all the others, records its running time under the pivot's hash.+ (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns (dissoc info :pivot/query-hash) rff more-queries pivot-limit)first-query (maybe-userland first-query)](binding [qp.pipeline/*execute* (or execute qp.pipeline/*execute*)qp.pipeline/*reduce* (or reduce qp.pipeline/*reduce*)]@@ -892,7 +894,10 @@;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.(qp.setup/with-qp-setup [query query](let [query (cond-> query- (seq (:info query)) qp/userland-query)+ ;; Hash the query before it gets rewritten so its running time is recorded under the same hash+ ;; that `metabase.query-processor.card/enrich-strategy` looks up for the cache strategy.+ (seq (:info query)) (-> (assoc-in [:info :pivot/query-hash] (qp.util/query-hash query))+ qp/userland-query))rff (or rff qp.reducible/default-rff);; Everything between here and the first `qp/process-query`;; runs outside the QP's own middleware, so wrap it in the same exception-catching middlewarediff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljcindex 4fc9d4d49ff..8b9e7b2f70e 100644--- a/test/metabase/lib/serialize_test.cljc+++ b/test/metabase/lib/serialize_test.cljc@@ -328,7 +328,8 @@{:metadata/own-model-query? true:metadata/model-metadata [{:lib/type :metadata/column, :name "ID", :base-type :type/Integer}]:pivot/original-query clean-query- :pivot/result-metadata :none})+ :pivot/result-metadata :none+ :pivot/query-hash nil})(deftest ^:parallel internal-keys-fixture-covers-the-schemas-test(testing "the fixtures carry every internal key the schemas declare, so these tests cannot drift"diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.cljindex 6daa199f351..febcb4d6b5d 100644--- a/test/metabase/query_processor/card_test.clj+++ b/test/metabase/query_processor/card_test.clj@@ -15,15 +15,20 @@[metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]+ [metabase.queries.core :as queries][metabase.query-processor.card :as qp.card]+ [metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.middleware.results-metadata :as qp.results-metadata]+ [metabase.query-processor.pivot :as qp.pivot];; asserts on the store's miscellaneous-value slot that the results-metadata middleware writes^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp][metabase.query-processor.test-util :as qp.test-util]+ [metabase.query-processor.util :as qp.util][metabase.test :as mt][metabase.util :as u]- [metabase.util.json :as json]))+ [metabase.util.json :as json]+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -241,6 +246,37 @@result))(is (= [[100]] (mt/formatted-rows [int] result))))))))+(deftest pivot-card-records-average-execution-time-under-card-query-hash-test+ (testing "Running a pivot Card records its execution time under the hash the cache strategy looks up, so a TTL+ strategy gets a real `:avg-execution-ms` instead of 0"+ (mt/test-driver :h2+ (mt/with-temp [:model/Card card {:dataset_query (mt/mbql-query venues+ {:aggregation [[:count]]+ :breakout [$category_id $price]})+ :display :pivot}]+ (doseq [native? [false true]]+ (testing (str "use-native-pivot-tables = " native?)+ (mt/with-temporary-setting-values [use-native-pivot-tables native?]+ (mt/with-dynamic-fn-redefs [qp.card/cache-strategy (constantly {:type :ttl+ :multiplier 10+ :min_duration_ms 0})]+ (let [card-query-hash (qp.util/query-hash+ (qp.card/query-for-card card nil (qp.constraints/default-query-constraints) nil))]+ (t2/delete! :model/Query :query_hash card-query-hash)+ (binding [qp.util/*execute-async?* false+ qp.pivot/*check-pivot-parity?* false]+ (is (=? {:status :completed}+ (mt/as-admin+ (qp.card/process-query-for-card+ card :api+ :make-run (fn [qp _export-format]+ (fn [query info]+ (qp (update query :info merge info) nil))))))))+ (is (some? (queries/average-execution-time-ms card-query-hash)))+ (is (=? {:type :ttl, :avg-execution-ms (queries/average-execution-time-ms card-query-hash)}+ (:cache-strategy+ (qp.card/query-for-card card nil (qp.constraints/default-query-constraints) nil)))))))))))))+(deftest nested-query-permissions-test(testing "Reading a Card is not enough to run it when its source query is a Card we cannot read"(mt/with-no-data-perms-for-all-users!