Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[pivot] Fix restoring pivot query results from cache (#84718)

Oleksandr Yakushevbb00f4cd3ff4
diff --git a/src/metabase/lib/schema/info.cljc b/src/metabase/lib/schema/info.cljc
index 0f220c76d6b..6759d1f38db 100644
--- a/src/metabase/lib/schema/info.cljc
+++ b/src/metabase/lib/schema/info.cljc
@@ -88,6 +88,11 @@
[:metadata/model-metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]
[:metadata/own-model-query? {:optional true} :boolean]
[:pivot/original-query {:optional true} [:maybe [:ref :metabase.lib.schema/query]]]
+ ;; Hash of the pivot query as the caller submitted it, before the pivot QP rewrote it into subqueries. Added by
+ ;; [[metabase.query-processor.pivot]] to the first subquery only, so that
+ ;; [[metabase.query-processor.middleware.process-userland-query]] records the whole pivot's running time under the
+ ;; hash that the cache strategy's `:avg-execution-ms` lookup uses.
+ [:pivot/query-hash {:optional true} [:maybe [:ref ::hash]]]
;; Pivot QP runs multiple queries, and in the dataset api, we need to have access to the original query
;; so that we can pass it to the pivot.qp for downloads on unsaved questions
;; this gets added by [[metabase.query-processor.pivot]] for pivot queries; it's merged in to other metadata
diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.clj
index a2edead9b1a..5d6cdf0068b 100644
--- a/src/metabase/query_processor/middleware/process_userland_query.clj
+++ b/src/metabase/query_processor/middleware/process_userland_query.clj
@@ -289,7 +289,8 @@
;; catch below. Reading the EE dynamic vars directly from the catch block does NOT work — Clojure pops the
;; EE `binding` blocks during stack unwind, so by the time control reaches the catch the values are gone.
(binding [*execution-context-ref* (atom nil)]
- (let [query (assoc-in query [:info :query-hash] (qp.util/query-hash query))
+ (let [query (assoc-in query [:info :query-hash] (or (get-in query [:info :pivot/query-hash])
+ (qp.util/query-hash query)))
execution-info (query-execution-info query)]
(letfn [(rff* [metadata]
(let [result (rff metadata)
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index 887db389328..13af4deeb1b 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -37,6 +37,7 @@
[metabase.query-processor.schema :as qp.schema]
[metabase.query-processor.settings :as qp.settings]
[metabase.query-processor.setup :as qp.setup]
+ [metabase.query-processor.util :as qp.util]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
@@ -234,7 +235,8 @@
;; Single query - use normal QP pipeline to preserve userland metadata
(qp/process-query (maybe-userland first-query) rff)
;; Multiple queries - use custom pivot pipeline
- (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns info rff more-queries pivot-limit)
+ ;; Only the first query, whose execution spans all the others, records its running time under the pivot's hash.
+ (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns (dissoc info :pivot/query-hash) rff more-queries pivot-limit)
first-query (maybe-userland first-query)]
(binding [qp.pipeline/*execute* (or execute qp.pipeline/*execute*)
qp.pipeline/*reduce* (or reduce qp.pipeline/*reduce*)]
@@ -892,7 +894,10 @@
;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.
(qp.setup/with-qp-setup [query query]
(let [query (cond-> query
- (seq (:info query)) qp/userland-query)
+ ;; Hash the query before it gets rewritten so its running time is recorded under the same hash
+ ;; that `metabase.query-processor.card/enrich-strategy` looks up for the cache strategy.
+ (seq (:info query)) (-> (assoc-in [:info :pivot/query-hash] (qp.util/query-hash query))
+ qp/userland-query))
rff (or rff qp.reducible/default-rff)
;; Everything between here and the first `qp/process-query`
;; runs outside the QP's own middleware, so wrap it in the same exception-catching middleware
diff --git a/test/metabase/lib/serialize_test.cljc b/test/metabase/lib/serialize_test.cljc
index 4fc9d4d49ff..8b9e7b2f70e 100644
--- a/test/metabase/lib/serialize_test.cljc
+++ b/test/metabase/lib/serialize_test.cljc
@@ -328,7 +328,8 @@
{:metadata/own-model-query? true
:metadata/model-metadata [{:lib/type :metadata/column, :name "ID", :base-type :type/Integer}]
:pivot/original-query clean-query
- :pivot/result-metadata :none})
+ :pivot/result-metadata :none
+ :pivot/query-hash nil})
(deftest ^:parallel internal-keys-fixture-covers-the-schemas-test
(testing "the fixtures carry every internal key the schemas declare, so these tests cannot drift"
diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.clj
index 6daa199f351..febcb4d6b5d 100644
--- a/test/metabase/query_processor/card_test.clj
+++ b/test/metabase/query_processor/card_test.clj
@@ -15,15 +15,20 @@
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.queries.core :as queries]
[metabase.query-processor.card :as qp.card]
+ [metabase.query-processor.middleware.constraints :as qp.constraints]
[metabase.query-processor.middleware.results-metadata :as qp.results-metadata]
+ [metabase.query-processor.pivot :as qp.pivot]
;; asserts on the store's miscellaneous-value slot that the results-metadata middleware writes
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
[metabase.query-processor.test-util :as qp.test-util]
+ [metabase.query-processor.util :as qp.util]
[metabase.test :as mt]
[metabase.util :as u]
- [metabase.util.json :as json]))
+ [metabase.util.json :as json]
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -241,6 +246,37 @@
result))
(is (= [[100]] (mt/formatted-rows [int] result))))))))
+(deftest pivot-card-records-average-execution-time-under-card-query-hash-test
+ (testing "Running a pivot Card records its execution time under the hash the cache strategy looks up, so a TTL
+ strategy gets a real `:avg-execution-ms` instead of 0"
+ (mt/test-driver :h2
+ (mt/with-temp [:model/Card card {:dataset_query (mt/mbql-query venues
+ {:aggregation [[:count]]
+ :breakout [$category_id $price]})
+ :display :pivot}]
+ (doseq [native? [false true]]
+ (testing (str "use-native-pivot-tables = " native?)
+ (mt/with-temporary-setting-values [use-native-pivot-tables native?]
+ (mt/with-dynamic-fn-redefs [qp.card/cache-strategy (constantly {:type :ttl
+ :multiplier 10
+ :min_duration_ms 0})]
+ (let [card-query-hash (qp.util/query-hash
+ (qp.card/query-for-card card nil (qp.constraints/default-query-constraints) nil))]
+ (t2/delete! :model/Query :query_hash card-query-hash)
+ (binding [qp.util/*execute-async?* false
+ qp.pivot/*check-pivot-parity?* false]
+ (is (=? {:status :completed}
+ (mt/as-admin
+ (qp.card/process-query-for-card
+ card :api
+ :make-run (fn [qp _export-format]
+ (fn [query info]
+ (qp (update query :info merge info) nil))))))))
+ (is (some? (queries/average-execution-time-ms card-query-hash)))
+ (is (=? {:type :ttl, :avg-execution-ms (queries/average-execution-time-ms card-query-hash)}
+ (:cache-strategy
+ (qp.card/query-for-card card nil (qp.constraints/default-query-constraints) nil)))))))))))))
+
(deftest nested-query-permissions-test
(testing "Reading a Card is not enough to run it when its source query is a Card we cannot read"
(mt/with-no-data-perms-for-all-users!