Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Cache empty query results (#75946)

Alexander Polyankin6bc02506d79c
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.clj
index 9e5f1e168a2..81d6803b246 100644
--- a/src/metabase/query_processor/middleware/cache.clj
+++ b/src/metabase/query_processor/middleware/cache.clj
@@ -113,35 +113,34 @@
(log/errorf e "Error saving query results to cache: %s" (ex-message e))))))
(defn- save-results-xform [start-time-ns metadata query-hash strategy rf]
- (let [has-rows? (volatile! false)]
- (add-object-to-cache! (assoc metadata
- :cache-version cache-version
- :last-ran (t/zoned-date-time)))
- (fn
- ([] (rf))
-
- ([result]
- (add-object-to-cache! (if (map? result)
- (m/dissoc-in result [:data :rows])
- {}))
- (let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)
- min-duration-ms (:min-duration-ms strategy 0)
- eligible? (and @has-rows?
- (> duration-ms min-duration-ms))]
- (log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"
- (i/short-hex-hash query-hash)
- (u/format-milliseconds duration-ms)
- (u/format-milliseconds min-duration-ms)
- (if eligible? "eligible" "not eligible"))
- (when eligible?
- (cache-results! query-hash))
- (rf (cond-> result
- (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))
-
- ([acc row]
- (add-object-to-cache! row)
- (vreset! has-rows? true)
- (rf acc row)))))
+ (add-object-to-cache! (assoc metadata
+ :cache-version cache-version
+ :last-ran (t/zoned-date-time)))
+ (fn
+ ([] (rf))
+
+ ([result]
+ (add-object-to-cache! (if (map? result)
+ (m/dissoc-in result [:data :rows])
+ {}))
+ (let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)
+ min-duration-ms (:min-duration-ms strategy 0)
+ ;; cache any query that ran long enough -- including ones that returned no rows, so a slow empty result
+ ;; doesn't get re-run at full cost on every request
+ eligible? (> duration-ms min-duration-ms)]
+ (log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"
+ (i/short-hex-hash query-hash)
+ (u/format-milliseconds duration-ms)
+ (u/format-milliseconds min-duration-ms)
+ (if eligible? "eligible" "not eligible"))
+ (when eligible?
+ (cache-results! query-hash))
+ (rf (cond-> result
+ (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))
+
+ ([acc row]
+ (add-object-to-cache! row)
+ (rf acc row))))
;;; ----------------------------------------------------- Fetch ------------------------------------------------------
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index 213b05a2d90..a053db53e3c 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -143,20 +143,24 @@
:stages [{:lib/type :mbql.stage/mbql, :source-table 2, :abc :def}]}
query-kvs))
+(def ^:private ^:dynamic *rows*
+ "Rows the mock query execution returns. Bind to `[]` to exercise the empty-result path."
+ [[:toucan 71]
+ [:bald-eagle 92]
+ [:hummingbird 11]
+ [:owl 10]
+ [:chicken 69]
+ [:robin 96]
+ [:osprey 72]
+ [:flamingo 70]])
+
(defn- run-query* [& {:as query-kvs}]
;; clear out stale values in save/purge channels
(while (a/poll! *save-chan*))
(while (a/poll! *purge-chan*))
(let [qp (cache/maybe-return-cached-results qp.pipeline/*run*)
metadata {}
- rows [[:toucan 71]
- [:bald-eagle 92]
- [:hummingbird 11]
- [:owl 10]
- [:chicken 69]
- [:robin 96]
- [:osprey 72]
- [:flamingo 70]]
+ rows *rows*
query (test-query query-kvs)]
(binding [driver.settings/*query-timeout-ms* 2000
qp.pipeline/*execute* (fn [_driver _query respond]
@@ -203,6 +207,16 @@
(is (= :cached
(run-query))))))
+(deftest cache-empty-results-test
+ (testing "a query that returns no rows is still cached, so a slow empty result isn't re-run at full cost every time"
+ (with-mock-cache! [save-chan]
+ (binding [*rows* []]
+ (is (= :not-cached
+ (run-query)))
+ (mt/wait-for-result save-chan)
+ (is (= :cached
+ (run-query)))))))
+
(deftest expired-results-test
(testing "If cached resutls are past their TTL, the cached results shouldn't be returned"
(with-mock-cache! [save-chan]