The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Cache empty query results (#75946)
Alexander Polyankin6bc02506d79c
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex 9e5f1e168a2..81d6803b246 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -113,35 +113,34 @@(log/errorf e "Error saving query results to cache: %s" (ex-message e))))))(defn- save-results-xform [start-time-ns metadata query-hash strategy rf]- (let [has-rows? (volatile! false)]- (add-object-to-cache! (assoc metadata- :cache-version cache-version- :last-ran (t/zoned-date-time)))- (fn- ([] (rf))-- ([result]- (add-object-to-cache! (if (map? result)- (m/dissoc-in result [:data :rows])- {}))- (let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)- min-duration-ms (:min-duration-ms strategy 0)- eligible? (and @has-rows?- (> duration-ms min-duration-ms))]- (log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"- (i/short-hex-hash query-hash)- (u/format-milliseconds duration-ms)- (u/format-milliseconds min-duration-ms)- (if eligible? "eligible" "not eligible"))- (when eligible?- (cache-results! query-hash))- (rf (cond-> result- (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))-- ([acc row]- (add-object-to-cache! row)- (vreset! has-rows? true)- (rf acc row)))))+ (add-object-to-cache! (assoc metadata+ :cache-version cache-version+ :last-ran (t/zoned-date-time)))+ (fn+ ([] (rf))++ ([result]+ (add-object-to-cache! (if (map? result)+ (m/dissoc-in result [:data :rows])+ {}))+ (let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)+ min-duration-ms (:min-duration-ms strategy 0)+ ;; cache any query that ran long enough -- including ones that returned no rows, so a slow empty result+ ;; doesn't get re-run at full cost on every request+ eligible? (> duration-ms min-duration-ms)]+ (log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"+ (i/short-hex-hash query-hash)+ (u/format-milliseconds duration-ms)+ (u/format-milliseconds min-duration-ms)+ (if eligible? "eligible" "not eligible"))+ (when eligible?+ (cache-results! query-hash))+ (rf (cond-> result+ (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))++ ([acc row]+ (add-object-to-cache! row)+ (rf acc row))));;; ----------------------------------------------------- Fetch ------------------------------------------------------diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 213b05a2d90..a053db53e3c 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -143,20 +143,24 @@:stages [{:lib/type :mbql.stage/mbql, :source-table 2, :abc :def}]}query-kvs))+(def ^:private ^:dynamic *rows*+ "Rows the mock query execution returns. Bind to `[]` to exercise the empty-result path."+ [[:toucan 71]+ [:bald-eagle 92]+ [:hummingbird 11]+ [:owl 10]+ [:chicken 69]+ [:robin 96]+ [:osprey 72]+ [:flamingo 70]])+(defn- run-query* [& {:as query-kvs}];; clear out stale values in save/purge channels(while (a/poll! *save-chan*))(while (a/poll! *purge-chan*))(let [qp (cache/maybe-return-cached-results qp.pipeline/*run*)metadata {}- rows [[:toucan 71]- [:bald-eagle 92]- [:hummingbird 11]- [:owl 10]- [:chicken 69]- [:robin 96]- [:osprey 72]- [:flamingo 70]]+ rows *rows*query (test-query query-kvs)](binding [driver.settings/*query-timeout-ms* 2000qp.pipeline/*execute* (fn [_driver _query respond]@@ -203,6 +207,16 @@(is (= :cached(run-query))))))+(deftest cache-empty-results-test+ (testing "a query that returns no rows is still cached, so a slow empty result isn't re-run at full cost every time"+ (with-mock-cache! [save-chan]+ (binding [*rows* []]+ (is (= :not-cached+ (run-query)))+ (mt/wait-for-result save-chan)+ (is (= :cached+ (run-query)))))))+(deftest expired-results-test(testing "If cached resutls are past their TTL, the cached results shouldn't be returned"(with-mock-cache! [save-chan]