The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Prevent serving overly stale query cache results (#78689)
Ben Grabowfc9441add09a
diff --git a/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj b/enterprise/backend/test/metabase_enterprise/cache/strategies_test.cljindex fa5da52c669..2c35bb9731d 100644--- a/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj@@ -17,72 +17,76 @@(deftest caching-strategies(mt/with-empty-h2-app-db!- (mt/with-premium-features #{:cache-granular-controls}- (let [query (mt/mbql-query venues {:order-by [[:asc $id]] :limit 5})- mkres (fn [input]- {:cache/details (if input- {:cached true, :updated_at input, :hash some?}- {:stored true, :hash some?})- :row_count 5- :status :completed})]- (mt/with-model-cleanup [[:model/QueryCache :updated_at]]- (testing "strategy = ttl"- (let [query (assoc query :cache-strategy {:type :ttl- :multiplier 10- :min-duration-ms 0- :avg-execution-ms 500})]- (testing "Results are stored and available immediately"- (mt/with-clock #t "2024-02-13T10:00:00Z"- (is (=? (mkres nil)- (-> (qp/process-query query) (dissoc :data))))- (is (=? (mkres #t "2024-02-13T10:00:00Z")- (-> (qp/process-query query) (dissoc :data))))))- (testing "4 seconds past that results are still there - 10 * 500 = 5 seconds"- (mt/with-clock #t "2024-02-13T10:00:04Z"- (is (=? (mkres #t "2024-02-13T10:00:00Z")- (-> (qp/process-query query) (dissoc :data))))))- (testing "6 seconds later results are unavailable"- (mt/with-clock #t "2024-02-13T10:00:06Z"- (is (=? (mkres nil)- (-> (qp/process-query query) (dissoc :data)))))))))- (mt/with-model-cleanup [[:model/QueryCache :updated_at]]- (testing "strategy = duration"- (let [query (assoc query :cache-strategy {:type :duration- :duration 1- :unit "minutes"- :min-duration-ms 0})]- (testing "Results are stored and available immediately"- (mt/with-clock #t "2024-02-13T10:00:00Z"- (is (=? (mkres nil)- (-> (qp/process-query query) (dissoc :data))))- (is (=? (mkres #t "2024-02-13T10:00:00Z")- (-> (qp/process-query query) (dissoc :data)))))- (mt/with-clock #t "2024-02-13T10:00:59Z"- (is (=? (mkres #t "2024-02-13T10:00:00Z")- (-> (qp/process-query query) (dissoc :data)))))- (mt/with-clock #t "2024-02-13T10:01:01Z"- (is (=? (mkres nil)- (-> (qp/process-query query) (dissoc :data)))))))))- (mt/with-model-cleanup [[:model/QueryCache :updated_at]]- (testing "strategy = schedule"- (let [query (assoc query :cache-strategy {:type :schedule- :schedule "0/2 * * * *"- :invalidated-at (t/offset-date-time #t "2024-02-13T10:00:00Z")})]- (testing "Results are stored and available immediately"- (mt/with-clock #t "2024-02-13T10:01:00Z"- (is (=? (mkres nil)- (-> (qp/process-query query) (dissoc :data))))- (is (=? (mkres #t "2024-02-13T10:01:00Z")- (-> (qp/process-query query) (dissoc :data))))))- (let [query (assoc-in query [:cache-strategy :invalidated-at] (t/offset-date-time #t "2024-02-13T10:02:00Z"))]- (testing "Cache is invalidated when schedule ran after the query"- (mt/with-clock #t "2024-02-13T10:03:00Z"+ ;; these assertions are about where each strategy puts its freshness boundary, so early refreshing is turned off:+ ;; with it on, the tail of each window recomputes instead of serving, which is covered in+ ;; `metabase.query-processor.middleware.cache-test/early-refresh-test`+ (mt/with-temporary-setting-values [query-caching-early-refresh-ratio 0.0]+ (mt/with-premium-features #{:cache-granular-controls}+ (let [query (mt/mbql-query venues {:order-by [[:asc $id]] :limit 5})+ mkres (fn [input]+ {:cache/details (if input+ {:cached true, :updated_at input, :hash some?}+ {:stored true, :hash some?})+ :row_count 5+ :status :completed})]+ (mt/with-model-cleanup [[:model/QueryCache :updated_at]]+ (testing "strategy = ttl"+ (let [query (assoc query :cache-strategy {:type :ttl+ :multiplier 10+ :min-duration-ms 0+ :avg-execution-ms 500})]+ (testing "Results are stored and available immediately"+ (mt/with-clock #t "2024-02-13T10:00:00Z"+ (is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data))))+ (is (=? (mkres #t "2024-02-13T10:00:00Z")+ (-> (qp/process-query query) (dissoc :data))))))+ (testing "4 seconds past that results are still there - 10 * 500 = 5 seconds"+ (mt/with-clock #t "2024-02-13T10:00:04Z"+ (is (=? (mkres #t "2024-02-13T10:00:00Z")+ (-> (qp/process-query query) (dissoc :data))))))+ (testing "6 seconds later results are unavailable"+ (mt/with-clock #t "2024-02-13T10:00:06Z"(is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data)))))))))+ (mt/with-model-cleanup [[:model/QueryCache :updated_at]]+ (testing "strategy = duration"+ (let [query (assoc query :cache-strategy {:type :duration+ :duration 1+ :unit "minutes"+ :min-duration-ms 0})]+ (testing "Results are stored and available immediately"+ (mt/with-clock #t "2024-02-13T10:00:00Z"+ (is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data))))+ (is (=? (mkres #t "2024-02-13T10:00:00Z")+ (-> (qp/process-query query) (dissoc :data)))))+ (mt/with-clock #t "2024-02-13T10:00:59Z"+ (is (=? (mkres #t "2024-02-13T10:00:00Z")+ (-> (qp/process-query query) (dissoc :data)))))+ (mt/with-clock #t "2024-02-13T10:01:01Z"+ (is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data)))))))))+ (mt/with-model-cleanup [[:model/QueryCache :updated_at]]+ (testing "strategy = schedule"+ (let [query (assoc query :cache-strategy {:type :schedule+ :schedule "0/2 * * * *"+ :invalidated-at (t/offset-date-time #t "2024-02-13T10:00:00Z")})]+ (testing "Results are stored and available immediately"+ (mt/with-clock #t "2024-02-13T10:01:00Z"+ (is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data))))+ (is (=? (mkres #t "2024-02-13T10:01:00Z")(-> (qp/process-query query) (dissoc :data))))))- (testing "schedule did not run - cache is still intact"- (mt/with-clock #t "2024-02-13T10:08:00Z"- (is (=? (mkres #t "2024-02-13T10:03:00Z")- (-> (qp/process-query query) (dissoc :data))))))))))))))+ (let [query (assoc-in query [:cache-strategy :invalidated-at] (t/offset-date-time #t "2024-02-13T10:02:00Z"))]+ (testing "Cache is invalidated when schedule ran after the query"+ (mt/with-clock #t "2024-02-13T10:03:00Z"+ (is (=? (mkres nil)+ (-> (qp/process-query query) (dissoc :data))))))+ (testing "schedule did not run - cache is still intact"+ (mt/with-clock #t "2024-02-13T10:08:00Z"+ (is (=? (mkres #t "2024-02-13T10:03:00Z")+ (-> (qp/process-query query) (dissoc :data)))))))))))))))(deftest e2e-advanced-caching(binding [search.ingestion/*force-sync* truediff --git a/src/metabase/cache/core.clj b/src/metabase/cache/core.cljindex e44dce512ae..bd3c5e19025 100644--- a/src/metabase/cache/core.clj+++ b/src/metabase/cache/core.clj@@ -16,5 +16,6 @@root-strategy][metabase.cache.settingsenable-query-caching+ query-caching-early-refresh-ratioquery-caching-max-kbquery-caching-max-ttl])diff --git a/src/metabase/cache/settings.clj b/src/metabase/cache/settings.cljindex 8cbb8692391..10b20763b92 100644--- a/src/metabase/cache/settings.clj+++ b/src/metabase/cache/settings.clj@@ -42,6 +42,17 @@global-max-caching-kb (u/format-bytes (* global-max-caching-kb 1024)))))))(setting/set-value-of-type! :integer :query-caching-max-kb new-value)))+(defsetting query-caching-early-refresh-ratio+ (deferred-tru "Refresh cached results this fraction of their cache duration before they expire, so requests keep being served from cache instead of waiting for a recomputation. Set to 0 to only refresh once results have expired.")+ ;; Expressed as a fraction of each entry's own cache duration rather than an absolute time so that it scales with+ ;; the configured duration: a flat window longer than a short duration would refresh on every request. Values+ ;; outside [0, 1) are allowed and behave sensibly at the limits -- 0 or less refreshes nothing early, 1 or more+ ;; refreshes on every request that can take the lease, so cached results stop being reused.+ :type :double+ :default 0.1+ :export? false+ :audit :getter)+(defsetting query-caching-max-ttl(deferred-tru "The absolute maximum time to keep any cached query results, in seconds."):type :doublediff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex db043434bf9..d45afbaa4ba 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -213,14 +213,63 @@(boolean (and updated-at(not (t/before? (t/instant updated-at) (t/instant invalidated-at))))))+(defn- fresh-duration-ms+ "How long `strategy` keeps an entry fresh, in milliseconds, or nil when that isn't a fixed length of time."+ [strategy]+ (case (:type strategy)+ :ttl (when-let [avg-execution-ms (:avg-execution-ms strategy)]+ (long (* (:multiplier strategy) avg-execution-ms)))+ :duration (when (and (:duration strategy) (:unit strategy))+ (t/as (t/duration (:duration strategy) (keyword (:unit strategy))) :millis))+ nil))++(defn- due-for-early-refresh?+ "Whether an entry last written at `updated-at` is close enough to expiring - inside the last+ [[cache/query-caching-early-refresh-ratio]] of its `window-ms` freshness window - that it should be refreshed now,+ while it can still be served.++ Refreshing only once an entry has expired means somebody always eats the recomputation, and every other request in+ that moment is served an expired entry or waits. Starting one window-fraction early means the refresh usually lands+ before anything goes stale. Exactly one request per window pays for it: the rest lose the lease and are served the+ entry, which is still fresh.++ A fraction rather than a fixed lead time so it scales with each strategy's own duration: a lead time longer than+ the duration would put an entry in the refresh zone the moment it was written. Returns false when `window-ms` is+ nil, since a strategy whose boundary doesn't slide (`:schedule`) has no \"about to expire\"."+ [updated-at invalidated-at window-ms early-refresh-ratio]+ (boolean (and updated-at+ window-ms+ ;; how much of the window is left: the boundary slides with the clock, so the distance from it to+ ;; `updated-at` is the time until this entry falls behind it+ (let [remaining-ms (.toMillis (t/duration (t/instant invalidated-at) (t/instant updated-at)))]+ (< remaining-ms (* early-refresh-ratio window-ms))))))++(defn- not-too-stale?+ "Whether an entry last written at `updated-at` is close enough to `invalidated-at` (the strategy's freshness+ boundary) that serving it is still serving roughly what the caller asked for -- within+ [[*refresh-lease-duration-ms*]] of it.++ Serving an expired entry is only defensible while its replacement is on the way, and that is the one thing an+ expired entry doesn't tell you: it can sit untouched for weeks until the first request arrives, wins the lease, and+ starts refreshing, while every other request in that batch loses the lease. Without a bound those losers are served+ results from an arbitrarily older window -- a monthly subscription batch gets last month's numbers (#78339). The+ lease duration is the bound because it is how long a refresh may legitimately be in flight; past it, the entry is+ old enough that recomputing beats answering with it."+ [updated-at invalidated-at]+ (boolean (and updated-at+ (not (t/before? (t/instant updated-at)+ (t/minus (t/instant invalidated-at) (t/millis *refresh-lease-duration-ms*)))))))+(mu/defn- maybe-serve-cached-results :- [:tuple#_status [:enum ::fresh ::stale ::miss ::canceled]#_result :any]"Look up the cache entry for `query-hash` and decide what to do (stale-while-revalidate):- `[::fresh result]` -- entry is within its TTL; serve it.- - `[::stale result]` -- entry is expired but another process holds the refresh lease; serve it stale while that- process refreshes, so we don't stampede the data warehouse.- - `[::miss nil]` -- no entry, or it's expired and *this* process won the lease; the caller must recompute.+ - `[::stale result]` -- entry is expired but not too stale to stand in for a fresh one, and another process holds+ the refresh lease; serve it while that process refreshes, so we don't stampede the data+ warehouse.+ - `[::miss nil]` -- no entry; or it's expired, or nearly so, and *this* process won the lease; or it's too+ stale to serve to anyone. The caller must recompute.- `[::canceled nil]` -- the request was canceled."[ignore-cache?query-hash :- bytes?@@ -229,29 +278,49 @@(if ignore-cache?[::miss nil](try- (or (i/with-cached-results *backend* query-hash [is updated-at]- (when is- (let [invalidated-at (backend.db/strategy->invalidated-at strategy)]- (cond- ;; can't determine freshness for this strategy -> don't serve from cache- (nil? invalidated-at)- nil-- ;; within its TTL -> serve the fresh entry- (cache-fresh? updated-at invalidated-at)- (when-let [result (reduce-cached-stream is rff query-hash)]- [::fresh result])-- ;; expired, and we won the refresh lease -> recompute (don't serve stale)- (i/try-acquire-refresh-lease! *backend* query-hash *refresh-lease-duration-ms*)- nil-- ;; expired, another process is refreshing -> serve stale- :else- (when-let [result (reduce-cached-stream is rff query-hash)]- (log/debugf "Serving stale cached results for hash '%s' while another process refreshes"- (i/short-hex-hash query-hash))- [::stale result])))))+ (or (i/cached-results+ *backend*+ query-hash+ (fn [is updated-at]+ (when is+ (let [invalidated-at (backend.db/strategy->invalidated-at strategy)]+ (cond+ ;; can't determine freshness for this strategy -> don't serve from cache+ (nil? invalidated-at)+ nil++ ;; still fresh, but about to expire, and we won the lease -> refresh it now+ ;; so it doesn't lapse into staleness for whoever asks next+ (and (cache-fresh? updated-at invalidated-at)+ (due-for-early-refresh?+ updated-at+ invalidated-at+ (fresh-duration-ms strategy)+ (cache/query-caching-early-refresh-ratio))+ (i/try-acquire-refresh-lease! *backend* query-hash *refresh-lease-duration-ms*))+ nil++ ;; within its TTL -> serve the fresh entry+ (cache-fresh? updated-at invalidated-at)+ (when-let [result (reduce-cached-stream is rff query-hash)]+ [::fresh result])++ ;; expired, and we won the refresh lease -> recompute (don't serve stale)+ (i/try-acquire-refresh-lease! *backend* query-hash *refresh-lease-duration-ms*)+ nil++ ;; another process is refreshing, and the entry is still close enough to what+ ;; was asked for -> serve it stale rather than stampede the warehouse+ (not-too-stale? updated-at invalidated-at)+ (when-let [result (reduce-cached-stream is rff query-hash)]+ (log/debugf "Serving stale cached results written at %s for hash '%s' while another process refreshes"+ updated-at (i/short-hex-hash query-hash))+ [::stale result])++ ;; the entry is too far past its window to answer with, whoever holds the+ ;; lease -> recompute (#78339)+ :else+ nil)))))[::miss nil])(catch EofException _(log/debug "Request is closed; no one to return cached results to")diff --git a/src/metabase/query_processor/middleware/cache_backend/interface.clj b/src/metabase/query_processor/middleware/cache_backend/interface.cljindex 401a5f6a8b2..096ed34e721 100644--- a/src/metabase/query_processor/middleware/cache_backend/interface.clj+++ b/src/metabase/query_processor/middleware/cache_backend/interface.clj@@ -48,20 +48,6 @@abandoned and may be taken over. Backends that can't coordinate across processes should return `true` (degrading tothe no-stampede-protection behavior)."))-(defmacro with-cached-results- "Macro version for consuming `cached-results` from a `backend`.-- (with-cached-results backend query-hash [is updated-at]- ...)-- InputStream `is` (and `updated-at`) will be `nil` if there is no cache entry at all. `is` is only valid within- `body`."- {:style/indent 3}- [backend query-hash [is-binding updated-at-binding] & body]- `(cached-results ~backend ~query-hash- (fn [~(vary-meta is-binding assoc :tag 'java.io.InputStream) ~updated-at-binding]- ~@body)))-(defmulti cache-backend"Return an instance of a cache backend, which is any object that implements `QueryProcessorCacheBackend`.diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 4168eec202f..bd9fdfc9299 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -341,6 +341,82 @@(is (= :cached(run-query :cache-strategy strategy)))))))+(deftest early-refresh-test+ ;; (ttl-strategy) TTL = :multiplier 60 * :avg-execution-ms 1000 = 60 seconds, so the default 10% ratio puts the+ ;; refresh zone in the last 6 seconds+ (let [t0 #t "2026-01-01T00:00:00Z[UTC]"]+ (testing "an entry with most of its window left is served, not refreshed"+ (with-mock-cache! [save-chan]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (mt/with-clock (t/plus t0 (t/seconds 30))+ (is (= :cached+ (run-query))))))+ (testing "an entry inside the last tenth of its window is refreshed before it can expire"+ (with-mock-cache! [save-chan]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (mt/with-clock (t/plus t0 (t/seconds 57))+ (is (= :not-cached+ (run-query))+ "the request that wins the lease recomputes rather than serving an entry about to expire")+ (mt/wait-for-result save-chan))))+ (testing "a request that loses the early-refresh lease is still served the (fresh) entry"+ (with-mock-cache! [save-chan]+ (let [query-hash (qp.util/query-hash (test-query nil))]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (mt/with-clock (t/plus t0 (t/seconds 57))+ (is (true? (i/try-acquire-refresh-lease! cache/*backend* query-hash (u/minutes->ms 5))))+ (is (= :cached+ (run-query))+ "only the lease winner pays for the early refresh")))))+ (testing "early refreshing is off when the ratio is 0"+ (with-mock-cache! [save-chan]+ (mt/with-temporary-setting-values [query-caching-early-refresh-ratio 0.0]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (mt/with-clock (t/plus t0 (t/seconds 57))+ (is (= :cached+ (run-query)))))))))++(deftest stale-while-revalidate-staleness-is-bounded-test+ (testing "a lease loser is not served an entry that expired long before the refresh holding the lease started; past+ that bound it recomputes instead (#78339)"+ (with-mock-cache! [save-chan]+ ;; (ttl-strategy) TTL = :multiplier 60 * :avg-execution-ms 1000 = 60 seconds+ (let [t0 #t "2026-01-01T00:00:00Z[UTC]"+ query-hash (qp.util/query-hash (test-query nil))]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (testing "the entry expired 30 days ago and another process just claimed the refresh lease"+ (mt/with-clock (t/plus t0 (t/days 30))+ (is (true? (i/try-acquire-refresh-lease! cache/*backend* query-hash (u/minutes->ms 5))))+ (is (= :not-cached+ (run-query))+ "a 30-day-old entry is too stale to serve to the lease loser")))))))++(deftest stale-while-revalidate-within-lease-window-test+ (testing "a lease loser is still served an entry that only just expired, so concurrent requests don't stampede the+ warehouse while a refresh is in flight"+ (with-mock-cache! [save-chan]+ (let [t0 #t "2026-01-01T00:00:00Z[UTC]"+ query-hash (qp.util/query-hash (test-query nil))]+ (mt/with-clock t0+ (run-query)+ (mt/wait-for-result save-chan))+ (testing "the entry expired 1 second ago and another process holds the refresh lease"+ (mt/with-clock (t/plus t0 (t/seconds 61))+ (is (true? (i/try-acquire-refresh-lease! cache/*backend* query-hash (u/minutes->ms 5))))+ (is (= :cached+ (run-query))+ "just-expired results are fine to serve while the refresh is in flight")))))))+(deftest ignore-valid-results-when-caching-is-disabled-test(testing "if caching is disabled then cache shouldn't be used even if there's something valid in there"(with-mock-cache! [save-chan]