Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Introduce multi-process stale-while-revalidate behavior with query caching (#75949)

Alexander Polyankin32c871a46eea
diff --git a/enterprise/backend/src/metabase_enterprise/cache/strategies.clj b/enterprise/backend/src/metabase_enterprise/cache/strategies.clj
index 0ea13f37525..aeff6cab2b7 100644
--- a/enterprise/backend/src/metabase_enterprise/cache/strategies.clj
+++ b/enterprise/backend/src/metabase_enterprise/cache/strategies.clj
@@ -63,32 +63,33 @@
;;; Strategy execution
-(defmulti ^:private fetch-cache-stmt-ee*
- "Generate prepared statement for a db cache backend for a given strategy"
- {:arglists '([strategy query-hash])}
- (fn [strategy _hash] (:type strategy)))
+(defmulti ^:private strategy->invalidated-at*
+ "Freshness boundary for a given strategy: cache entries older than this are stale."
+ {:arglists '([strategy])}
+ :type)
-(defmethod fetch-cache-stmt-ee* :ttl [strategy query-hash]
- (backend.db/fetch-cache-stmt-ttl strategy query-hash))
+(defmethod strategy->invalidated-at* :ttl [strategy]
+ (backend.db/invalidated-at-ttl strategy))
-(defmethod fetch-cache-stmt-ee* :duration [strategy query-hash]
+(defmethod strategy->invalidated-at* :duration [strategy]
(if-not (and (:duration strategy) (:unit strategy))
(log/debugf "Caching strategy %s should have :duration and :unit" (pr-str strategy))
- (let [duration (t/duration (:duration strategy) (keyword (:unit strategy)))
- duration-ago (t/minus (t/offset-date-time) duration)
- invalidated-at (t/max duration-ago (:invalidated-at strategy))]
- (backend.db/select-cache query-hash invalidated-at))))
+ (let [duration (t/duration (:duration strategy) (keyword (:unit strategy)))
+ duration-ago (t/minus (t/offset-date-time) duration)]
+ (if-let [invalidated-at (:invalidated-at strategy)]
+ (t/max duration-ago invalidated-at)
+ duration-ago))))
-(defmethod fetch-cache-stmt-ee* :schedule [{:keys [invalidated-at] :as strategy} query-hash]
+(defmethod strategy->invalidated-at* :schedule [{:keys [invalidated-at] :as strategy}]
(if-not invalidated-at
(log/debugf "Caching strategy %s has not run yet" (pr-str strategy))
- (backend.db/select-cache query-hash invalidated-at)))
+ invalidated-at))
-(defmethod fetch-cache-stmt-ee* :nocache [_ _ _]
+(defmethod strategy->invalidated-at* :nocache [_]
nil)
-(defenterprise fetch-cache-stmt
- "Returns prepared statement to query for db cache backend."
+(defenterprise strategy->invalidated-at
+ "Freshness boundary timestamp for `strategy`: cache entries with `updated_at` older than this are stale."
:feature :cache-granular-controls
- [strategy query-hash]
- (fetch-cache-stmt-ee* strategy query-hash))
+ [strategy]
+ (strategy->invalidated-at* strategy))
diff --git a/resources/migrations/060/20260616_query_cache_refresh_lease.yaml b/resources/migrations/060/20260616_query_cache_refresh_lease.yaml
new file mode 100644
index 00000000000..65bbed7d84c
--- /dev/null
+++ b/resources/migrations/060/20260616_query_cache_refresh_lease.yaml
@@ -0,0 +1,27 @@
+## Add query_cache.refresh_started_at to coordinate a single cross-process refresh of an expired entry
+## (stale-while-revalidate). One process claims the refresh lease via a conditional UPDATE; the others
+## serve the stale cached result instead of all stampeding the data warehouse.
+##
+## Lives in 060 (the oldest backported release) so the same changeSet is shared across all versions >= 60.
+
+databaseChangeLog:
+ - logicalFilePath: migrations/060_update_migrations.yaml
+
+ - changeSet:
+ id: v60.query-cache-refresh-lease
+ author: ranquild
+ comment: Add query_cache.refresh_started_at as a cross-process refresh lease for stale-while-revalidate caching
+ changes:
+ - addColumn:
+ tableName: query_cache
+ columns:
+ - column:
+ name: refresh_started_at
+ type: ${timestamp_type}
+ remarks: When a process claimed the lease to recompute this expired entry; NULL when no refresh is in progress. Used to elect a single refresher across processes so the others serve stale results.
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: query_cache
+ columnName: refresh_started_at
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.clj
index 81d6803b246..e5fb836cfc6 100644
--- a/src/metabase/query_processor/middleware/cache.clj
+++ b/src/metabase/query_processor/middleware/cache.clj
@@ -180,44 +180,102 @@
acc
(rf acc row))))))))
-(mu/defn- maybe-reduce-cached-results :- [:tuple
- #_status
- [:enum ::ok ::miss ::canceled]
- #_result
- :any]
- "Reduces cached results if there is a hit. Otherwise, returns `::miss` directly."
+(defn- reduce-cached-stream
+ "Deserialize and reduce a cached-results `InputStream` `is` with `rff`. Returns the reduced result, or nil if the
+ stream is absent or was written by an incompatible cache version."
+ [is rff query-hash]
+ (when is
+ (impl/with-reducible-deserialized-results [[metadata reducible-rows] is]
+ (log/debugf "Found cached results for hash '%s'. Version: %s"
+ (i/short-hex-hash query-hash) (pr-str (:cache-version metadata)))
+ (when (and (= (:cache-version metadata) cache-version)
+ reducible-rows)
+ (log/trace "Reducing cached rows...")
+ (u/prog1 (qp.pipeline/*reduce* (cached-results-rff rff query-hash) metadata reducible-rows)
+ (log/trace "All cached rows reduced"))))))
+
+(def ^:dynamic *refresh-lease-duration-ms*
+ "How long a claimed stale-while-revalidate refresh lease is honored before another process may take it over (e.g. if
+ the claiming process crashed mid-refresh). Should comfortably exceed a normal query's run time."
+ (u/minutes->ms 5))
+
+(defn- cache-fresh?
+ "Whether a cache entry last written at `updated-at` is still within its TTL given `invalidated-at` (the strategy's
+ freshness boundary, which must be non-nil)."
+ [updated-at invalidated-at]
+ (boolean (and updated-at
+ (not (t/before? (t/instant updated-at) (t/instant invalidated-at))))))
+
+(mu/defn- maybe-serve-cached-results :- [:tuple
+ #_status [:enum ::fresh ::stale ::miss ::canceled]
+ #_result :any]
+ "Look up the cache entry for `query-hash` and decide what to do (stale-while-revalidate):
+ - `[::fresh result]` -- entry is within its TTL; serve it.
+ - `[::stale result]` -- entry is expired but another process holds the refresh lease; serve it stale while that
+ process refreshes, so we don't stampede the data warehouse.
+ - `[::miss nil]` -- no entry, or it's expired and *this* process won the lease; the caller must recompute.
+ - `[::canceled nil]` -- the request was canceled."
[ignore-cache?
- query-hash :- bytes?
- strategy :- :map
- rff :- ::qp.schema/rff]
- (try
- (or (when-not ignore-cache?
- (log/debugf "Looking for cached results for query with hash '%s' satisfying %s"
- (i/short-hex-hash query-hash) (pr-str strategy))
- (i/with-cached-results *backend* query-hash strategy [is]
- (if is
- (impl/with-reducible-deserialized-results [[metadata reducible-rows] is]
- (log/debugf "Found cached results for hash '%s'. Version: %s"
- (i/short-hex-hash query-hash) (pr-str (:cache-version metadata)))
- (when (and (= (:cache-version metadata) cache-version)
- reducible-rows)
- (log/trace "Reducing cached rows...")
- (let [result (qp.pipeline/*reduce* (cached-results-rff rff query-hash) metadata reducible-rows)]
- (log/trace "All cached rows reduced")
- [::ok result])))
- (log/debugf "Not found cached results for hash '%s'" (i/short-hex-hash query-hash)))))
- [::miss nil])
- (catch EofException _
- (log/debug "Request is closed; no one to return cached results to")
- [::canceled nil])
- (catch Throwable e
- (log/errorf e "Error attempting to fetch cached results for query with hash %s: %s"
- (i/short-hex-hash query-hash)
- (ex-message e))
- [::miss nil])))
+ query-hash :- bytes?
+ strategy :- :map
+ rff :- ::qp.schema/rff]
+ (if ignore-cache?
+ [::miss nil]
+ (try
+ (or (i/with-cached-results *backend* query-hash [is updated-at]
+ (when is
+ (let [invalidated-at (backend.db/strategy->invalidated-at strategy)]
+ (cond
+ ;; can't determine freshness for this strategy -> don't serve from cache
+ (nil? invalidated-at)
+ nil
+
+ ;; within its TTL -> serve the fresh entry
+ (cache-fresh? updated-at invalidated-at)
+ (when-let [result (reduce-cached-stream is rff query-hash)]
+ [::fresh result])
+
+ ;; expired, and we won the refresh lease -> recompute (don't serve stale)
+ (i/try-acquire-refresh-lease! *backend* query-hash *refresh-lease-duration-ms*)
+ nil
+
+ ;; expired, another process is refreshing -> serve stale
+ :else
+ (when-let [result (reduce-cached-stream is rff query-hash)]
+ (log/debugf "Serving stale cached results for hash '%s' while another process refreshes"
+ (i/short-hex-hash query-hash))
+ [::stale result])))))
+ [::miss nil])
+ (catch EofException _
+ (log/debug "Request is closed; no one to return cached results to")
+ [::canceled nil])
+ (catch Throwable e
+ (log/errorf e "Error attempting to fetch cached results for query with hash %s: %s"
+ (i/short-hex-hash query-hash)
+ (ex-message e))
+ [::miss nil]))))
;;; --------------------------------------------------- Middleware ---------------------------------------------------
+(defn- run-and-cache!
+ "Run `query` through `qp` and save the results to the cache (if eligible). Used on a cache miss, or when this process
+ holds the stale-while-revalidate refresh lease."
+ [qp query query-hash cache-strategy rff]
+ (let [start-time-ns (System/nanoTime)
+ orig-reduce qp.pipeline/*reduce*]
+ (log/trace "Running query and saving cached results (if eligible)...")
+ (binding [qp.pipeline/*reduce* (fn reduce'
+ [rff metadata rows]
+ {:post [(some? %)]}
+ (impl/do-with-serialization
+ (fn [in-fn result-fn]
+ (binding [*in-fn* in-fn
+ *result-fn* result-fn]
+ (orig-reduce rff metadata rows)))))]
+ (qp query
+ (fn [metadata]
+ (save-results-xform start-time-ns metadata query-hash cache-strategy (rff metadata)))))))
+
(mu/defn- run-query-with-cache :- :some
[qp {:keys [cache-strategy middleware], :as query} :- ::qp.schema/any-query
rff :- ::qp.schema/rff]
@@ -225,29 +283,11 @@
;; after normalization, instead of before. This is necessary to make caching work properly with sandboxed users, see
;; #14388.
(let [query-hash (qp.util/query-hash query)
- [status result] (maybe-reduce-cached-results (:ignore-cached-results? middleware) query-hash cache-strategy rff)]
+ [status result] (maybe-serve-cached-results (:ignore-cached-results? middleware) query-hash cache-strategy rff)]
(case status
- ::ok
- result
-
- ::canceled
- ::canceled
-
- ::miss
- (let [start-time-ns (System/nanoTime)
- orig-reduce qp.pipeline/*reduce*]
- (log/trace "Running query and saving cached results (if eligible)...")
- (binding [qp.pipeline/*reduce* (fn reduce'
- [rff metadata rows]
- {:post [(some? %)]}
- (impl/do-with-serialization
- (fn [in-fn result-fn]
- (binding [*in-fn* in-fn
- *result-fn* result-fn]
- (orig-reduce rff metadata rows)))))]
- (qp query
- (fn [metadata]
- (save-results-xform start-time-ns metadata query-hash cache-strategy (rff metadata)))))))))
+ (::fresh ::stale) result
+ ::canceled ::canceled
+ ::miss (run-and-cache! qp query query-hash cache-strategy rff))))
(defn- has-cache-strategy? [cache-strategy]
(some? cache-strategy))
diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.clj
index c485c23111d..daa49327a98 100644
--- a/src/metabase/query_processor/middleware/cache_backend/db.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj
@@ -35,38 +35,64 @@
[{:keys [results]}]
results)
-(defn select-cache
- "Select the result form the cache"
- [query-hash updated-at]
- (t2/select-one-fn results-as-bytes :model/QueryCache
- {:select [:results]
- :where [:and
- [:= :query_hash query-hash]
- [:>= :updated_at updated-at]]
- :order-by [[:updated_at :desc]]}))
-
-(defn fetch-cache-stmt-ttl
- "Make a prepared statement for :ttl caching strategy"
- [strategy query-hash]
- (if-not (:avg-execution-ms strategy)
- (log/debugf "Caching strategy %s needs :avg-execution-ms to work" (pr-str strategy))
- (let [max-age-ms (* (:multiplier strategy)
- (:avg-execution-ms strategy))
- invalidated-at (t/max (ms-ago max-age-ms) (:invalidated-at strategy))]
- (select-cache query-hash invalidated-at))))
-
-(defenterprise fetch-cache-stmt
- "Returns prepared statement for a given strategy and query hash - on EE. Returns `::oss` on OSS."
+(defn select-latest-cache-entry
+ "The most recent cache entry for `query-hash` *regardless of TTL* (`query_hash` is the PK, so there's at most one):
+ `{:results <raw bytes>, :updated-at <timestamp>}`, or nil if there's no entry. The caller compares `:updated-at`
+ against the strategy's freshness boundary to decide whether to serve the entry, serve it stale while refreshing, or
+ recompute (see [[strategy->invalidated-at]]).
+
+ Reads the `results` blob *inside* the query reduction (via `select-one-fn`): an H2 `JdbcBlob` is only valid while
+ its result set is open, so materializing the row first and reading the blob afterwards throws \"object is already
+ closed\"."
+ [query-hash]
+ (t2/select-one-fn (fn [row]
+ {:results (results-as-bytes row)
+ :updated-at (:updated_at row)})
+ [:model/QueryCache :results :updated_at]
+ :query_hash query-hash))
+
+(defn invalidated-at-ttl
+ "Freshness boundary for a `:ttl` strategy: cache entries with `updated_at` older than this are stale. Returns nil when
+ the strategy is missing `:avg-execution-ms` (so a TTL can't be computed and nothing should be served from cache)."
+ [strategy]
+ (when-let [avg-execution-ms (:avg-execution-ms strategy)]
+ ;; `:multiplier` can be fractional, so round to whole milliseconds (`ms-ago`/`u.date/add` want an integer)
+ (let [boundary (ms-ago (long (* (:multiplier strategy) avg-execution-ms)))]
+ (if-let [invalidated-at (:invalidated-at strategy)]
+ (t/max boundary invalidated-at)
+ boundary))))
+
+(defenterprise strategy->invalidated-at
+ "Freshness boundary timestamp for `strategy`: cache entries with `updated_at` strictly older than this are stale and
+ must be refreshed. Returns nil when nothing can be served from cache for this strategy (e.g. on OSS for a non-`:ttl`
+ strategy). EE overrides this to also handle `:duration` and `:schedule`."
metabase-enterprise.cache.strategies
- [strategy hash]
+ [strategy]
(when (= :ttl (:type strategy))
- (fetch-cache-stmt-ttl strategy hash)))
+ (invalidated-at-ttl strategy)))
-(defn- cached-results [query-hash strategy respond]
- (if-let [cached (fetch-cache-stmt strategy query-hash)]
- (with-open [is (encryption/maybe-decrypt-stream (ByteArrayInputStream. cached))]
- (respond is))
- (respond nil)))
+(def ^:private lease-free-sentinel
+ "Substituted (via COALESCE) for a NULL `refresh_started_at` so that 'no refresh in progress' counts as a free lease.
+ Any timestamp older than every realistic lease cutoff works."
+ (t/offset-date-time "1970-01-01T00:00Z"))
+
+(defn try-acquire-refresh-lease!
+ "Atomically claim, across processes, the right to recompute the expired entry for `query-hash`, via a conditional
+ UPDATE on `refresh_started_at`. Returns true iff this process won the lease (and so should recompute); false means
+ another process is already refreshing it (and we should serve stale instead). A lease older than `lease-ms` is
+ considered abandoned (e.g. the claimer crashed) and can be taken over.
+
+ Updates the raw table, not the `:model/QueryCache` model: the model's `:hook/updated-at-timestamped?` before-update
+ hook makes toucan2 run a non-atomic SELECT-then-UPDATE-by-PK, which would move the conditional lease check into the
+ SELECT and let two processes both win. The raw table keeps this a single atomic conditional UPDATE; we set
+ `updated_at` ourselves in place of the hook."
+ [query-hash lease-ms]
+ (let [now (t/offset-date-time)]
+ (pos? (t2/update! (t2/table-name :model/QueryCache)
+ {:query_hash query-hash
+ [:coalesce :refresh_started_at lease-free-sentinel] [:< (ms-ago lease-ms)]}
+ {:refresh_started_at now
+ :updated_at now}))))
(defn- purge-old-cache-entries!
"Delete any cache entries that are older than the global max age `max-cache-entry-age-seconds` (currently 3 months)."
@@ -89,8 +115,9 @@
timestamp (t/offset-date-time)]
(try
(app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}
- (constantly {:updated_at timestamp
- :results final-results}))
+ (constantly {:updated_at timestamp
+ :results final-results
+ :refresh_started_at nil}))
(catch Throwable e
(log/error e "Error saving query results to cache.")))
nil))
@@ -98,12 +125,18 @@
(defmethod i/cache-backend :db
[_]
(reify i/CacheBackend
- (cached-results [_ query-hash strategy respond]
- (cached-results query-hash strategy respond))
+ (cached-results [_ query-hash respond]
+ (if-let [{:keys [results updated-at]} (select-latest-cache-entry query-hash)]
+ (with-open [is (encryption/maybe-decrypt-stream (ByteArrayInputStream. results))]
+ (respond is updated-at))
+ (respond nil nil)))
(save-results! [_ query-hash is]
(save-results! query-hash is)
nil)
(purge-old-entries! [_ max-age-seconds]
- (purge-old-cache-entries! max-age-seconds))))
+ (purge-old-cache-entries! max-age-seconds))
+
+ (try-acquire-refresh-lease! [_ query-hash lease-ms]
+ (try-acquire-refresh-lease! query-hash lease-ms))))
diff --git a/src/metabase/query_processor/middleware/cache_backend/interface.clj b/src/metabase/query_processor/middleware/cache_backend/interface.clj
index 54b462013c0..ae37b043bd6 100644
--- a/src/metabase/query_processor/middleware/cache_backend/interface.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/interface.clj
@@ -14,15 +14,18 @@
cache entry key. `results` are passed as a compressed byte array.
The implementation is responsible for purging old cache entries when appropriate."
- (cached-results [this ^bytes query-hash strategy respond]
- "Call `respond` with cached results for the query (as an `InputStream` to the raw bytes) if present and not
- expired; otherwise, call `respond` with `nil.
+ (cached-results [this ^bytes query-hash respond]
+ "Call `(respond is updated-at)` with the most recent cache entry for `query-hash` *regardless of TTL* -- the results
+ as an `InputStream` to the raw bytes, plus the entry's `updated-at` timestamp -- or `(respond nil nil)` if there's no
+ entry at all. The caller compares `updated-at` against the strategy's freshness boundary to decide whether to serve
+ the entry, serve it stale while a single process refreshes, or recompute. This method *must* return the result of
+ `respond`.
- (cached-results [_ hash _ respond]
- (with-open [is (...)]
- (respond is)))
-
- `strategy` should be a map with cache configuration. This method *must* return the result of `respond`.")
+ (cached-results [_ hash respond]
+ (if-let [entry (...)]
+ (with-open [is (...)]
+ (respond is (:updated-at entry)))
+ (respond nil nil)))")
(save-results! [this ^bytes query-hash ^bytes results]
"Add a cache entry with the `results` of running query with byte array `query-hash`. This should replace any prior
@@ -30,19 +33,28 @@
(purge-old-entries! [this max-age-seconds]
"Purge all cache entries older than `max-age-seconds`. Will be called periodically when this backend is in use.
- `max-age-seconds` may be floating-point."))
+ `max-age-seconds` may be floating-point.")
+
+ (try-acquire-refresh-lease! [this ^bytes query-hash lease-ms]
+ "Atomically claim, across processes, the right to recompute the expired entry for `query-hash`. Returns true iff
+ this caller won the lease (and should recompute + [[save-results!]]); false means another process is already
+ refreshing it, so this caller should serve stale results instead. A lease held longer than `lease-ms` is considered
+ abandoned and may be taken over. Backends that can't coordinate across processes should return `true` (degrading to
+ the no-stampede-protection behavior)."))
(defmacro with-cached-results
"Macro version for consuming `cached-results` from a `backend`.
- (with-cached-results backend query-hash strategy [is]
+ (with-cached-results backend query-hash [is updated-at]
...)
- InputStream `is` will be `nil` if no cached results were available."
- {:style/indent 4}
- [backend query-hash strategy [is-binding] & body]
- `(cached-results ~backend ~query-hash ~strategy (fn [~(vary-meta is-binding assoc :tag 'java.io.InputStream)]
- ~@body)))
+ InputStream `is` (and `updated-at`) will be `nil` if there is no cache entry at all. `is` is only valid within
+ `body`."
+ {:style/indent 3}
+ [backend query-hash [is-binding updated-at-binding] & body]
+ `(cached-results ~backend ~query-hash
+ (fn [~(vary-meta is-binding assoc :tag 'java.io.InputStream) ~updated-at-binding]
+ ~@body)))
(defmulti cache-backend
"Return an instance of a cache backend, which is any object that implements `QueryProcessorCacheBackend`.
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index a053db53e3c..a3d12fc3573 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -18,6 +18,7 @@
[metabase.lib.test-util :as lib.tu]
[metabase.queries.models.query :as query]
[metabase.query-processor.middleware.cache :as cache]
+ [metabase.query-processor.middleware.cache-backend.db :as backend.db]
[metabase.query-processor.middleware.cache-backend.interface :as i]
[metabase.query-processor.middleware.cache.impl :as impl]
[metabase.query-processor.middleware.process-userland-query :as process-userland-query]
@@ -59,7 +60,8 @@
(defn- test-backend
"In in-memory cache backend implementation."
[save-chan purge-chan]
- (let [store (atom nil)]
+ (let [store (atom nil)
+ leases (atom {})]
(reify
pretty/PrettyPrintable
(pretty [_]
@@ -69,27 +71,20 @@
[hash (u/format-nanoseconds (.getNano (t/duration created (t/instant))))]))))
i/CacheBackend
- (cached-results [this query-hash strategy respond]
- (assert (= :ttl (:type strategy)))
- (assert (contains? strategy :avg-execution-ms))
- (let [hex-hash (codecs/bytes->hex query-hash)
- max-age-ms (* (:multiplier strategy)
- (:avg-execution-ms strategy))]
+
+ (cached-results [this query-hash respond]
+ (let [hex-hash (codecs/bytes->hex query-hash)]
(log/tracef "Fetch results for %s store: %s" hex-hash (pretty/pretty this))
- (if-let [^bytes results (when-let [{:keys [created results]} (some (fn [[hash entry]]
- (when (= hash hex-hash)
- entry))
- @store)]
- (when (t/after? created (t/minus (t/instant) (t/millis max-age-ms)))
- results))]
+ (if-let [{:keys [^bytes results created]} (get @store hex-hash)]
(with-open [is (java.io.ByteArrayInputStream. results)]
- (respond is))
- (respond nil))))
+ (respond is created))
+ (respond nil nil))))
(save-results! [this query-hash results]
(let [hex-hash (codecs/bytes->hex query-hash)]
(swap! store assoc hex-hash {:results results
:created (t/instant)})
+ (swap! leases dissoc hex-hash) ; release the refresh lease now that fresh results are stored
(log/tracef "Save results for %s --> store: %s" hex-hash (pretty/pretty this)))
(a/>!! save-chan results))
@@ -99,7 +94,19 @@
(t/after? created (t/minus (t/instant) (t/seconds max-age-seconds))))
store))))
(log/tracef "Purge old entries --> store: %s" (pretty/pretty this))
- (a/>!! purge-chan ::purge)))))
+ (a/>!! purge-chan ::purge))
+
+ (try-acquire-refresh-lease! [_this query-hash lease-ms]
+ (let [hex-hash (codecs/bytes->hex query-hash)
+ now (t/instant)
+ won (volatile! false)]
+ (swap! leases (fn [ls]
+ (let [held (get ls hex-hash)]
+ (if (or (nil? held) (t/before? held now))
+ (do (vreset! won true)
+ (assoc ls hex-hash (t/plus now (t/millis lease-ms))))
+ ls))))
+ @won)))))
(defn do-with-mock-cache! [f]
(mt/with-open-channels [save-chan (a/chan 10)
@@ -226,6 +233,32 @@
(is (= :not-cached
(run-query :cache-strategy (assoc (ttl-strategy) :multiplier 0.1)))))))
+(deftest refresh-lease-test
+ (testing "try-acquire-refresh-lease! (the db backend) elects a single refresher across processes via a conditional UPDATE"
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/QueryCache {query-hash :query_hash} {:query_hash (byte-array (range 32))
+ :results (byte-array [0])
+ :updated_at (t/offset-date-time)}]
+ (testing "the first caller wins the lease"
+ (is (true? (backend.db/try-acquire-refresh-lease! query-hash (u/minutes->ms 5)))))
+ (testing "a concurrent caller loses while the lease is still held"
+ (is (false? (backend.db/try-acquire-refresh-lease! query-hash (u/minutes->ms 5)))))
+ (testing "an abandoned lease (older than the caller's tolerance) can be taken over"
+ (is (true? (backend.db/try-acquire-refresh-lease! query-hash -1)))))))
+
+(deftest stale-while-revalidate-test
+ (testing "an expired entry whose refresh lease is already held by another process is served stale instead of
+ recomputed, so concurrent requests don't stampede the data warehouse"
+ (with-mock-cache! [save-chan]
+ (let [strategy (assoc (ttl-strategy) :multiplier 0.1)
+ query-hash (qp.util/query-hash (test-query {:cache-strategy strategy}))]
+ (run-query :cache-strategy strategy)
+ (mt/wait-for-result save-chan)
+ (Thread/sleep 200)
+ (is (true? (i/try-acquire-refresh-lease! cache/*backend* query-hash 600000)))
+ (is (= :cached
+ (run-query :cache-strategy strategy)))))))
+
(deftest ignore-valid-results-when-caching-is-disabled-test
(testing "if caching is disabled then cache shouldn't be used even if there's something valid in there"
(with-mock-cache! [save-chan]
@@ -305,8 +338,8 @@
(let [query-hash (qp.util/query-hash (test-query nil))]
(testing "Cached results should exist"
(is (true?
- (i/cached-results cache/*backend* query-hash (ttl-strategy)
- some?))))
+ (i/cached-results cache/*backend* query-hash
+ (fn [is _updated-at] (some? is))))))
(i/save-results! cache/*backend* query-hash (byte-array [0 0 0]))
(testing "Invalid cache entry should be handled gracefully"
(is (= :not-cached