Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

QUE2-756: Failed cache refresh (e.g. result too large) leaves refresh lease held and keeps serving stale cache (#78132)

Ben Grabowcbaff9c03ef9
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index a538fad0015..3041a95d2c0 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -14,7 +14,7 @@
:deprecated-var 91
:discouraged-java-method 4
:discouraged-namespace 81
- :discouraged-var 143
+ :discouraged-var 144
:equals-true 28
:invalid-arity 1
:main-without-gen-class 1
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.clj
index e5fb836cfc6..93deb56b585 100644
--- a/src/metabase/query_processor/middleware/cache.clj
+++ b/src/metabase/query_processor/middleware/cache.clj
@@ -93,24 +93,27 @@
(*result-fn*)))
(defn- cache-results!
- "Save the final results of a query."
- [query-hash]
+ "Save the final results of a query. Returns true if the results were saved successfully."
+ [result-fn query-hash]
(log/infof "Caching results for next time for query with hash %s. %s"
(pr-str (i/short-hex-hash query-hash)) (u/emoji "💾"))
(try
- (let [bytez (serialized-bytes)]
+ (let [bytez (result-fn)]
(if-not (instance? (Class/forName "[B") bytez)
- (log/errorf "Cannot cache results: expected byte array, got %s" (class bytez))
+ (do
+ (log/errorf "Cannot cache results: expected byte array, got %s" (class bytez))
+ false)
(do
(log/trace "Got serialized bytes; saving to cache backend")
(i/save-results! *backend* query-hash bytez)
(log/debug "Successfully cached results for query.")
- (schedule-purge! *backend*))))
- :done
+ (schedule-purge! *backend*)
+ true)))
(catch Throwable e
(if (= (:type (ex-data e)) ::impl/max-bytes)
(log/debugf e "Not caching results: results are larger than %s KB" (cache/query-caching-max-kb))
- (log/errorf e "Error saving query results to cache: %s" (ex-message e))))))
+ (log/errorf e "Error saving query results to cache: %s" (ex-message e)))
+ false)))
(defn- save-results-xform [start-time-ns metadata query-hash strategy rf]
(add-object-to-cache! (assoc metadata
@@ -133,10 +136,14 @@
(u/format-milliseconds duration-ms)
(u/format-milliseconds min-duration-ms)
(if eligible? "eligible" "not eligible"))
- (when eligible?
- (cache-results! query-hash))
- (rf (cond-> result
- (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))
+ (let [stored? (boolean (when eligible?
+ (cache-results! serialized-bytes query-hash)))]
+ ;; fresh results weren't saved (too large, save error, or no longer cache-eligible): any existing entry is
+ ;; outdated and the refresh lease is still held, so delete it rather than let it keep being served stale
+ (when-not stored?
+ (i/delete-entry! *backend* query-hash))
+ (rf (cond-> result
+ (map? result) (update :cache/details assoc :hash query-hash :stored stored?))))))
([acc row]
(add-object-to-cache! row)
diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.clj
index f45e7e6dd30..b771018688e 100644
--- a/src/metabase/query_processor/middleware/cache_backend/db.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj
@@ -93,6 +93,18 @@
[:coalesce :refresh_started_at lease-free-sentinel] [:< (ms-ago lease-ms)]}
{:refresh_started_at (t/offset-date-time)})))
+(defn delete-entry!
+ "Delete the cache entry for `query-hash`, if one exists. Deleting the row also releases any held refresh lease, so
+ after a refresh fails to save new results the outdated blob is neither served stale to lease losers nor treated as
+ fresh for the rest of its window. Never throws: this runs during query result reduction, and a failed cleanup
+ shouldn't fail a query that already ran successfully."
+ [^bytes query-hash]
+ (try
+ (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash)
+ (catch Throwable e
+ (log/error e "Error deleting outdated cache entry")))
+ nil)
+
(defn- purge-old-cache-entries!
"Delete any cache entries that are older than the global max age `max-cache-entry-age-seconds` (currently 3 months)."
[max-age-seconds]
@@ -137,5 +149,8 @@
(purge-old-entries! [_ max-age-seconds]
(purge-old-cache-entries! max-age-seconds))
+ (delete-entry! [_ query-hash]
+ (delete-entry! query-hash))
+
(try-acquire-refresh-lease! [_ query-hash lease-ms]
(try-acquire-refresh-lease! query-hash lease-ms))))
diff --git a/src/metabase/query_processor/middleware/cache_backend/interface.clj b/src/metabase/query_processor/middleware/cache_backend/interface.clj
index ae37b043bd6..401a5f6a8b2 100644
--- a/src/metabase/query_processor/middleware/cache_backend/interface.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/interface.clj
@@ -35,6 +35,12 @@
"Purge all cache entries older than `max-age-seconds`. Will be called periodically when this backend is in use.
`max-age-seconds` may be floating-point.")
+ (delete-entry! [this ^bytes query-hash]
+ "Delete the cache entry for `query-hash`, if one exists, releasing any held refresh lease. Called when a query ran
+ but its results could not be saved to the cache (e.g. they exceed `query-caching-max-kb`), so the outdated entry
+ doesn't keep being served to other callers. Implementations must not throw: this is called during query result
+ reduction, and a failed cleanup shouldn't fail a query that already ran successfully.")
+
(try-acquire-refresh-lease! [this ^bytes query-hash lease-ms]
"Atomically claim, across processes, the right to recompute the expired entry for `query-hash`. Returns true iff
this caller won the lease (and should recompute + [[save-results!]]); false means another process is already
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index 520dec7ecb4..7dabc40f5c1 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -96,6 +96,12 @@
(log/tracef "Purge old entries --> store: %s" (pretty/pretty this))
(a/>!! purge-chan ::purge))
+ (delete-entry! [this query-hash]
+ (let [hex-hash (codecs/bytes->hex query-hash)]
+ (swap! store dissoc hex-hash)
+ (swap! leases dissoc hex-hash)
+ (log/tracef "Delete entry for %s --> store: %s" hex-hash (pretty/pretty this))))
+
(try-acquire-refresh-lease! [_this query-hash lease-ms]
(let [hex-hash (codecs/bytes->hex query-hash)
now (t/instant)
@@ -264,6 +270,64 @@
(testing "updated_at was left alone -- results have not actually been rewritten yet"
(is (= (t/instant original-updated-at) (t/instant updated_at)))))))))
+(deftest delete-entry-test
+ (testing "delete-entry! (the db backend) removes the cache entry, and with it any held refresh lease"
+ #_{:clj-kondo/ignore [:discouraged-var]}
+ (mt/with-temp [:model/QueryCache {query-hash :query_hash} {:query_hash (byte-array (range 32))
+ :results (byte-array [0])
+ :updated_at (t/offset-date-time)}]
+ (is (true? (backend.db/try-acquire-refresh-lease! query-hash (u/minutes->ms 5))))
+ (backend.db/delete-entry! query-hash)
+ (is (nil? (t2/select-one :model/QueryCache :query_hash query-hash))))))
+
+(deftest failed-refresh-deletes-outdated-entry-test
+ (testing "a refresh that fails to save (e.g. results exceed query-caching-max-kb) deletes the outdated entry
+ instead of leaving it to be served to later requests"
+ (with-mock-cache! [save-chan]
+ (run-query)
+ (mt/wait-for-result save-chan)
+ (is (= :cached
+ (run-query)))
+ (mt/with-temporary-setting-values [query-caching-max-kb 0]
+ (run-query :middleware {:ignore-cached-results? true})
+ (mt/wait-for-result save-chan))
+ (testing "the still-fresh-but-outdated entry is gone, so a normal load recomputes"
+ (is (= :not-cached
+ (run-query)))))))
+
+(deftest failed-refresh-releases-refresh-lease-test
+ (testing "when the lease winner's refresh fails to save, the expired entry is deleted so other processes recompute
+ instead of serving it stale for the rest of the lease window"
+ (with-mock-cache! [save-chan]
+ (let [strategy (assoc (ttl-strategy) :multiplier 0.1)]
+ (run-query :cache-strategy strategy)
+ (mt/wait-for-result save-chan)
+ (Thread/sleep 200)
+ (mt/with-temporary-setting-values [query-caching-max-kb 0]
+ (testing "this run wins the refresh lease, recomputes, and fails to save"
+ (is (= :not-cached
+ (run-query :cache-strategy strategy))))
+ (mt/wait-for-result save-chan)
+ (testing "the next request recomputes rather than being served the stale entry"
+ (is (= :not-cached
+ (run-query :cache-strategy strategy)))))))))
+
+(deftest not-eligible-refresh-deletes-outdated-entry-test
+ (testing "when the lease winner's rerun is no longer cache-eligible (ran under min-duration-ms), the expired entry
+ is deleted so other processes recompute instead of serving it stale for the rest of the lease window"
+ (with-mock-cache! [save-chan]
+ (run-query :cache-strategy (assoc (ttl-strategy) :multiplier 0.1))
+ (mt/wait-for-result save-chan)
+ (Thread/sleep 200)
+ (binding [*query-caching-min-ttl* 1000]
+ (let [strategy (assoc (ttl-strategy) :multiplier 0.1)]
+ (testing "this run wins the refresh lease, recomputes, and doesn't save (query is now too fast)"
+ (is (= :not-cached
+ (run-query :cache-strategy strategy))))
+ (testing "the next request recomputes rather than being served the stale entry"
+ (is (= :not-cached
+ (run-query :cache-strategy strategy)))))))))
+
(deftest stale-while-revalidate-test
(testing "an expired entry whose refresh lease is already held by another process is served stale instead of
recomputed, so concurrent requests don't stampede the data warehouse"
@@ -767,20 +831,29 @@
(m/dissoc-in [:data :results_metadata :checksum])))
"Query should be cached and results should match those ran without cache"))))))
+(def stub-no-op-cache-backend
+ (reify i/CacheBackend
+ (cached-results [_ _ respond] (respond nil nil))
+ (save-results! [_ _ _])
+ (purge-old-entries! [_ _])
+ (delete-entry! [_ _])
+ (try-acquire-refresh-lease! [_ _ _] true)))
+
(deftest ^:parallel caching-big-resultsets
- (testing "Make sure we can save large result sets without tripping over internal async buffers"
- (is (= 10000 (count (transduce identity
- (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)
- (repeat 10000 [1]))))))
- (testing "Make sure we don't block somewhere if we decide not to save results"
- (is (= 10000 (count (transduce identity
- (#'cache/save-results-xform (System/currentTimeMillis) {} (byte 0) (ttl-strategy) conj)
- (repeat 10000 [1]))))))
- (testing "Make sure we properly handle situations where we abort serialization (e.g. due to result being too big)"
- (let [max-bytes (* (metabase.cache.core/query-caching-max-kb) 1024)]
- (is (= max-bytes (count (transduce identity
- (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)
- (repeat max-bytes [1]))))))))
+ (binding [cache/*backend* stub-no-op-cache-backend]
+ (testing "Make sure we can save large result sets without tripping over internal async buffers"
+ (is (= 10000 (count (transduce identity
+ (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)
+ (repeat 10000 [1]))))))
+ (testing "Make sure we don't block somewhere if we decide not to save results"
+ (is (= 10000 (count (transduce identity
+ (#'cache/save-results-xform (System/currentTimeMillis) {} (byte 0) (ttl-strategy) conj)
+ (repeat 10000 [1]))))))
+ (testing "Make sure we properly handle situations where we abort serialization (e.g. due to result being too big)"
+ (let [max-bytes (* (metabase.cache.core/query-caching-max-kb) 1024)]
+ (is (= max-bytes (count (transduce identity
+ (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)
+ (repeat max-bytes [1])))))))))
(deftest perms-checks-should-still-apply-test
(testing "Double-check that perms checks still happen even for cached results"