The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
QUE2-756: Failed cache refresh (e.g. result too large) leaves refresh lease held and keeps serving stale cache (#78132)
Ben Grabowcbaff9c03ef9
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex a538fad0015..3041a95d2c0 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -14,7 +14,7 @@:deprecated-var 91:discouraged-java-method 4:discouraged-namespace 81- :discouraged-var 143+ :discouraged-var 144:equals-true 28:invalid-arity 1:main-without-gen-class 1diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex e5fb836cfc6..93deb56b585 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -93,24 +93,27 @@(*result-fn*)))(defn- cache-results!- "Save the final results of a query."- [query-hash]+ "Save the final results of a query. Returns true if the results were saved successfully."+ [result-fn query-hash](log/infof "Caching results for next time for query with hash %s. %s"(pr-str (i/short-hex-hash query-hash)) (u/emoji "💾"))(try- (let [bytez (serialized-bytes)]+ (let [bytez (result-fn)](if-not (instance? (Class/forName "[B") bytez)- (log/errorf "Cannot cache results: expected byte array, got %s" (class bytez))+ (do+ (log/errorf "Cannot cache results: expected byte array, got %s" (class bytez))+ false)(do(log/trace "Got serialized bytes; saving to cache backend")(i/save-results! *backend* query-hash bytez)(log/debug "Successfully cached results for query.")- (schedule-purge! *backend*))))- :done+ (schedule-purge! *backend*)+ true)))(catch Throwable e(if (= (:type (ex-data e)) ::impl/max-bytes)(log/debugf e "Not caching results: results are larger than %s KB" (cache/query-caching-max-kb))- (log/errorf e "Error saving query results to cache: %s" (ex-message e))))))+ (log/errorf e "Error saving query results to cache: %s" (ex-message e)))+ false)))(defn- save-results-xform [start-time-ns metadata query-hash strategy rf](add-object-to-cache! (assoc metadata@@ -133,10 +136,14 @@(u/format-milliseconds duration-ms)(u/format-milliseconds min-duration-ms)(if eligible? "eligible" "not eligible"))- (when eligible?- (cache-results! query-hash))- (rf (cond-> result- (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))+ (let [stored? (boolean (when eligible?+ (cache-results! serialized-bytes query-hash)))]+ ;; fresh results weren't saved (too large, save error, or no longer cache-eligible): any existing entry is+ ;; outdated and the refresh lease is still held, so delete it rather than let it keep being served stale+ (when-not stored?+ (i/delete-entry! *backend* query-hash))+ (rf (cond-> result+ (map? result) (update :cache/details assoc :hash query-hash :stored stored?))))))([acc row](add-object-to-cache! row)diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.cljindex f45e7e6dd30..b771018688e 100644--- a/src/metabase/query_processor/middleware/cache_backend/db.clj+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj@@ -93,6 +93,18 @@[:coalesce :refresh_started_at lease-free-sentinel] [:< (ms-ago lease-ms)]}{:refresh_started_at (t/offset-date-time)})))+(defn delete-entry!+ "Delete the cache entry for `query-hash`, if one exists. Deleting the row also releases any held refresh lease, so+ after a refresh fails to save new results the outdated blob is neither served stale to lease losers nor treated as+ fresh for the rest of its window. Never throws: this runs during query result reduction, and a failed cleanup+ shouldn't fail a query that already ran successfully."+ [^bytes query-hash]+ (try+ (t2/delete! (t2/table-name :model/QueryCache) :query_hash query-hash)+ (catch Throwable e+ (log/error e "Error deleting outdated cache entry")))+ nil)+(defn- purge-old-cache-entries!"Delete any cache entries that are older than the global max age `max-cache-entry-age-seconds` (currently 3 months)."[max-age-seconds]@@ -137,5 +149,8 @@(purge-old-entries! [_ max-age-seconds](purge-old-cache-entries! max-age-seconds))+ (delete-entry! [_ query-hash]+ (delete-entry! query-hash))+(try-acquire-refresh-lease! [_ query-hash lease-ms](try-acquire-refresh-lease! query-hash lease-ms))))diff --git a/src/metabase/query_processor/middleware/cache_backend/interface.clj b/src/metabase/query_processor/middleware/cache_backend/interface.cljindex ae37b043bd6..401a5f6a8b2 100644--- a/src/metabase/query_processor/middleware/cache_backend/interface.clj+++ b/src/metabase/query_processor/middleware/cache_backend/interface.clj@@ -35,6 +35,12 @@"Purge all cache entries older than `max-age-seconds`. Will be called periodically when this backend is in use.`max-age-seconds` may be floating-point.")+ (delete-entry! [this ^bytes query-hash]+ "Delete the cache entry for `query-hash`, if one exists, releasing any held refresh lease. Called when a query ran+ but its results could not be saved to the cache (e.g. they exceed `query-caching-max-kb`), so the outdated entry+ doesn't keep being served to other callers. Implementations must not throw: this is called during query result+ reduction, and a failed cleanup shouldn't fail a query that already ran successfully.")+(try-acquire-refresh-lease! [this ^bytes query-hash lease-ms]"Atomically claim, across processes, the right to recompute the expired entry for `query-hash`. Returns true iffthis caller won the lease (and should recompute + [[save-results!]]); false means another process is alreadydiff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 520dec7ecb4..7dabc40f5c1 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -96,6 +96,12 @@(log/tracef "Purge old entries --> store: %s" (pretty/pretty this))(a/>!! purge-chan ::purge))+ (delete-entry! [this query-hash]+ (let [hex-hash (codecs/bytes->hex query-hash)]+ (swap! store dissoc hex-hash)+ (swap! leases dissoc hex-hash)+ (log/tracef "Delete entry for %s --> store: %s" hex-hash (pretty/pretty this))))+(try-acquire-refresh-lease! [_this query-hash lease-ms](let [hex-hash (codecs/bytes->hex query-hash)now (t/instant)@@ -264,6 +270,64 @@(testing "updated_at was left alone -- results have not actually been rewritten yet"(is (= (t/instant original-updated-at) (t/instant updated_at)))))))))+(deftest delete-entry-test+ (testing "delete-entry! (the db backend) removes the cache entry, and with it any held refresh lease"+ #_{:clj-kondo/ignore [:discouraged-var]}+ (mt/with-temp [:model/QueryCache {query-hash :query_hash} {:query_hash (byte-array (range 32))+ :results (byte-array [0])+ :updated_at (t/offset-date-time)}]+ (is (true? (backend.db/try-acquire-refresh-lease! query-hash (u/minutes->ms 5))))+ (backend.db/delete-entry! query-hash)+ (is (nil? (t2/select-one :model/QueryCache :query_hash query-hash))))))++(deftest failed-refresh-deletes-outdated-entry-test+ (testing "a refresh that fails to save (e.g. results exceed query-caching-max-kb) deletes the outdated entry+ instead of leaving it to be served to later requests"+ (with-mock-cache! [save-chan]+ (run-query)+ (mt/wait-for-result save-chan)+ (is (= :cached+ (run-query)))+ (mt/with-temporary-setting-values [query-caching-max-kb 0]+ (run-query :middleware {:ignore-cached-results? true})+ (mt/wait-for-result save-chan))+ (testing "the still-fresh-but-outdated entry is gone, so a normal load recomputes"+ (is (= :not-cached+ (run-query)))))))++(deftest failed-refresh-releases-refresh-lease-test+ (testing "when the lease winner's refresh fails to save, the expired entry is deleted so other processes recompute+ instead of serving it stale for the rest of the lease window"+ (with-mock-cache! [save-chan]+ (let [strategy (assoc (ttl-strategy) :multiplier 0.1)]+ (run-query :cache-strategy strategy)+ (mt/wait-for-result save-chan)+ (Thread/sleep 200)+ (mt/with-temporary-setting-values [query-caching-max-kb 0]+ (testing "this run wins the refresh lease, recomputes, and fails to save"+ (is (= :not-cached+ (run-query :cache-strategy strategy))))+ (mt/wait-for-result save-chan)+ (testing "the next request recomputes rather than being served the stale entry"+ (is (= :not-cached+ (run-query :cache-strategy strategy)))))))))++(deftest not-eligible-refresh-deletes-outdated-entry-test+ (testing "when the lease winner's rerun is no longer cache-eligible (ran under min-duration-ms), the expired entry+ is deleted so other processes recompute instead of serving it stale for the rest of the lease window"+ (with-mock-cache! [save-chan]+ (run-query :cache-strategy (assoc (ttl-strategy) :multiplier 0.1))+ (mt/wait-for-result save-chan)+ (Thread/sleep 200)+ (binding [*query-caching-min-ttl* 1000]+ (let [strategy (assoc (ttl-strategy) :multiplier 0.1)]+ (testing "this run wins the refresh lease, recomputes, and doesn't save (query is now too fast)"+ (is (= :not-cached+ (run-query :cache-strategy strategy))))+ (testing "the next request recomputes rather than being served the stale entry"+ (is (= :not-cached+ (run-query :cache-strategy strategy)))))))))+(deftest stale-while-revalidate-test(testing "an expired entry whose refresh lease is already held by another process is served stale instead ofrecomputed, so concurrent requests don't stampede the data warehouse"@@ -767,20 +831,29 @@(m/dissoc-in [:data :results_metadata :checksum])))"Query should be cached and results should match those ran without cache"))))))+(def stub-no-op-cache-backend+ (reify i/CacheBackend+ (cached-results [_ _ respond] (respond nil nil))+ (save-results! [_ _ _])+ (purge-old-entries! [_ _])+ (delete-entry! [_ _])+ (try-acquire-refresh-lease! [_ _ _] true)))+(deftest ^:parallel caching-big-resultsets- (testing "Make sure we can save large result sets without tripping over internal async buffers"- (is (= 10000 (count (transduce identity- (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)- (repeat 10000 [1]))))))- (testing "Make sure we don't block somewhere if we decide not to save results"- (is (= 10000 (count (transduce identity- (#'cache/save-results-xform (System/currentTimeMillis) {} (byte 0) (ttl-strategy) conj)- (repeat 10000 [1]))))))- (testing "Make sure we properly handle situations where we abort serialization (e.g. due to result being too big)"- (let [max-bytes (* (metabase.cache.core/query-caching-max-kb) 1024)]- (is (= max-bytes (count (transduce identity- (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)- (repeat max-bytes [1]))))))))+ (binding [cache/*backend* stub-no-op-cache-backend]+ (testing "Make sure we can save large result sets without tripping over internal async buffers"+ (is (= 10000 (count (transduce identity+ (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)+ (repeat 10000 [1]))))))+ (testing "Make sure we don't block somewhere if we decide not to save results"+ (is (= 10000 (count (transduce identity+ (#'cache/save-results-xform (System/currentTimeMillis) {} (byte 0) (ttl-strategy) conj)+ (repeat 10000 [1]))))))+ (testing "Make sure we properly handle situations where we abort serialization (e.g. due to result being too big)"+ (let [max-bytes (* (metabase.cache.core/query-caching-max-kb) 1024)]+ (is (= max-bytes (count (transduce identity+ (#'cache/save-results-xform 0 {} (byte 0) (ttl-strategy) conj)+ (repeat max-bytes [1])))))))))(deftest perms-checks-should-still-apply-test(testing "Double-check that perms checks still happen even for cached results"