Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert "Bound concurrent query-cache writes" (#77781)

Alexander Polyankincb41874c74dd
diff --git a/src/metabase/cache/core.clj b/src/metabase/cache/core.clj
index 62b79fcf615..e44dce512ae 100644
--- a/src/metabase/cache/core.clj
+++ b/src/metabase/cache/core.clj
@@ -16,6 +16,5 @@
root-strategy]
[metabase.cache.settings
enable-query-caching
- query-caching-max-concurrent-writes
query-caching-max-kb
query-caching-max-ttl])
diff --git a/src/metabase/cache/settings.clj b/src/metabase/cache/settings.clj
index 8445d8cd449..8cbb8692391 100644
--- a/src/metabase/cache/settings.clj
+++ b/src/metabase/cache/settings.clj
@@ -47,20 +47,3 @@
:type :double
:default (* 60.0 60.0 24.0 35.0) ; 35 days
:audit :getter)
-
-(defsetting query-caching-max-concurrent-writes
- (deferred-tru
- (str "Maximum number of queries that can write their results to the query cache at the same time; 0 means no "
- "limit. Queries over the limit run normally but skip caching that run, bounding the CPU and memory spent "
- "storing results during a burst of cache misses. Each concurrent write buffers a result capped by "
- "query-caching-max-kb (2 MB by default), so e.g. a limit of 32 bounds the worst-case burst cost at "
- "roughly 200 MB including growth transients."))
- :type :integer
- :default 0
- :visibility :internal
- :export? false
- :setter :none
- :doc (str "Maximum number of queries that can write their results to the query cache at the same time; "
- "0 (the default) means no limit. Queries over the limit run normally but skip caching that run, "
- "bounding the CPU and memory spent storing results during a burst of cache misses. E.g. a limit "
- "of 32 bounds the worst-case burst cost at roughly 200 MB."))
diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.clj
index 40cfcd554fe..e5fb836cfc6 100644
--- a/src/metabase/query_processor/middleware/cache.clj
+++ b/src/metabase/query_processor/middleware/cache.clj
@@ -28,7 +28,6 @@
[metabase.util.malli :as mu]
[metabase.util.performance :refer [get-in]])
(:import
- (java.util.concurrent Semaphore)
(org.eclipse.jetty.io EofException)))
(set! *warn-on-reflection* true)
@@ -47,15 +46,6 @@
;;; ------------------------------------------------------ Save ------------------------------------------------------
-(def ^:dynamic *write-permits*
- "Semaphore capping concurrent cache writes ([[cache/query-caching-max-concurrent-writes]]) so a burst of cache misses
- can't spike CPU/memory with unbounded concurrent result serialization. When exhausted, queries run normally but skip
- caching that run. nil (the setting is unset or not a positive integer) means no limit. The root value is a delay so
- the setting isn't read at namespace load (AOT-safe) -- deref with `force`. Bind a bare `Semaphore` in tests."
- (delay (let [n (cache/query-caching-max-concurrent-writes)]
- (when (pos-int? n)
- (Semaphore. n)))))
-
(def ^:private purge-interval-seconds
"How often, at most, to purge cache entries older than [[cache/query-caching-max-ttl]]. Purging on every save is
wasteful: the max TTL is measured in days, and on instances with a high cache-miss rate the repeated DELETEs are
@@ -136,19 +126,17 @@
(let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)
min-duration-ms (:min-duration-ms strategy 0)
;; cache any query that ran long enough -- including ones that returned no rows, so a slow empty result
- ;; doesn't get re-run at full cost on every request. *result-fn* is nil when the run skipped serialization
- ;; because the concurrent cache-write limit was reached.
- eligible? (> duration-ms min-duration-ms)
- stored? (boolean (and eligible? *result-fn*))]
+ ;; doesn't get re-run at full cost on every request
+ eligible? (> duration-ms min-duration-ms)]
(log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"
(i/short-hex-hash query-hash)
(u/format-milliseconds duration-ms)
(u/format-milliseconds min-duration-ms)
(if eligible? "eligible" "not eligible"))
- (when stored?
+ (when eligible?
(cache-results! query-hash))
(rf (cond-> result
- (map? result) (update :cache/details assoc :hash query-hash :stored stored?)))))
+ (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))
([acc row]
(add-object-to-cache! row)
@@ -269,32 +257,9 @@
;;; --------------------------------------------------- Middleware ---------------------------------------------------
-(defn- reduce-with-serialization!
- "Reduce with `orig-reduce`, serializing every reduced object into the current cache entry -- unless the concurrent
- cache-write limit ([[*write-permits*]]) is reached, in which case reduce without serializing: that run just isn't
- cached ([[*result-fn*]] stays nil, so [[save-results-xform]] reports `:stored false` and skips the save)."
- [orig-reduce rff metadata rows]
- (let [^Semaphore permits (force *write-permits*)]
- (if (and permits (not (.tryAcquire permits)))
- (do (log/throttle (u/minutes->ms 1)
- (log/warn "Not caching results: the concurrent cache-write limit was reached"))
- (orig-reduce rff metadata rows))
- (try
- (impl/do-with-serialization
- (fn [in-fn result-fn]
- (binding [*in-fn* in-fn
- *result-fn* result-fn]
- (orig-reduce rff metadata rows))))
- (finally
- (some-> permits .release))))))
-
(defn- run-and-cache!
"Run `query` through `qp` and save the results to the cache (if eligible). Used on a cache miss, or when this process
- holds the stale-while-revalidate refresh lease.
-
- Serialization holds a [[*write-permits*]] permit, scoped to the reduce so warehouse wait time doesn't consume the
- budget. When none is available the query still runs and returns results, they just aren't written to the cache (a
- refresh lease this process holds then just expires and a later run refreshes the entry)."
+ holds the stale-while-revalidate refresh lease."
[qp query query-hash cache-strategy rff]
(let [start-time-ns (System/nanoTime)
orig-reduce qp.pipeline/*reduce*]
@@ -302,7 +267,11 @@
(binding [qp.pipeline/*reduce* (fn reduce'
[rff metadata rows]
{:post [(some? %)]}
- (reduce-with-serialization! orig-reduce rff metadata rows))]
+ (impl/do-with-serialization
+ (fn [in-fn result-fn]
+ (binding [*in-fn* in-fn
+ *result-fn* result-fn]
+ (orig-reduce rff metadata rows)))))]
(qp query
(fn [metadata]
(save-results-xform start-time-ns metadata query-hash cache-strategy (rff metadata)))))))
diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.clj
index 9ed82aedc5f..520dec7ecb4 100644
--- a/test/metabase/query_processor/middleware/cache_test.clj
+++ b/test/metabase/query_processor/middleware/cache_test.clj
@@ -39,8 +39,7 @@
[pretty.core :as pretty]
[toucan2.core :as t2])
(:import
- (java.time ZonedDateTime)
- (java.util.concurrent Semaphore)))
+ (java.time ZonedDateTime)))
(set! *warn-on-reflection* true)
@@ -316,29 +315,6 @@
(is (= :not-cached
(run-query)))))))
-(deftest concurrent-write-limit-test
- (testing "when no cache-write permits are available, the query still runs but its results are not cached"
- (with-mock-cache! [save-chan]
- (binding [cache/*write-permits* (Semaphore. 0)]
- (is (= :not-cached
- (run-query)))
- (is (= :metabase.test.util.async/timed-out
- (mt/wait-for-result save-chan 100))
- "nothing should have been written to the cache backend")
- ;; the second run also runs uncached: nothing was stored the first time
- (is (= :not-cached
- (run-query)))))))
-
-(deftest concurrent-write-limit-release-test
- (testing "permits are released after each run, so sequential queries all get cached"
- (with-mock-cache! [save-chan]
- (binding [cache/*write-permits* (Semaphore. 1)]
- (run-query)
- (mt/wait-for-result save-chan)
- (is (= :cached
- (run-query)))
- (is (= 1 (.availablePermits ^Semaphore cache/*write-permits*)))))))
-
(deftest ignore-cached-results-test
(testing "check that :ignore-cached-results? in middleware is respected when returning results..."
(with-mock-cache! [save-chan]