The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Revert "Bound concurrent query-cache writes" (#77781)
Alexander Polyankincb41874c74dd
diff --git a/src/metabase/cache/core.clj b/src/metabase/cache/core.cljindex 62b79fcf615..e44dce512ae 100644--- a/src/metabase/cache/core.clj+++ b/src/metabase/cache/core.clj@@ -16,6 +16,5 @@root-strategy][metabase.cache.settingsenable-query-caching- query-caching-max-concurrent-writesquery-caching-max-kbquery-caching-max-ttl])diff --git a/src/metabase/cache/settings.clj b/src/metabase/cache/settings.cljindex 8445d8cd449..8cbb8692391 100644--- a/src/metabase/cache/settings.clj+++ b/src/metabase/cache/settings.clj@@ -47,20 +47,3 @@:type :double:default (* 60.0 60.0 24.0 35.0) ; 35 days:audit :getter)--(defsetting query-caching-max-concurrent-writes- (deferred-tru- (str "Maximum number of queries that can write their results to the query cache at the same time; 0 means no "- "limit. Queries over the limit run normally but skip caching that run, bounding the CPU and memory spent "- "storing results during a burst of cache misses. Each concurrent write buffers a result capped by "- "query-caching-max-kb (2 MB by default), so e.g. a limit of 32 bounds the worst-case burst cost at "- "roughly 200 MB including growth transients."))- :type :integer- :default 0- :visibility :internal- :export? false- :setter :none- :doc (str "Maximum number of queries that can write their results to the query cache at the same time; "- "0 (the default) means no limit. Queries over the limit run normally but skip caching that run, "- "bounding the CPU and memory spent storing results during a burst of cache misses. E.g. a limit "- "of 32 bounds the worst-case burst cost at roughly 200 MB."))diff --git a/src/metabase/query_processor/middleware/cache.clj b/src/metabase/query_processor/middleware/cache.cljindex 40cfcd554fe..e5fb836cfc6 100644--- a/src/metabase/query_processor/middleware/cache.clj+++ b/src/metabase/query_processor/middleware/cache.clj@@ -28,7 +28,6 @@[metabase.util.malli :as mu][metabase.util.performance :refer [get-in]])(:import- (java.util.concurrent Semaphore)(org.eclipse.jetty.io EofException)))(set! *warn-on-reflection* true)@@ -47,15 +46,6 @@;;; ------------------------------------------------------ Save -------------------------------------------------------(def ^:dynamic *write-permits*- "Semaphore capping concurrent cache writes ([[cache/query-caching-max-concurrent-writes]]) so a burst of cache misses- can't spike CPU/memory with unbounded concurrent result serialization. When exhausted, queries run normally but skip- caching that run. nil (the setting is unset or not a positive integer) means no limit. The root value is a delay so- the setting isn't read at namespace load (AOT-safe) -- deref with `force`. Bind a bare `Semaphore` in tests."- (delay (let [n (cache/query-caching-max-concurrent-writes)]- (when (pos-int? n)- (Semaphore. n)))))-(def ^:private purge-interval-seconds"How often, at most, to purge cache entries older than [[cache/query-caching-max-ttl]]. Purging on every save iswasteful: the max TTL is measured in days, and on instances with a high cache-miss rate the repeated DELETEs are@@ -136,19 +126,17 @@(let [duration-ms (/ (- (System/nanoTime) start-time-ns) 1e6)min-duration-ms (:min-duration-ms strategy 0);; cache any query that ran long enough -- including ones that returned no rows, so a slow empty result- ;; doesn't get re-run at full cost on every request. *result-fn* is nil when the run skipped serialization- ;; because the concurrent cache-write limit was reached.- eligible? (> duration-ms min-duration-ms)- stored? (boolean (and eligible? *result-fn*))]+ ;; doesn't get re-run at full cost on every request+ eligible? (> duration-ms min-duration-ms)](log/infof "Query %s took %s to run; minimum for cache eligibility is %s; %s"(i/short-hex-hash query-hash)(u/format-milliseconds duration-ms)(u/format-milliseconds min-duration-ms)(if eligible? "eligible" "not eligible"))- (when stored?+ (when eligible?(cache-results! query-hash))(rf (cond-> result- (map? result) (update :cache/details assoc :hash query-hash :stored stored?)))))+ (map? result) (update :cache/details assoc :hash query-hash :stored (boolean eligible?))))))([acc row](add-object-to-cache! row)@@ -269,32 +257,9 @@;;; --------------------------------------------------- Middleware ----------------------------------------------------(defn- reduce-with-serialization!- "Reduce with `orig-reduce`, serializing every reduced object into the current cache entry -- unless the concurrent- cache-write limit ([[*write-permits*]]) is reached, in which case reduce without serializing: that run just isn't- cached ([[*result-fn*]] stays nil, so [[save-results-xform]] reports `:stored false` and skips the save)."- [orig-reduce rff metadata rows]- (let [^Semaphore permits (force *write-permits*)]- (if (and permits (not (.tryAcquire permits)))- (do (log/throttle (u/minutes->ms 1)- (log/warn "Not caching results: the concurrent cache-write limit was reached"))- (orig-reduce rff metadata rows))- (try- (impl/do-with-serialization- (fn [in-fn result-fn]- (binding [*in-fn* in-fn- *result-fn* result-fn]- (orig-reduce rff metadata rows))))- (finally- (some-> permits .release))))))-(defn- run-and-cache!"Run `query` through `qp` and save the results to the cache (if eligible). Used on a cache miss, or when this process- holds the stale-while-revalidate refresh lease.-- Serialization holds a [[*write-permits*]] permit, scoped to the reduce so warehouse wait time doesn't consume the- budget. When none is available the query still runs and returns results, they just aren't written to the cache (a- refresh lease this process holds then just expires and a later run refreshes the entry)."+ holds the stale-while-revalidate refresh lease."[qp query query-hash cache-strategy rff](let [start-time-ns (System/nanoTime)orig-reduce qp.pipeline/*reduce*]@@ -302,7 +267,11 @@(binding [qp.pipeline/*reduce* (fn reduce'[rff metadata rows]{:post [(some? %)]}- (reduce-with-serialization! orig-reduce rff metadata rows))]+ (impl/do-with-serialization+ (fn [in-fn result-fn]+ (binding [*in-fn* in-fn+ *result-fn* result-fn]+ (orig-reduce rff metadata rows)))))](qp query(fn [metadata](save-results-xform start-time-ns metadata query-hash cache-strategy (rff metadata)))))))diff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex 9ed82aedc5f..520dec7ecb4 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -39,8 +39,7 @@[pretty.core :as pretty][toucan2.core :as t2])(:import- (java.time ZonedDateTime)- (java.util.concurrent Semaphore)))+ (java.time ZonedDateTime)))(set! *warn-on-reflection* true)@@ -316,29 +315,6 @@(is (= :not-cached(run-query)))))))-(deftest concurrent-write-limit-test- (testing "when no cache-write permits are available, the query still runs but its results are not cached"- (with-mock-cache! [save-chan]- (binding [cache/*write-permits* (Semaphore. 0)]- (is (= :not-cached- (run-query)))- (is (= :metabase.test.util.async/timed-out- (mt/wait-for-result save-chan 100))- "nothing should have been written to the cache backend")- ;; the second run also runs uncached: nothing was stored the first time- (is (= :not-cached- (run-query)))))))--(deftest concurrent-write-limit-release-test- (testing "permits are released after each run, so sequential queries all get cached"- (with-mock-cache! [save-chan]- (binding [cache/*write-permits* (Semaphore. 1)]- (run-query)- (mt/wait-for-result save-chan)- (is (= :cached- (run-query)))- (is (= 1 (.availablePermits ^Semaphore cache/*write-permits*)))))))-(deftest ignore-cached-results-test(testing "check that :ignore-cached-results? in middleware is respected when returning results..."(with-mock-cache! [save-chan]