The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Throttle heavy log4j calls (#76482)
Nathan Voxlandbd049b7d08ac
diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljcindex 91192435761..6ad4e60ba44 100644--- a/src/metabase/lib/metadata/calculation.cljc+++ b/src/metabase/lib/metadata/calculation.cljc@@ -308,7 +308,9 @@(try(describe-query query)(catch #?(:clj Throwable :cljs js/Error) e- (log/errorf e "Error calculating display name for query: %s" (ex-message e))+ ;; Throttled: a search reindex can call this for many failing metric Cards, don't log them all.+ (log/throttle (* 10 1000)+ (log/errorf e "Error calculating display name for query: %s" (ex-message e)))nil))))(defmulti display-info-methoddiff --git a/src/metabase/util/log.clj b/src/metabase/util/log.cljindex a34ab954ce7..75c895a9aec 100644--- a/src/metabase/util/log.clj+++ b/src/metabase/util/log.clj@@ -14,6 +14,7 @@[metabase.config.core :as config][metabase.util.format :as u.format][metabase.util.log.capture]+ [metabase.util.log.throttle][metabase.util.performance :as perf][net.cgrand.macrovich :as macros])(:import@@ -314,6 +315,23 @@`(binding [clojure.tools.logging/*logger-factory* clojure.tools.logging.impl/disabled-logger-factory]~@body))+(defmacro throttle+ "Evaluate `body` at most once per `interval-ms`, throttled per call site.++ Intended to wrap a log call on a hot path so it can't flood the logs:++ (log/throttle (* 60 1000)+ (log/errorf e \"Couldn't do the thing: %s\" (ex-message e)))++ When throttled, `body` is skipped *entirely* — including the underlying logger lookup.+ Returns the value of `body`, or `nil` when throttled."+ {:arglists '([interval-ms & body])}+ [interval-ms & body]+ (let [m (meta &form)+ k (keyword (str *ns*) (str (:line m) \- (:column m)))]+ `(when (metabase.util.log.throttle/allow? ~k ~interval-ms)+ ~@body)))+(defn- copy-ex-info"Copies an ExceptionInfo into a new ExceptionInfo with different ex-data & cause, but all other data the same"^ExceptionInfodiff --git a/src/metabase/util/log.cljs b/src/metabase/util/log.cljsindex f2d0552ff98..3f449d8dd6f 100644--- a/src/metabase/util/log.cljs+++ b/src/metabase/util/log.cljs@@ -5,7 +5,8 @@[goog.string.format :as gstring.format][lambdaisland.glogi :as log][lambdaisland.glogi.console :as glogi-console]- [metabase.util.log.capture])+ [metabase.util.log.capture]+ [metabase.util.log.throttle])(:require-macros[metabase.util.log]))diff --git a/src/metabase/util/log/throttle.cljc b/src/metabase/util/log/throttle.cljcnew file mode 100644index 00000000000..b4cd6a75538--- /dev/null+++ b/src/metabase/util/log/throttle.cljc@@ -0,0 +1,32 @@+(ns metabase.util.log.throttle+ "Runtime support for [[metabase.util.log/throttle]]. Kept separate from `metabase.util.log` so the+ state and decision logic can be shared by both the CLJ and CLJS expansions of the macro (mirroring+ `metabase.util.log.capture`).")++(defonce ^{:doc "Map of throttle key -> epoch-ms the key last fired. Bounded by the number of call sites."}+ state+ (atom {}))++(defn- now-ms ^long []+ #?(:clj (System/currentTimeMillis)+ :cljs (.now js/Date)))++(defn allow?+ "Return `true` (and record the current time against `k`) if `k` has not fired within the last+ `interval-ms`, otherwise `false`. Thread-safe; at most one caller per interval is allowed through.++ Optimized for the throttled (suppressed) case, which dominates on the hot paths this is meant for: that+ path is a lock-free read of `state` plus a map lookup — no CAS, no allocation. The atomic `swap!` runs+ only when the window has actually elapsed (≈ once per `interval-ms` per key), with a double-check inside+ to keep it race-correct."+ [k ^long interval-ms]+ (let [now (now-ms)]+ (if (< (- now (long (get @state k 0))) interval-ms)+ ;; fast path: still within the window — no write, no allocation+ false+ ;; slow path: window elapsed; try to claim the slot.+ (let [[old _] (swap-vals! state (fn [m]+ (if (>= (- now (long (get m k 0))) interval-ms)+ (assoc m k now)+ m)))]+ (>= (- now (long (get old k 0))) interval-ms)))))diff --git a/test/metabase/util/log/throttle_test.cljc b/test/metabase/util/log/throttle_test.cljcnew file mode 100644index 00000000000..dbcf09e7f6f--- /dev/null+++ b/test/metabase/util/log/throttle_test.cljc@@ -0,0 +1,48 @@+(ns metabase.util.log.throttle-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.util.log.throttle :as log.throttle]))++(deftest allow?-test+ (testing "the first call for a key is allowed, then suppressed until the interval elapses"+ (let [k (str (gensym "k"))]+ (swap! log.throttle/state dissoc k)+ (testing "first call within a fresh window is allowed"+ (is (true? (log.throttle/allow? k 60000))))+ (testing "subsequent calls within the window are suppressed"+ (is (false? (log.throttle/allow? k 60000)))+ (is (false? (log.throttle/allow? k 60000))))+ (testing "once the window has elapsed (stale timestamp), the next call is allowed again"+ (swap! log.throttle/state dissoc k)+ (is (true? (log.throttle/allow? k 60000))))))+ (testing "different keys throttle independently"+ (let [k1 (str (gensym "k")) k2 (str (gensym "k"))]+ (swap! log.throttle/state dissoc k1)+ (swap! log.throttle/state dissoc k2)+ (is (true? (log.throttle/allow? k1 60000)))+ (is (true? (log.throttle/allow? k2 60000)))+ (is (false? (log.throttle/allow? k1 60000)))))+ (testing "a zero interval always allows"+ (let [k (str (gensym "k"))]+ (swap! log.throttle/state dissoc k)+ (is (true? (log.throttle/allow? k 0)))+ (is (true? (log.throttle/allow? k 0)))))+ #?(:clj+ (testing "concurrent callers racing on a fresh key let exactly one through"+ ;; Regression: an earlier implementation tracked the winner in a volatile! that leaked across+ ;; swap! retries, so a thread that lost the CAS could still be allowed, firing twice per window.+ (dotimes [_ 50]+ (let [k (str (gensym "k"))+ n 32+ start (java.util.concurrent.CountDownLatch. 1)+ allowed (atom 0)]+ (swap! log.throttle/state dissoc k)+ (let [futures (vec (for [_ (range n)]+ (future+ (.await start)+ (when (log.throttle/allow? k 60000)+ (swap! allowed inc)))))]+ (.countDown start)+ (run! deref futures))+ (is (= 1 @allowed)+ "exactly one of the racing threads should be allowed through per interval"))))))diff --git a/test/metabase/util/log_test.clj b/test/metabase/util/log_test.cljindex e74616606e9..5b872ccbd42 100644--- a/test/metabase/util/log_test.clj+++ b/test/metabase/util/log_test.clj@@ -1,7 +1,9 @@(ns metabase.util.log-test(:require[clojure.test :refer :all]- [metabase.util.log :as log])+ [metabase.util.log :as log]+ [metabase.util.log.capture :as log.capture]+ [metabase.util.log.throttle :as log.throttle])(:import(org.apache.logging.log4j ThreadContext)))@@ -70,3 +72,31 @@(catch Exception _ nil))(/ 1 0))(catch Exception e e)))))))++(deftest throttle-test+ (testing "log/throttle evaluates body at most once per interval, per call site"+ (testing "repeated calls at one call site within the window log only once"+ (reset! log.throttle/state {})+ (log.capture/with-log-messages-for-level [messages [metabase.util.log-test :error]]+ (dotimes [_ 5]+ (log/throttle 60000+ (log/error "boom")))+ (is (= 1 (count (messages))))))+ (testing "after the window has elapsed (state cleared), the same site logs again"+ (reset! log.throttle/state {})+ (log.capture/with-log-messages-for-level [messages [metabase.util.log-test :error]]+ (log/throttle 60000+ (log/error "boom"))+ (is (= 1 (count (messages))))))+ (testing "the throttled body is skipped entirely — side effects don't run when suppressed"+ (reset! log.throttle/state {})+ (let [calls (atom 0)]+ (dotimes [_ 5]+ (log/throttle 60000+ (swap! calls inc)))+ (is (= 1 @calls))))+ (testing "returns the body value when allowed, nil when throttled"+ (reset! log.throttle/state {})+ (is (= [:ok nil]+ (vec (for [_ (range 2)]+ (log/throttle 60000 :ok))))))))