The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix: `core.async/timeout` combined with `core.async/go` prevented fat dynamic var GC (#75769)
Timothy S. Deane11255220d3a
diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.cljindex b2fc7ebc19f..d199c79167d 100644--- a/src/metabase/query_processor/setup.clj+++ b/src/metabase/query_processor/setup.clj@@ -22,7 +22,12 @@[metabase.util.log :as log][metabase.util.malli :as mu]^{:clj-kondo/ignore [:discouraged-namespace]}- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent ScheduledFuture ScheduledThreadPoolExecutor ThreadFactory TimeUnit)+ (org.apache.logging.log4j ThreadContext)))++(set! *warn-on-reflection* true)(mu/defn- query-type :- [:enum :query :native :internal :mbql/query][query :- ::qp.schema/any-query]@@ -188,32 +193,60 @@(setting/with-database db(f query))))))+(defonce ^:private query-timeout-executor+ ;; one daemon thread for the whole JVM; the scheduled work is a single non-blocking `a/put!`.+ ;; `setRemoveOnCancelPolicy` makes cancelled tasks leave the queue immediately instead of at their deadline.+ (delay+ (doto (ScheduledThreadPoolExecutor. 1+ (reify ThreadFactory+ (newThread [_ r]+ (doto (Thread. ^Runnable r "query-timeout-scheduler")+ (.setDaemon true)))))+ (.setRemoveOnCancelPolicy true))))+(defn- schedule-query-timeout-cancel!"Schedule a put of `::timeout` to `canceled-chan` after `*query-timeout-ms*`. Driver execution wires this channelto `Statement.cancel()` (see [[metabase.driver.sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement!]]),which for MySQL/MariaDB issues `KILL QUERY` on a side connection — so this is the cross-driver path that actually- stops a running query on the server when the timeout fires. `done-ch` closes when the query completes, releasing- the go-block early. The timer never reads from `canceled-chan`, since callers (notably tests) may bind a regular- channel where `alts!`/`<!` would consume the cancel signal away from the driver's listener."- [canceled-chan done-ch]- (let [timeout-ms (long driver.settings/*query-timeout-ms*)]- (a/go- (let [timeout-ch (a/timeout timeout-ms)- [_ port] (a/alts! [done-ch timeout-ch])]- (when (identical? port timeout-ch)- (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)- (a/put! canceled-chan ::timeout))))))+ stops a running query on the server when the timeout fires. The timer never reads from `canceled-chan`, since+ callers (notably tests) may bind a regular channel where `alts!`/`<!` would consume the cancel signal away from+ the driver's listener.++ Returns a `ScheduledFuture`; the caller must cancel it when the query completes."+ ^ScheduledFuture [canceled-chan]+ (let [timeout-ms (long driver.settings/*query-timeout-ms*)+ ;; capture the log4j ThreadContext (an immutable map of strings) at schedule time: the warn below fires on+ ;; the scheduler thread, which otherwise has no query/job attribution for log filtering. Strings only — the+ ;; closure must stay cheap, since it lives until the timeout fires or the query completes.+ log-context (ThreadContext/getImmutableContext)]+ ;; Deliberately a plain fn on a Java scheduler rather than an `a/go` block: `go` conveys the dynamic binding+ ;; frame (including the bound metadata provider and its cache) into a handler parked on an uncancellable+ ;; `a/timeout` channel, pinning ~1MB per QP invocation in the static timer queue until the deadline — which+ ;; OOMed instances under sustained query rates (#75748). A plain fn captures only `canceled-chan` and the+ ;; context strings above, and cancelling the future releases even those as soon as the query finishes.+ (.schedule ^ScheduledThreadPoolExecutor @query-timeout-executor+ ^Runnable (fn []+ ;; `put!` returns false if the chan is already closed (the pipeline closes it on+ ;; successful reduction) — don't log a spurious warning for a completed query.+ (when (a/put! canceled-chan ::timeout)+ ;; the scheduler thread runs nothing else, so there is no prior context to preserve+ (ThreadContext/putAll log-context)+ (try+ (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)+ (finally+ (ThreadContext/clearMap)))))+ timeout-ms+ TimeUnit/MILLISECONDS)))(mu/defn- do-with-canceled-chan :- fn?[f :- [:=> [:cat ::qp.schema/any-query] :any]](fn [query]- (let [done-ch (a/promise-chan)]- (try- (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]- (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan* done-ch)- (f query))- (finally- (a/put! done-ch ::query-finished))))))+ (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]+ (let [timeout-task (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan*)]+ (try+ (f query)+ (finally+ (.cancel timeout-task false)))))))(def ^:private setup-middleware"Setup middleware has the signaturediff --git a/test/metabase/query_processor/setup_test.clj b/test/metabase/query_processor/setup_test.cljindex 60774cdf924..f6f10ef8d1f 100644--- a/test/metabase/query_processor/setup_test.clj+++ b/test/metabase/query_processor/setup_test.clj@@ -3,8 +3,11 @@[clojure.core.async :as a][clojure.test :refer :all][metabase.driver.settings :as driver.settings]+ [metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]+ [metabase.lib.test-metadata :as meta][metabase.query-processor.pipeline :as qp.pipeline]- [metabase.query-processor.setup :as qp.setup]))+ [metabase.query-processor.setup :as qp.setup]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]))(set! *warn-on-reflection* true)@@ -52,6 +55,52 @@(is (nil? @observed-after-close)"a closed chan should poll nil — the timer must not have written ::timeout to it")))))+;; not ^:parallel: relies on (System/gc), which pauses the whole JVM and would distort parallel siblings' timing.+(deftest completed-query-releases-metadata-provider-test+ (testing "metadata providers must be GC-able once their QP invocation completes (#75748)"+ ;; Regression guard for #75748: the query-timeout timer originally (#74776) spawned an `a/go` block, which+ ;; conveyed the thread's dynamic binding frame — including the bound metadata provider and everything it had+ ;; cached — into a parked handler on an `a/timeout` channel. That channel lives in core.async's static timer+ ;; queue (https://github.com/clojure/core.async/blob/master/src/main/clojure/clojure/core/async/impl/timers.clj)+ ;; for the full *query-timeout-ms* regardless of when the query completes, so every QP invocation pinned+ ;; ~1MB until the deadline; under sustained call rates this OOMed instances. Whatever the timer's+ ;; implementation, it must not retain the binding frame, and must release anything it does retain as soon as+ ;; the query completes.+ (let [capture! (fn []+ ;; fresh wrapper per invocation so each provider's reachability is independently trackable+ ;; (the underlying mock is a global and would never be collectable)+ (let [mp (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider)+ wr (java.lang.ref.WeakReference. mp)]+ (qp.store/with-metadata-provider mp+ (let [f (#'qp.setup/do-with-canceled-chan+ (fn [_query]+ ;; realistic duration: ensures the timer go-block parks before we complete, as+ ;; it would for any real query. Sub-ms bodies race go-block dispatch and make+ ;; retention nondeterministic.+ (Thread/sleep 50)+ ::done))]+ (f {:type :internal})))+ wr))+ wrs (vec (repeatedly 5 capture!))+ ;; unrelated to the timer: the single most recently bound provider can stay reachable through what is+ ;; likely a stack-slot/register GC root (not fully diagnosed). Claim that slot with a throwaway provider+ ;; so it cannot hold any of the five we count.+ _ (qp.store/with-metadata-provider+ (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider)+ ::nothing)+ alive (fn []+ (count (keep #(.get ^java.lang.ref.WeakReference %) wrs)))]+ ;; a single System/gc can be lazy about clearing weak refs; retry briefly until they clear or we give up.+ ;; (Relies on explicit GC being honored — this would fail spuriously under -XX:+DisableExplicitGC, which our+ ;; CI does not set.)+ (loop [n 0]+ (when (and (pos? (alive)) (< n 20))+ (System/gc)+ (Thread/sleep 50)+ (recur (inc n))))+ (is (zero? (alive))+ "completed QP invocations still pin their metadata providers via the query-timeout timer"))))+(deftest ^:parallel canceled-chan-timer-does-not-consume-external-cancel-signal-test(testing "Timer must not consume the cancel signal when callers bind a regular (a/chan)";; Regression for mongo `kill-an-in-flight-query-test`: that test binds `*canceled-chan*` to a regular