Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix: `core.async/timeout` combined with `core.async/go` prevented fat dynamic var GC (#75769)

Timothy S. Deane11255220d3a
diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.clj
index b2fc7ebc19f..d199c79167d 100644
--- a/src/metabase/query_processor/setup.clj
+++ b/src/metabase/query_processor/setup.clj
@@ -22,7 +22,12 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
^{:clj-kondo/ignore [:discouraged-namespace]}
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent ScheduledFuture ScheduledThreadPoolExecutor ThreadFactory TimeUnit)
+ (org.apache.logging.log4j ThreadContext)))
+
+(set! *warn-on-reflection* true)
(mu/defn- query-type :- [:enum :query :native :internal :mbql/query]
[query :- ::qp.schema/any-query]
@@ -188,32 +193,60 @@
(setting/with-database db
(f query))))))
+(defonce ^:private query-timeout-executor
+ ;; one daemon thread for the whole JVM; the scheduled work is a single non-blocking `a/put!`.
+ ;; `setRemoveOnCancelPolicy` makes cancelled tasks leave the queue immediately instead of at their deadline.
+ (delay
+ (doto (ScheduledThreadPoolExecutor. 1
+ (reify ThreadFactory
+ (newThread [_ r]
+ (doto (Thread. ^Runnable r "query-timeout-scheduler")
+ (.setDaemon true)))))
+ (.setRemoveOnCancelPolicy true))))
+
(defn- schedule-query-timeout-cancel!
"Schedule a put of `::timeout` to `canceled-chan` after `*query-timeout-ms*`. Driver execution wires this channel
to `Statement.cancel()` (see [[metabase.driver.sql-jdbc.execute/wire-up-canceled-chan-to-cancel-Statement!]]),
which for MySQL/MariaDB issues `KILL QUERY` on a side connection — so this is the cross-driver path that actually
- stops a running query on the server when the timeout fires. `done-ch` closes when the query completes, releasing
- the go-block early. The timer never reads from `canceled-chan`, since callers (notably tests) may bind a regular
- channel where `alts!`/`<!` would consume the cancel signal away from the driver's listener."
- [canceled-chan done-ch]
- (let [timeout-ms (long driver.settings/*query-timeout-ms*)]
- (a/go
- (let [timeout-ch (a/timeout timeout-ms)
- [_ port] (a/alts! [done-ch timeout-ch])]
- (when (identical? port timeout-ch)
- (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)
- (a/put! canceled-chan ::timeout))))))
+ stops a running query on the server when the timeout fires. The timer never reads from `canceled-chan`, since
+ callers (notably tests) may bind a regular channel where `alts!`/`<!` would consume the cancel signal away from
+ the driver's listener.
+
+ Returns a `ScheduledFuture`; the caller must cancel it when the query completes."
+ ^ScheduledFuture [canceled-chan]
+ (let [timeout-ms (long driver.settings/*query-timeout-ms*)
+ ;; capture the log4j ThreadContext (an immutable map of strings) at schedule time: the warn below fires on
+ ;; the scheduler thread, which otherwise has no query/job attribution for log filtering. Strings only — the
+ ;; closure must stay cheap, since it lives until the timeout fires or the query completes.
+ log-context (ThreadContext/getImmutableContext)]
+ ;; Deliberately a plain fn on a Java scheduler rather than an `a/go` block: `go` conveys the dynamic binding
+ ;; frame (including the bound metadata provider and its cache) into a handler parked on an uncancellable
+ ;; `a/timeout` channel, pinning ~1MB per QP invocation in the static timer queue until the deadline — which
+ ;; OOMed instances under sustained query rates (#75748). A plain fn captures only `canceled-chan` and the
+ ;; context strings above, and cancelling the future releases even those as soon as the query finishes.
+ (.schedule ^ScheduledThreadPoolExecutor @query-timeout-executor
+ ^Runnable (fn []
+ ;; `put!` returns false if the chan is already closed (the pipeline closes it on
+ ;; successful reduction) — don't log a spurious warning for a completed query.
+ (when (a/put! canceled-chan ::timeout)
+ ;; the scheduler thread runs nothing else, so there is no prior context to preserve
+ (ThreadContext/putAll log-context)
+ (try
+ (log/warnf "Query exceeded timeout of %d ms; canceling" timeout-ms)
+ (finally
+ (ThreadContext/clearMap)))))
+ timeout-ms
+ TimeUnit/MILLISECONDS)))
(mu/defn- do-with-canceled-chan :- fn?
[f :- [:=> [:cat ::qp.schema/any-query] :any]]
(fn [query]
- (let [done-ch (a/promise-chan)]
- (try
- (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]
- (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan* done-ch)
- (f query))
- (finally
- (a/put! done-ch ::query-finished))))))
+ (binding [qp.pipeline/*canceled-chan* (or qp.pipeline/*canceled-chan* (a/promise-chan))]
+ (let [timeout-task (schedule-query-timeout-cancel! qp.pipeline/*canceled-chan*)]
+ (try
+ (f query)
+ (finally
+ (.cancel timeout-task false)))))))
(def ^:private setup-middleware
"Setup middleware has the signature
diff --git a/test/metabase/query_processor/setup_test.clj b/test/metabase/query_processor/setup_test.clj
index 60774cdf924..f6f10ef8d1f 100644
--- a/test/metabase/query_processor/setup_test.clj
+++ b/test/metabase/query_processor/setup_test.clj
@@ -3,8 +3,11 @@
[clojure.core.async :as a]
[clojure.test :refer :all]
[metabase.driver.settings :as driver.settings]
+ [metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]
+ [metabase.lib.test-metadata :as meta]
[metabase.query-processor.pipeline :as qp.pipeline]
- [metabase.query-processor.setup :as qp.setup]))
+ [metabase.query-processor.setup :as qp.setup]
+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]))
(set! *warn-on-reflection* true)
@@ -52,6 +55,52 @@
(is (nil? @observed-after-close)
"a closed chan should poll nil — the timer must not have written ::timeout to it")))))
+;; not ^:parallel: relies on (System/gc), which pauses the whole JVM and would distort parallel siblings' timing.
+(deftest completed-query-releases-metadata-provider-test
+ (testing "metadata providers must be GC-able once their QP invocation completes (#75748)"
+ ;; Regression guard for #75748: the query-timeout timer originally (#74776) spawned an `a/go` block, which
+ ;; conveyed the thread's dynamic binding frame — including the bound metadata provider and everything it had
+ ;; cached — into a parked handler on an `a/timeout` channel. That channel lives in core.async's static timer
+ ;; queue (https://github.com/clojure/core.async/blob/master/src/main/clojure/clojure/core/async/impl/timers.clj)
+ ;; for the full *query-timeout-ms* regardless of when the query completes, so every QP invocation pinned
+ ;; ~1MB until the deadline; under sustained call rates this OOMed instances. Whatever the timer's
+ ;; implementation, it must not retain the binding frame, and must release anything it does retain as soon as
+ ;; the query completes.
+ (let [capture! (fn []
+ ;; fresh wrapper per invocation so each provider's reachability is independently trackable
+ ;; (the underlying mock is a global and would never be collectable)
+ (let [mp (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider)
+ wr (java.lang.ref.WeakReference. mp)]
+ (qp.store/with-metadata-provider mp
+ (let [f (#'qp.setup/do-with-canceled-chan
+ (fn [_query]
+ ;; realistic duration: ensures the timer go-block parks before we complete, as
+ ;; it would for any real query. Sub-ms bodies race go-block dispatch and make
+ ;; retention nondeterministic.
+ (Thread/sleep 50)
+ ::done))]
+ (f {:type :internal})))
+ wr))
+ wrs (vec (repeatedly 5 capture!))
+ ;; unrelated to the timer: the single most recently bound provider can stay reachable through what is
+ ;; likely a stack-slot/register GC root (not fully diagnosed). Claim that slot with a throwaway provider
+ ;; so it cannot hold any of the five we count.
+ _ (qp.store/with-metadata-provider
+ (lib.metadata.cached-provider/cached-metadata-provider meta/metadata-provider)
+ ::nothing)
+ alive (fn []
+ (count (keep #(.get ^java.lang.ref.WeakReference %) wrs)))]
+ ;; a single System/gc can be lazy about clearing weak refs; retry briefly until they clear or we give up.
+ ;; (Relies on explicit GC being honored — this would fail spuriously under -XX:+DisableExplicitGC, which our
+ ;; CI does not set.)
+ (loop [n 0]
+ (when (and (pos? (alive)) (< n 20))
+ (System/gc)
+ (Thread/sleep 50)
+ (recur (inc n))))
+ (is (zero? (alive))
+ "completed QP invocations still pin their metadata providers via the query-timeout timer"))))
+
(deftest ^:parallel canceled-chan-timer-does-not-consume-external-cancel-signal-test
(testing "Timer must not consume the cancel signal when callers bind a regular (a/chan)"
;; Regression for mongo `kill-an-in-flight-query-test`: that test binds `*canceled-chan*` to a regular