Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix flaky test caused by noise from batched query executions from other deftests (#80943)

Ben Grabowf5ac3e3c6433
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index f6f4da08394..632d7551600 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -32,7 +32,6 @@
:missing-protocol-method 3
:non-arg-vec-return-type-hint 1
:reduce-without-init 3
- :redundant-fn-wrapper 1
:syntax 2
:uninitialized-var 1
:unquote-not-syntax-quoted 1
diff --git a/src/metabase/batch_processing/core.clj b/src/metabase/batch_processing/core.clj
index 73343181515..7a3dcf75da1 100644
--- a/src/metabase/batch_processing/core.clj
+++ b/src/metabase/batch_processing/core.clj
@@ -7,6 +7,7 @@
(p/import-vars
[metabase.batch-processing.impl
+ flush!
shutdown!
start!
submit!])
diff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.clj
index 83df5dbb128..82e78093604 100644
--- a/src/metabase/batch_processing/impl.clj
+++ b/src/metabase/batch_processing/impl.clj
@@ -33,13 +33,18 @@
[:f ::fn-or-var]
[:grouper (ms/InstanceOfClass Grouper)]])
+(def ^:private flush-sentinel
+ "Submitted by [[flush!]] purely to get a promise that is delivered once the batch it lands in has been processed.
+ Stripped out before `f` sees it."
+ ::flush)
+
(mu/defn start! :- ::grouper-wrapper
"Wrapper around [[grouper/start!]]."
[f :- ::fn-or-var & options]
;; this wrapper is so we can use Vars which Grouper normally doesn't allow.
- #_{:clj-kondo/ignore [:redundant-fn-wrapper]}
(let [f* (fn [items]
- (f items))
+ (when-let [items (not-empty (remove #{flush-sentinel} items))]
+ (f items)))
grouper (apply grouper/start! f* options)]
{:f f, :grouper grouper}))
@@ -48,6 +53,19 @@
[grouper :- ::grouper-wrapper]
(grouper/shutdown! (:grouper grouper)))
+(mu/defn flush!
+ "Block until everything submitted to `grouper-wrapper` before this call has been processed. Batches are otherwise
+ only processed once the queue fills up or the interval elapses, so callers that need to observe the effects of their
+ own submissions have to flush first."
+ [grouper-wrapper :- ::grouper-wrapper]
+ (let [grouper (:grouper grouper-wrapper)
+ ;; submit before waking the dispatcher: the sentinel is then drained in the same batch as everything already
+ ;; queued, and its promise is delivered only after that whole batch has been processed.
+ result (grouper/submit! grouper flush-sentinel)]
+ (.wakeUp ^Grouper grouper)
+ @result
+ nil))
+
(mu/defn submit!
"A wrapper of [[grouper.core/submit!]] that returns nil instead of a promise.
We use grouper for fire-and-forget scenarios, so we don't care about the result."
diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.clj
index a2e25fbbd6b..16449edee30 100644
--- a/src/metabase/query_processor/middleware/process_userland_query.clj
+++ b/src/metabase/query_processor/middleware/process_userland_query.clj
@@ -89,6 +89,12 @@
(grouper/submit! @save-execution-metadata-queue execution-info')
(save-execution-metadata!* [execution-info']))))
+(defn flush-execution-metadata!
+ "Block until every `QueryExecution` submitted by [[save-execution-metadata!]] so far has been written. Needed by
+ anything that reads the `query_execution` table back and cannot tolerate the batching lag."
+ []
+ (grouper/flush! @save-execution-metadata-queue))
+
(defn- save-successful-execution-metadata! [cache-details is-sandboxed? query-execution result-rows]
(let [qe-map (assoc query-execution
:cache_hit (boolean (:cached cache-details))
diff --git a/test/metabase/batch_processing/impl_test.clj b/test/metabase/batch_processing/impl_test.clj
index 8713300ec07..d5749aa175b 100644
--- a/test/metabase/batch_processing/impl_test.clj
+++ b/test/metabase/batch_processing/impl_test.clj
@@ -17,3 +17,25 @@
(u/with-timeout 1000
(grouper/submit! g 1))
(is (= [1] @processed?))))))
+
+(deftest flush-test
+ (testing "flush! processes everything already submitted, without exposing its sentinel to the batch fn"
+ (let [batches (atom [])
+ g (grouper/start!
+ (fn [items]
+ (swap! batches conj (vec items)))
+ :capacity 5
+ :interval (* 60 1000))]
+ (try
+ (grouper/submit! g 1)
+ (grouper/submit! g 2)
+ (is (= [] @batches))
+ (u/with-timeout 5000
+ (grouper/flush! g))
+ (is (= [[1 2]] @batches))
+ (testing "a flush with nothing queued does not call the batch fn"
+ (u/with-timeout 5000
+ (grouper/flush! g))
+ (is (= [[1 2]] @batches)))
+ (finally
+ (grouper/shutdown! g))))))
diff --git a/test/metabase/internal_stats/query_executions_test.clj b/test/metabase/internal_stats/query_executions_test.clj
index fb9d8a2efd0..9c21c2f0555 100644
--- a/test/metabase/internal_stats/query_executions_test.clj
+++ b/test/metabase/internal_stats/query_executions_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer [deftest testing is]]
[java-time.api :as t]
[metabase.internal-stats.query-executions :as sut]
+ [metabase.query-processor.middleware.process-userland-query :as process-userland-query]
[metabase.query-processor.util :as qp.util]
[metabase.test :as mt]
[metabase.util :as u]))
@@ -19,6 +20,9 @@
(deftest query-execution-24h-filtering-test
(mt/initialize-if-needed! :db)
+ ;; `QueryExecution`s are written in batches on a background thread, so rows from earlier tests can land between the
+ ;; `before` and `after` snapshots below and inflate the `:internal` counts.
+ (process-userland-query/flush-execution-metadata!)
(t/with-clock (t/mock-clock 1583351015000)
(let [before (sut/query-executions-all-time-and-last-24h)
one-year-ago-defaults (assoc query-execution-defaults