The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix flaky test caused by noise from batched query executions from other deftests (#80943)
Ben Grabowf5ac3e3c6433
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex f6f4da08394..632d7551600 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -32,7 +32,6 @@:missing-protocol-method 3:non-arg-vec-return-type-hint 1:reduce-without-init 3- :redundant-fn-wrapper 1:syntax 2:uninitialized-var 1:unquote-not-syntax-quoted 1diff --git a/src/metabase/batch_processing/core.clj b/src/metabase/batch_processing/core.cljindex 73343181515..7a3dcf75da1 100644--- a/src/metabase/batch_processing/core.clj+++ b/src/metabase/batch_processing/core.clj@@ -7,6 +7,7 @@(p/import-vars[metabase.batch-processing.impl+ flush!shutdown!start!submit!])diff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.cljindex 83df5dbb128..82e78093604 100644--- a/src/metabase/batch_processing/impl.clj+++ b/src/metabase/batch_processing/impl.clj@@ -33,13 +33,18 @@[:f ::fn-or-var][:grouper (ms/InstanceOfClass Grouper)]])+(def ^:private flush-sentinel+ "Submitted by [[flush!]] purely to get a promise that is delivered once the batch it lands in has been processed.+ Stripped out before `f` sees it."+ ::flush)+(mu/defn start! :- ::grouper-wrapper"Wrapper around [[grouper/start!]]."[f :- ::fn-or-var & options];; this wrapper is so we can use Vars which Grouper normally doesn't allow.- #_{:clj-kondo/ignore [:redundant-fn-wrapper]}(let [f* (fn [items]- (f items))+ (when-let [items (not-empty (remove #{flush-sentinel} items))]+ (f items)))grouper (apply grouper/start! f* options)]{:f f, :grouper grouper}))@@ -48,6 +53,19 @@[grouper :- ::grouper-wrapper](grouper/shutdown! (:grouper grouper)))+(mu/defn flush!+ "Block until everything submitted to `grouper-wrapper` before this call has been processed. Batches are otherwise+ only processed once the queue fills up or the interval elapses, so callers that need to observe the effects of their+ own submissions have to flush first."+ [grouper-wrapper :- ::grouper-wrapper]+ (let [grouper (:grouper grouper-wrapper)+ ;; submit before waking the dispatcher: the sentinel is then drained in the same batch as everything already+ ;; queued, and its promise is delivered only after that whole batch has been processed.+ result (grouper/submit! grouper flush-sentinel)]+ (.wakeUp ^Grouper grouper)+ @result+ nil))+(mu/defn submit!"A wrapper of [[grouper.core/submit!]] that returns nil instead of a promise.We use grouper for fire-and-forget scenarios, so we don't care about the result."diff --git a/src/metabase/query_processor/middleware/process_userland_query.clj b/src/metabase/query_processor/middleware/process_userland_query.cljindex a2e25fbbd6b..16449edee30 100644--- a/src/metabase/query_processor/middleware/process_userland_query.clj+++ b/src/metabase/query_processor/middleware/process_userland_query.clj@@ -89,6 +89,12 @@(grouper/submit! @save-execution-metadata-queue execution-info')(save-execution-metadata!* [execution-info']))))+(defn flush-execution-metadata!+ "Block until every `QueryExecution` submitted by [[save-execution-metadata!]] so far has been written. Needed by+ anything that reads the `query_execution` table back and cannot tolerate the batching lag."+ []+ (grouper/flush! @save-execution-metadata-queue))+(defn- save-successful-execution-metadata! [cache-details is-sandboxed? query-execution result-rows](let [qe-map (assoc query-execution:cache_hit (boolean (:cached cache-details))diff --git a/test/metabase/batch_processing/impl_test.clj b/test/metabase/batch_processing/impl_test.cljindex 8713300ec07..d5749aa175b 100644--- a/test/metabase/batch_processing/impl_test.clj+++ b/test/metabase/batch_processing/impl_test.clj@@ -17,3 +17,25 @@(u/with-timeout 1000(grouper/submit! g 1))(is (= [1] @processed?))))))++(deftest flush-test+ (testing "flush! processes everything already submitted, without exposing its sentinel to the batch fn"+ (let [batches (atom [])+ g (grouper/start!+ (fn [items]+ (swap! batches conj (vec items)))+ :capacity 5+ :interval (* 60 1000))]+ (try+ (grouper/submit! g 1)+ (grouper/submit! g 2)+ (is (= [] @batches))+ (u/with-timeout 5000+ (grouper/flush! g))+ (is (= [[1 2]] @batches))+ (testing "a flush with nothing queued does not call the batch fn"+ (u/with-timeout 5000+ (grouper/flush! g))+ (is (= [[1 2]] @batches)))+ (finally+ (grouper/shutdown! g))))))diff --git a/test/metabase/internal_stats/query_executions_test.clj b/test/metabase/internal_stats/query_executions_test.cljindex fb9d8a2efd0..9c21c2f0555 100644--- a/test/metabase/internal_stats/query_executions_test.clj+++ b/test/metabase/internal_stats/query_executions_test.clj@@ -3,6 +3,7 @@[clojure.test :refer [deftest testing is]][java-time.api :as t][metabase.internal-stats.query-executions :as sut]+ [metabase.query-processor.middleware.process-userland-query :as process-userland-query][metabase.query-processor.util :as qp.util][metabase.test :as mt][metabase.util :as u]))@@ -19,6 +20,9 @@(deftest query-execution-24h-filtering-test(mt/initialize-if-needed! :db)+ ;; `QueryExecution`s are written in batches on a background thread, so rows from earlier tests can land between the+ ;; `before` and `after` snapshots below and inflate the `:internal` counts.+ (process-userland-query/flush-execution-metadata!)(t/with-clock (t/mock-clock 1583351015000)(let [before (sut/query-executions-all-time-and-last-24h)one-year-ago-defaults (assoc query-execution-defaults