Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-4651: `SendMeteringEvents` does full table scan of `query_execution` (#83034)

Eric Normand503f96cb55de
diff --git a/src/metabase/internal_stats/db.clj b/src/metabase/internal_stats/db.clj
index 4c1d6c188f5..2c6a14a12ef 100644
--- a/src/metabase/internal_stats/db.clj
+++ b/src/metabase/internal_stats/db.clj
@@ -117,10 +117,14 @@
[started-after :- ms/TemporalInstant]
(t2/select-one query-execution-statistics {:where [:> :started_at started-after]}))
-(mu/defn query-execution-statistics-on
- "The QueryExecution counts per embedding client for executions started on the day of `date`."
- [date :- ms/TemporalInstant]
- (t2/select-one query-execution-statistics {:where [:= [:cast :started_at :date] [:cast date :date]]}))
+(mu/defn query-execution-statistics-between
+ "The QueryExecution counts per embedding client for executions started in `[start, end)`."
+ [start :- ms/TemporalInstant
+ end :- ms/TemporalInstant]
+ ;; A bare range on `started_at` lets the DB use `idx_query_execution_started_at`; casting the column would not.
+ (t2/select-one query-execution-statistics {:where [:and
+ [:>= :started_at start]
+ [:< :started_at end]]}))
(defn- and-not-nil
([not-nil-field]
diff --git a/src/metabase/internal_stats/query_executions.clj b/src/metabase/internal_stats/query_executions.clj
index ea855dbad58..681b38f8eda 100644
--- a/src/metabase/internal_stats/query_executions.clj
+++ b/src/metabase/internal_stats/query_executions.clj
@@ -15,7 +15,7 @@
(defn query-execution-last-utc-day
"Calculate query executions over a window of the the previous UTC day 00:00-23:59"
[]
- (let [yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
- (-> (internal-stats.db/query-execution-statistics-on yesterday-utc)
+ (let [today-start-utc (t/truncate-to (t/offset-date-time (t/zone-offset "+00")) :days)]
+ (-> (internal-stats.db/query-execution-statistics-between (t/minus today-start-utc (t/days 1)) today-start-utc)
(dissoc :row_count)
(update-keys #(keyword (str "query_executions_" (name %)))))))
diff --git a/test/metabase/internal_stats/query_executions_test.clj b/test/metabase/internal_stats/query_executions_test.clj
index 9c21c2f0555..a435b7a4dc0 100644
--- a/test/metabase/internal_stats/query_executions_test.clj
+++ b/test/metabase/internal_stats/query_executions_test.clj
@@ -127,3 +127,17 @@
:query_executions_public_link 2
:query_executions_internal 1}
(sut/query-execution-last-utc-day))))))))
+
+(deftest query-execution-last-utc-day-boundaries-test
+ (testing "GHY-4651: the previous UTC day is [yesterday 00:00Z, today 00:00Z), filtered as a range on started_at"
+ (t/with-clock (t/mock-clock 1583351015000)
+ (let [today-start (t/truncate-to (t/offset-date-time (t/zone-offset "+00")) :days)
+ sdk-at (fn [started-at]
+ (assoc query-execution-defaults
+ :embedding_client "embedding-sdk-react"
+ :started_at started-at))]
+ (mt/with-temp [:model/QueryExecution _ (sdk-at (t/minus today-start (t/days 1) (t/millis 1)))
+ :model/QueryExecution _ (sdk-at (t/minus today-start (t/days 1)))
+ :model/QueryExecution _ (sdk-at (t/minus today-start (t/millis 1)))
+ :model/QueryExecution _ (sdk-at today-start)]
+ (is (= 2 (:query_executions_sdk_embed (sut/query-execution-last-utc-day)))))))))
diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.clj
index ee6370a50dc..6db0e1062e1 100644
--- a/test/metabase/premium_features/token_check_test.clj
+++ b/test/metabase/premium_features/token_check_test.clj
@@ -17,7 +17,9 @@
[metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
- [toucan2.core :as t2]))
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]
+ [toucan2.pipeline :as t2.pipeline]))
(set! *warn-on-reflection* true)
@@ -493,6 +495,38 @@
(is (contains? stats :embedding-dashboard-count))
(is (contains? stats :embedding-question-count)))))
+(defn- capture-compiled-queries
+ "Call `f` and return every compiled `[sql & params]` query it ran against the app DB."
+ [f]
+ (let [queries (atom [])]
+ (methodical/add-aux-method-with-unique-key!
+ #'t2.pipeline/transduce-execute-with-connection
+ :around :default
+ (fn [next-method rf conn query-type model query]
+ (swap! queries conj query)
+ (next-method rf conn query-type model query))
+ ::query-capture)
+ (try
+ (f)
+ (finally
+ (methodical/remove-aux-method-with-unique-key!
+ #'t2.pipeline/transduce-execute-with-connection
+ :around :default
+ ::query-capture)))
+ @queries))
+
+(deftest metering-stats-uses-started-at-index-test
+ (testing "GHY-4651: the metering query on query_execution must use idx_query_execution_started_at, not a full table scan.
+ SendMeteringEvents runs this every 15 minutes, and query_execution can hold millions of rows."
+ ;; H2 names the access path in its EXPLAIN output, so the check does not depend on table size or planner costs.
+ (when (= :h2 (mdb/db-type))
+ (let [qe-queries (->> (capture-compiled-queries token-check/metering-stats)
+ (filter (fn [[sql]] (re-find #"(?i)from\s+\"?query_execution\"?" sql))))]
+ (is (seq qe-queries))
+ (doseq [[sql & params] qe-queries
+ :let [plan (-> (t2/query (into [(str "EXPLAIN " sql)] params)) first :plan)]]
+ (is (re-find #"IDX_QUERY_EXECUTION_STARTED_AT: STARTED_AT" plan)))))))
+
;;; ------------------------------------------------ db-hash-aware-token-checker ------------------------------------------------
(defn- make-db-hash-aware-checker