The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-4651: `SendMeteringEvents` does full table scan of `query_execution` (#83034)
Eric Normand503f96cb55de
diff --git a/src/metabase/internal_stats/db.clj b/src/metabase/internal_stats/db.cljindex 4c1d6c188f5..2c6a14a12ef 100644--- a/src/metabase/internal_stats/db.clj+++ b/src/metabase/internal_stats/db.clj@@ -117,10 +117,14 @@[started-after :- ms/TemporalInstant](t2/select-one query-execution-statistics {:where [:> :started_at started-after]}))-(mu/defn query-execution-statistics-on- "The QueryExecution counts per embedding client for executions started on the day of `date`."- [date :- ms/TemporalInstant]- (t2/select-one query-execution-statistics {:where [:= [:cast :started_at :date] [:cast date :date]]}))+(mu/defn query-execution-statistics-between+ "The QueryExecution counts per embedding client for executions started in `[start, end)`."+ [start :- ms/TemporalInstant+ end :- ms/TemporalInstant]+ ;; A bare range on `started_at` lets the DB use `idx_query_execution_started_at`; casting the column would not.+ (t2/select-one query-execution-statistics {:where [:and+ [:>= :started_at start]+ [:< :started_at end]]}))(defn- and-not-nil([not-nil-field]diff --git a/src/metabase/internal_stats/query_executions.clj b/src/metabase/internal_stats/query_executions.cljindex ea855dbad58..681b38f8eda 100644--- a/src/metabase/internal_stats/query_executions.clj+++ b/src/metabase/internal_stats/query_executions.clj@@ -15,7 +15,7 @@(defn query-execution-last-utc-day"Calculate query executions over a window of the the previous UTC day 00:00-23:59"[]- (let [yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]- (-> (internal-stats.db/query-execution-statistics-on yesterday-utc)+ (let [today-start-utc (t/truncate-to (t/offset-date-time (t/zone-offset "+00")) :days)]+ (-> (internal-stats.db/query-execution-statistics-between (t/minus today-start-utc (t/days 1)) today-start-utc)(dissoc :row_count)(update-keys #(keyword (str "query_executions_" (name %)))))))diff --git a/test/metabase/internal_stats/query_executions_test.clj b/test/metabase/internal_stats/query_executions_test.cljindex 9c21c2f0555..a435b7a4dc0 100644--- a/test/metabase/internal_stats/query_executions_test.clj+++ b/test/metabase/internal_stats/query_executions_test.clj@@ -127,3 +127,17 @@:query_executions_public_link 2:query_executions_internal 1}(sut/query-execution-last-utc-day))))))))++(deftest query-execution-last-utc-day-boundaries-test+ (testing "GHY-4651: the previous UTC day is [yesterday 00:00Z, today 00:00Z), filtered as a range on started_at"+ (t/with-clock (t/mock-clock 1583351015000)+ (let [today-start (t/truncate-to (t/offset-date-time (t/zone-offset "+00")) :days)+ sdk-at (fn [started-at]+ (assoc query-execution-defaults+ :embedding_client "embedding-sdk-react"+ :started_at started-at))]+ (mt/with-temp [:model/QueryExecution _ (sdk-at (t/minus today-start (t/days 1) (t/millis 1)))+ :model/QueryExecution _ (sdk-at (t/minus today-start (t/days 1)))+ :model/QueryExecution _ (sdk-at (t/minus today-start (t/millis 1)))+ :model/QueryExecution _ (sdk-at today-start)]+ (is (= 2 (:query_executions_sdk_embed (sut/query-execution-last-utc-day)))))))))diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex ee6370a50dc..6db0e1062e1 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -17,7 +17,9 @@[metabase.util :as u][metabase.util.json :as json][metabase.util.malli.registry :as mr]- [toucan2.core :as t2]))+ [methodical.core :as methodical]+ [toucan2.core :as t2]+ [toucan2.pipeline :as t2.pipeline]))(set! *warn-on-reflection* true)@@ -493,6 +495,38 @@(is (contains? stats :embedding-dashboard-count))(is (contains? stats :embedding-question-count)))))+(defn- capture-compiled-queries+ "Call `f` and return every compiled `[sql & params]` query it ran against the app DB."+ [f]+ (let [queries (atom [])]+ (methodical/add-aux-method-with-unique-key!+ #'t2.pipeline/transduce-execute-with-connection+ :around :default+ (fn [next-method rf conn query-type model query]+ (swap! queries conj query)+ (next-method rf conn query-type model query))+ ::query-capture)+ (try+ (f)+ (finally+ (methodical/remove-aux-method-with-unique-key!+ #'t2.pipeline/transduce-execute-with-connection+ :around :default+ ::query-capture)))+ @queries))++(deftest metering-stats-uses-started-at-index-test+ (testing "GHY-4651: the metering query on query_execution must use idx_query_execution_started_at, not a full table scan.+ SendMeteringEvents runs this every 15 minutes, and query_execution can hold millions of rows."+ ;; H2 names the access path in its EXPLAIN output, so the check does not depend on table size or planner costs.+ (when (= :h2 (mdb/db-type))+ (let [qe-queries (->> (capture-compiled-queries token-check/metering-stats)+ (filter (fn [[sql]] (re-find #"(?i)from\s+\"?query_execution\"?" sql))))]+ (is (seq qe-queries))+ (doseq [[sql & params] qe-queries+ :let [plan (-> (t2/query (into [(str "EXPLAIN " sql)] params)) first :plan)]]+ (is (re-find #"IDX_QUERY_EXECUTION_STARTED_AT: STARTED_AT" plan)))))))+;;; ------------------------------------------------ db-hash-aware-token-checker ------------------------------------------------(defn- make-db-hash-aware-checker