The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[perf] Expand the `qp.perf-test` benchmark suite (#65806)
Braden Shepherdson413b32e2779d
diff --git a/test/metabase/query_processor/perf_test.clj b/test/metabase/query_processor/perf_test.cljindex 37b2a15a3a0..893f81490e0 100644--- a/test/metabase/query_processor/perf_test.clj+++ b/test/metabase/query_processor/perf_test.clj@@ -1,23 +1,16 @@(ns metabase.query-processor.perf-test(:require[criterium.core :as crit]- [metabase.lib.computed :as lib.computed][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]+ [metabase.lib.options :as lib.options][metabase.lib.test-metadata :as meta]- [metabase.lib.test-util :as lib.tu][metabase.lib.test-util.huge-query-metadata-providers :as lib.tu.huge][metabase.lib.test-util.metadata-providers.mock :as lib.tu.mock][metabase.query-processor.compile :as qp.compile][metabase.util.malli :as mu]))- ;; TODO: (Braden 10/31/2025) Benchmarks to add:- ;; - Dummy stages, eg. a bonus SELECT * stage on top- ;; - What about 4 stages?- ;; - Return only a few cols from a huge table- ;; - Big daisy-chain of joins-(def ^:private ids-per-table 100000)(def ^:private db-id 1111)@@ -27,7 +20,9 @@field-index)))(defn- mock-metadata- "5 tables with 10 columns each."+ "Creates N tables with M columns each.++ The first column is a PK `ID` and all the others are `column_1` `:type/Integer` tables."[n-tables fields-each](let [tables (for [index (range 1 (inc n-tables))](merge (meta/table-metadata :orders)@@ -85,83 +80,213 @@[mp](lib/query mp (lib.metadata/table mp (id 1))))+(defn- simple-count+ [mp]+ (-> (lib/query mp (lib.metadata/table mp (id 1)))+ (lib/aggregate (lib/count))))++(defn- stage+count+ [mp]+ (-> (lib/query mp (lib.metadata/table mp (id 1)))+ lib/append-stage+ (lib/aggregate (lib/count))))++(defn- extra-stages+ [base-query extras]+ (if (pos? extras)+ (recur (lib/append-stage base-query) (dec extras))+ base-query))++(defn- join-chain-5+ ([mp] (join-chain-5 mp (fn [_table-index join-clause]+ join-clause)))+ ([mp fields-fn]+ (-> (lib/query mp (lib.metadata/table mp (id 1)))+ (lib/join (fields-fn 2 (lib/join-clause (lib.metadata/table mp (id 2))+ [(lib/= (lib.metadata/field mp (id 1 4))+ (lib.metadata/field mp (id 2 0)))])))+ (lib/join (fields-fn 3 (lib/join-clause (lib.metadata/table mp (id 3))+ [(lib/= (lib.metadata/field mp (id 2 4))+ (lib.metadata/field mp (id 3 0)))])))+ (lib/join (fields-fn 4 (lib/join-clause (lib.metadata/table mp (id 4))+ [(lib/= (lib.metadata/field mp (id 3 4))+ (lib.metadata/field mp (id 4 0)))])))+ (lib/join (fields-fn 5 (lib/join-clause (lib.metadata/table mp (id 5))+ [(lib/= (lib.metadata/field mp (id 4 4))+ (lib.metadata/field mp (id 5 0)))]))))))++(defn- join-chain-select-two-fields+ [mp]+ (-> (join-chain-5+ mp+ (fn [table-index join-clause]+ (lib/with-join-fields join-clause [(lib.metadata/field mp (id table-index 7))+ (lib.metadata/field mp (id table-index 8))])))+ (lib/with-fields [(lib.metadata/field mp (id 1 7))+ (lib.metadata/field mp (id 1 8))])))++(defn- big-case-expression+ ([mp] (big-case-expression (lib/query mp (lib.metadata/table mp (id 1))) mp))+ ([base-query _mp]+ (let [cols (lib/visible-columns base-query)+ col (nth cols 5)+ cases (for [n (range 1 31)]+ [(lib/= (lib/ref col) n)+ (str "Case " n)])]+ (-> base-query+ (lib/with-fields [(nth cols 7)+ (nth cols 8)])+ (lib/expression "big case" (lib/case cases "Default"))))))++(defn- big-case-expression-aggregated+ ([mp] (big-case-expression-aggregated (trivial-query mp) mp))+ ([base-query mp]+ (let [query (big-case-expression base-query mp)+ expr-fn #(lib.options/ensure-uuid [:expression {} "big case"])]+ (-> query+ (lib/aggregate (lib/sum (expr-fn)))+ (lib/aggregate (lib/avg (expr-fn)))+ (lib/aggregate (lib/min (expr-fn)))+ (lib/aggregate (lib/max (expr-fn)))))))++(defn- big-case-expression-extra-stages+ [mp n]+ (-> (trivial-query mp)+ (extra-stages n)+ (big-case-expression mp)))++(defn- big-case-expression-extra-stage+ [mp]+ (big-case-expression-extra-stages mp 1))++(defn- big-case-expression-extra-stage-aggregated+ [mp]+ (-> (trivial-query mp)+ (extra-stages 1)+ (big-case-expression-aggregated mp)))+(defn- compile-time"Given a query, compiles it several times and returns the stats."- [mp-fn query-fn mem-fn]+ [mp-fn query-fn](crit/report-result(crit/quick-benchmark(mu/disable-enforcement- (binding [lib.computed/*computed-cache* (mem-fn)]- (qp.compile/compile (query-fn (mp-fn)))))+ (qp.compile/compile (query-fn (mp-fn)))){})))-(defn- computed-cache-on []- (atom {}))-(defn- computed-cache-off []- nil)+#_{:clj-kondo/ignore [:unused-private-var]}+(defn- compile-time1+ "Times a single run rather than [[crit/quick-benchmark]]."+ [mp-fn query-fn _mem-fn]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (time (mu/disable-enforcement+ (qp.compile/compile (query-fn (mp-fn))))))(comment;; Preprocesses and compiles a query, with the `lib.computed` cache set throughout.;; Useful for testing performance impact from the REPL for the large queries above.;; Returns [nanoseconds result].(mu/disable-enforcement- (let [compcache (atom {})]- (binding [lib.computed/*computed-cache* compcache]- (crit/time-body- (let [mp (mp-small)- q (trivial-query mp)]- (update (qp.compile/compile q) :query count))))))-- (let [mp (mp-small)- q1 (trivial-query mp)- mp (lib.tu/metadata-provider-with-card-from-query mp 12 q1)- card (lib.metadata/card mp 12)- q2 (lib/query mp card)]- (lib/returned-columns (lib/query q2 (:dataset-query card))))-- ;; Runs Criterium benchmarks for a more statistically sound measurement of the speed of compiling various queries.- ;; Pass computed-cache-on or computed-cache-off as the third argument to control whether the lib.computed memoization- ;; is happening or not.- ;; Back of the envelope analysis from the birth of lib.computed, on my 2022 M1 Max:- ;; - Adding lib.computed added about 1ms of overhead to this trivial query against mp-small.- ;; - For mp-small, 2.5ms before lib.computed; now 2.8ms with the cache disabled and 2.6ms with it enabled.- ;; - For mp-medium, 21ms before lib.computed; now 20.2 with the cache disabled and 19.6ms with it enabled.- ;; - A small win for a trivial query with 100 columns in the table.- ;; - For mp-large, 460ms before lib.computed; now 253ms with cache disabled and 255ms with it enabled.- ;; - For mp-huge, it took about 62s each previously. Now about 11s with caching disabled and 8.8s with it enabled.- ;; Note that the lib.computed/*computed-cache* doesn't actually help much with these straight table queries;- ;; it's much more significant with nested queries, complex expressions and multiple joins.- (compile-time mp-small trivial-query computed-cache-off)- (compile-time mp-small trivial-query computed-cache-on)- (compile-time mp-medium trivial-query computed-cache-off) ; 13.15ms - down 6ms with post-`lib.computed` fixes- (compile-time mp-medium trivial-query computed-cache-on) ; 13.02ms - likewise- (compile-time mp-large trivial-query computed-cache-off) ; 128.2ms - down 120ms- (compile-time mp-large trivial-query computed-cache-on) ; 135.1ms - likewise+ (crit/time-body+ (let [mp (mp-small)+ q (trivial-query mp)]+ (update (qp.compile/compile q) :query count))))++ ;; ======================================== Core Benchmarks ========================================+ ;; Criterium benchmarks for a more statistically sound measurement of the speed of compiling various queries.+ ;; See https://docs.google.com/spreadsheets/d/1Gb-cpuLTziJeCIqh7mEKrjbFm93BAtzPBU1H655_Qdw/edit?gid=0#gid=0+ ;; for measurements from the latest 55, 56 and 57 releases.++ ;; Summary: 55 has some bad bits but is mostly good in practice; 56 got 3-4x worse generally from overhead+ ;; but on a few dimensions (wide tables, big expressions) it got much worse. Like 70ms in 55 vs. 6500ms in 56.+ ;; 57.2.1 after the first batch of perf fixes is equal to or better than 55 in some areas, but is still worse+ ;; in a few. More profiling and perf fixes to come!++ ;; Select all straight from the table+ (compile-time mp-small trivial-query)+ (compile-time mp-medium trivial-query)+ (compile-time mp-large trivial-query)++ ;; Count aggregation+ (compile-time mp-small simple-count)+ (compile-time mp-medium simple-count)+ (compile-time mp-large simple-count)++ ;; Stage + count; this simulates aggregating a table-like model+ (compile-time mp-small stage+count)+ (compile-time mp-medium stage+count)+ (compile-time mp-large stage+count)++ ;; Join chain, select all:+ (compile-time mp-small join-chain-5)+ (compile-time mp-medium join-chain-5)+ (compile-time mp-large join-chain-5)++ ;; Join chain, select just a few fields from each one+ (compile-time mp-small join-chain-select-two-fields)+ (compile-time mp-medium join-chain-select-two-fields)+ (compile-time mp-large join-chain-select-two-fields)++ ;; Two stages+ (compile-time mp-small #(extra-stages (trivial-query %) 1))+ (compile-time mp-medium #(extra-stages (trivial-query %) 1))+ (compile-time mp-large #(extra-stages (trivial-query %) 1))++ ;; Four stages+ (compile-time mp-small #(extra-stages (trivial-query %) 3))+ (compile-time mp-medium #(extra-stages (trivial-query %) 3))+ (compile-time mp-large #(extra-stages (trivial-query %) 3))++ ;; Eight stages+ (compile-time mp-small #(extra-stages (trivial-query %) 7))+ (compile-time mp-medium #(extra-stages (trivial-query %) 7))+ (compile-time mp-large #(extra-stages (trivial-query %) 7))++ ;; Big :case expression (30 cases on the same field)+ (compile-time mp-small big-case-expression)+ (compile-time mp-medium big-case-expression)+ (compile-time mp-large big-case-expression)++ ;; Big :case expression in second stage+ (compile-time mp-small big-case-expression-extra-stage)+ (compile-time mp-medium big-case-expression-extra-stage)+ (compile-time mp-large big-case-expression-extra-stage)++ ;; Big :case expression in **fourth** stage+ (compile-time mp-small #(big-case-expression-extra-stages % 3))+ (compile-time mp-medium #(big-case-expression-extra-stages % 3))+ (compile-time mp-large #(big-case-expression-extra-stages % 3))++ ;; Big :case expression aggregated+ (compile-time mp-small big-case-expression-aggregated)+ (compile-time mp-medium big-case-expression-aggregated)+ (compile-time mp-large big-case-expression-aggregated)++ ;; Big :case expression in second stage plus aggregation+ (compile-time mp-small big-case-expression-extra-stage-aggregated)+ (compile-time mp-medium big-case-expression-extra-stage-aggregated)+ (compile-time mp-large big-case-expression-extra-stage-aggregated);; WARN: Don't run this one unless you're going to lunch, it's really slow to run that many times even with the;; lib.computed caching on.- #_(compile-time mp-huge trivial-query computed-cache-on)+ #_(compile-time mp-huge trivial-query nil);; Instead, here's a single run with mp-huge:- ;; Takes 11114ms with caching off, and 8825ms with it on for me.- ;; Now with the quadratic behavior in add-alias-info fixed, it takes 1440ms!(crit/time-body(mu/disable-enforcement- (binding [lib.computed/*computed-cache* (computed-cache-off)]- (-> (qp.compile/compile (trivial-query (mp-huge)))- (update :query count)))))+ (-> (qp.compile/compile (trivial-query (mp-huge)))+ (update :query count))));; Using the "huge query" MetadataProvider, the big user query that came with QUE-2686.- ;; Starting point: 205ms- ;; Removed repeated MBQL 5 -> 4 -> 5 -> 4 conversions in `driver.sql.qp/preprocess`: down to 142ms- ;; Fixing add-alias-info quadratic lookups: down to 115ms(compile-time (constantly nil)- (fn [_mp] (lib.tu.huge/huge-query))- computed-cache-off)+ (fn [_mp] (lib.tu.huge/huge-query)));; Saving the result for comparison with optimized versions!(defonce ^:private original-result (qp.compile/compile (lib.tu.huge/huge-query)))+ (require '[clj-async-profiler.core :as prof])+ (prof/serve-ui 9111)+(def optimized-result(mu/disable-enforcement(let [q (lib.tu.huge/huge-query)