The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Set row limit based on last aggregation or join (#75899)
metambenb0e21b07d6ed
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 622805f206f..e9a24761ed0 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -1483,6 +1483,7 @@[lib.metricavailable-metrics][lib.limit+ aggregated-output?current-limitdisable-default-limitlimitdiff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljcindex 8f34bbd8162..a2f74ab086a 100644--- a/src/metabase/lib/limit.cljc+++ b/src/metabase/lib/limit.cljc@@ -1,15 +1,15 @@(ns metabase.lib.limit- (:refer-clojure :exclude [empty?])(:require[metabase.lib.aggregation :as lib.aggregation]+ [metabase.lib.join :as lib.join][metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.page :as lib.page]+ [metabase.lib.query :as lib.query][metabase.lib.schema :as lib.schema][metabase.lib.util :as lib.util][metabase.util :as u][metabase.util.i18n :as i18n]- [metabase.util.malli :as mu]- [metabase.util.performance :refer [empty?]]))+ [metabase.util.malli :as mu]))(defmethod lib.metadata.calculation/describe-top-level-key-method :limit[query stage-number _k]@@ -40,6 +40,18 @@[query](assoc-in query [:middleware :disable-max-results?] true))+(mu/defn aggregated-output? :- :boolean+ "Whether the rows produced by a `query`'s final stage are aggregated, i.e., whether the query is an aggregate query.++ A query is an aggregate query if it has a stage with an aggregation without a later stage containing a join."+ [query :- ::lib.schema/query]+ (loop [stage-number (dec (lib.query/stage-count query))]+ (cond+ (neg? stage-number) false+ (lib.aggregation/aggregations query stage-number) true+ (lib.join/joins query stage-number) false+ :else (recur (dec stage-number)))))+(mu/defn max-rows-limit :- [:maybe nat-int?]"Calculate the absolute maximum number of results that should be returned by this query (MBQL or native), useful fordoing the equivalent of@@ -55,7 +67,8 @@* If query has `:constraints` with `:max-results-bare-rows` or `:max-results`, returns the appropriate number- * `:max-results-bare-rows` is returned if set and Query does not have any aggregations+ * `:max-results-bare-rows` is returned if set and Query does not produce aggregated output+ (see [[aggregated-output?]])* `:max-results` is returned otherwise@@ -65,7 +78,7 @@(let [mbql-limit (when-not (lib.util/native-stage? query -1)(u/safe-min (:items (lib.page/current-page query -1))(current-limit query -1)))- constraints-limit (or (when (empty? (lib.aggregation/aggregations query -1))+ constraints-limit (or (when-not (aggregated-output? query)max-results-bare-rows)max-results)](u/safe-min mbql-limit constraints-limit)))diff --git a/src/metabase/query_processor/middleware/add_rows_truncated.clj b/src/metabase/query_processor/middleware/add_rows_truncated.cljindex 6bb567441d1..0b1e20ff74f 100644--- a/src/metabase/query_processor/middleware/add_rows_truncated.clj+++ b/src/metabase/query_processor/middleware/add_rows_truncated.clj@@ -1,23 +1,21 @@(ns metabase.query-processor.middleware.add-rows-truncated"Adds `:rows_truncated` to the query results if the results were truncated because of the query's constraints."- (:refer-clojure :exclude [empty?])(:require[metabase.lib.core :as lib][metabase.lib.schema :as lib.schema][metabase.query-processor.middleware.limit :as-alias limit][metabase.query-processor.schema :as qp.schema][metabase.query-processor.settings :as qp.settings]- [metabase.util.malli :as mu]- [metabase.util.performance :refer [empty?]]))+ [metabase.util.malli :as mu]))(defn- results-limit[{{:keys [max-results max-results-bare-rows]} :constraints:as query}]- (let [{aggregations :aggregation, :keys [limit page], ::limit/keys [original-limit]} (lib/query-stage query -1)]+ (let [{:keys [limit page], ::limit/keys [original-limit]} (lib/query-stage query -1)](or (when (and (or (not limit)(= original-limit nil))(not page)- (empty? aggregations))+ (not (lib/aggregated-output? query)))max-results-bare-rows)max-resultsqp.settings/absolute-max-results)))diff --git a/src/metabase/query_processor/middleware/limit.clj b/src/metabase/query_processor/middleware/limit.cljindex a3df6602e76..ddb5976e894 100644--- a/src/metabase/query_processor/middleware/limit.clj+++ b/src/metabase/query_processor/middleware/limit.clj@@ -1,6 +1,6 @@(ns metabase.query-processor.middleware.limit"Middleware that handles limiting the maximum number of rows returned by a query."- (:refer-clojure :exclude [empty? get-in])+ (:refer-clojure :exclude [get-in])(:require[metabase.lib.core :as lib][metabase.lib.schema :as lib.schema]@@ -9,7 +9,7 @@^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.malli :as mu]- [metabase.util.performance :refer [empty? get-in]]+ [metabase.util.performance :refer [get-in]][potemkin :as p]));;; provided as a convenience since this var used to live here. Prefer using directly from `qp.settings` going forward.@@ -35,7 +35,7 @@(and (not (lib/native-stage? query -1))(not original-limit)(not (lib/current-page query -1))- (empty? (lib/aggregations query -1)))+ (not (lib/aggregated-output? query)))(lib/limit -1 max-rows))))(defn determine-query-max-rowsdiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 4dc654c4a9c..5f538b0b06f 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -598,9 +598,7 @@" \"json_alias_test\""" ORDER BY"" \"json_alias_test\" ASC"- " ) AS \"__mb_source\""- "LIMIT"- " 1048575"]+ " ) AS \"__mb_source\""](str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))));;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBCdiff --git a/test/metabase/lib/limit_test.cljc b/test/metabase/lib/limit_test.cljcindex e446629fea8..f2c755567d7 100644--- a/test/metabase/lib/limit_test.cljc+++ b/test/metabase/lib/limit_test.cljc@@ -177,3 +177,32 @@:limit 10:aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]}]:constraints {:max-results 15, :max-results-bare-rows 5}})))))++(deftest ^:parallel max-rows-limit-aggregation-then-empty-stage-test+ (testing "aggregation in an earlier stage + empty (filter-only) final stage uses `:max-results` (#48439)"+ (is (= 10+ (lib/max-rows-limit+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ lib/append-stage+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))++(deftest ^:parallel max-rows-limit-no-aggregation-multi-stage-test+ (testing "no aggregation in any stage of a multi-stage query uses `:max-results-bare-rows`"+ (is (= 5+ (lib/max-rows-limit+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ lib/append-stage+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))++(deftest ^:parallel max-rows-limit-join-after-aggregation-test+ (testing "join in a stage after aggregation may re-explode rows, so `:max-results-bare-rows` applies"+ (is (= 5+ (lib/max-rows-limit+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/count))+ lib/append-stage+ (lib/join (lib/join-clause (meta/table-metadata :products)+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :products :id))]))+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))diff --git a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj b/test/metabase/query_processor/middleware/add_rows_truncated_test.cljindex 7098f113032..fc0d2bf61a6 100644--- a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj+++ b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj@@ -2,6 +2,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.add-rows-truncated-test]}}}}}}(:require[clojure.test :refer :all]+ [metabase.lib.core :as lib][metabase.lib.test-metadata :as meta][metabase.query-processor.middleware.add-rows-truncated :as add-rows-truncated][metabase.query-processor.reducible :as qp.reducible]@@ -45,6 +46,18 @@:max-results-bare-rows 5}}[[1] [1] [1] [1] [1]])))))+(deftest ^:parallel add-rows-truncated-aggregation-then-empty-stage-test+ (testing "aggregation in an earlier stage + empty final stage uses `:max-results`, not `:max-results-bare-rows` (#48439)"+ (is (= {:status :completed+ :row_count 5+ :data {:rows [[1] [1] [1] [1] [1]]}}+ (add-rows-truncated+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ lib/append-stage+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5}))+ [[1] [1] [1] [1] [1]])))))+(deftest ^:parallel e2e-test(let [result (qp/process-query(-> (mt/mbql-query venues {:order-by [[:asc $id]]})diff --git a/test/metabase/query_processor/middleware/limit_test.clj b/test/metabase/query_processor/middleware/limit_test.cljindex 547e32e1b8c..e72c6ea8ec3 100644--- a/test/metabase/query_processor/middleware/limit_test.clj+++ b/test/metabase/query_processor/middleware/limit_test.clj@@ -42,6 +42,14 @@:limit qp.settings/absolute-max-results}}(add-default-limit query))))))+(deftest ^:parallel aggregation-then-empty-stage-test+ (testing "do not add a :limit to the appended (filter-only) stage when an earlier stage aggregates (#48439)"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ lib/append-stage)]+ (is (nil? (lib/current-limit (limit/add-default-limit query) -1))+ "no :limit should be added to the last stage since the query is aggregated"))))+(deftest ^:parallel disable-max-results-test-2(testing "Don't apply the `absolute-max-results` limit when `disable-max-results` is used."(let [query (limit/disable-max-results {:database (meta/id)diff --git a/test/metabase/query_processor/nested_queries_test.clj b/test/metabase/query_processor/nested_queries_test.cljindex 347a65a34ff..4ef8960ac49 100644--- a/test/metabase/query_processor/nested_queries_test.clj+++ b/test/metabase/query_processor/nested_queries_test.clj@@ -1510,8 +1510,7 @@(when (= driver/*driver* :h2)(is (= (update q1-native :query (fn [s](format (str "SELECT \"__mb_source\".\"count\" AS \"count\" "- "FROM (%s) AS \"__mb_source\" "- "LIMIT 1048575")+ "FROM (%s) AS \"__mb_source\"")s)))(qp.compile/compile q2))))(is (= [[543]]