Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Set row limit based on last aggregation or join (#75899)

metambenb0e21b07d6ed
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index 622805f206f..e9a24761ed0 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -1483,6 +1483,7 @@
[lib.metric
available-metrics]
[lib.limit
+ aggregated-output?
current-limit
disable-default-limit
limit
diff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljc
index 8f34bbd8162..a2f74ab086a 100644
--- a/src/metabase/lib/limit.cljc
+++ b/src/metabase/lib/limit.cljc
@@ -1,15 +1,15 @@
(ns metabase.lib.limit
- (:refer-clojure :exclude [empty?])
(:require
[metabase.lib.aggregation :as lib.aggregation]
+ [metabase.lib.join :as lib.join]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
[metabase.lib.page :as lib.page]
+ [metabase.lib.query :as lib.query]
[metabase.lib.schema :as lib.schema]
[metabase.lib.util :as lib.util]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
- [metabase.util.malli :as mu]
- [metabase.util.performance :refer [empty?]]))
+ [metabase.util.malli :as mu]))
(defmethod lib.metadata.calculation/describe-top-level-key-method :limit
[query stage-number _k]
@@ -40,6 +40,18 @@
[query]
(assoc-in query [:middleware :disable-max-results?] true))
+(mu/defn aggregated-output? :- :boolean
+ "Whether the rows produced by a `query`'s final stage are aggregated, i.e., whether the query is an aggregate query.
+
+ A query is an aggregate query if it has a stage with an aggregation without a later stage containing a join."
+ [query :- ::lib.schema/query]
+ (loop [stage-number (dec (lib.query/stage-count query))]
+ (cond
+ (neg? stage-number) false
+ (lib.aggregation/aggregations query stage-number) true
+ (lib.join/joins query stage-number) false
+ :else (recur (dec stage-number)))))
+
(mu/defn max-rows-limit :- [:maybe nat-int?]
"Calculate the absolute maximum number of results that should be returned by this query (MBQL or native), useful for
doing the equivalent of
@@ -55,7 +67,8 @@
* If query has `:constraints` with `:max-results-bare-rows` or `:max-results`, returns the appropriate number
- * `:max-results-bare-rows` is returned if set and Query does not have any aggregations
+ * `:max-results-bare-rows` is returned if set and Query does not produce aggregated output
+ (see [[aggregated-output?]])
* `:max-results` is returned otherwise
@@ -65,7 +78,7 @@
(let [mbql-limit (when-not (lib.util/native-stage? query -1)
(u/safe-min (:items (lib.page/current-page query -1))
(current-limit query -1)))
- constraints-limit (or (when (empty? (lib.aggregation/aggregations query -1))
+ constraints-limit (or (when-not (aggregated-output? query)
max-results-bare-rows)
max-results)]
(u/safe-min mbql-limit constraints-limit)))
diff --git a/src/metabase/query_processor/middleware/add_rows_truncated.clj b/src/metabase/query_processor/middleware/add_rows_truncated.clj
index 6bb567441d1..0b1e20ff74f 100644
--- a/src/metabase/query_processor/middleware/add_rows_truncated.clj
+++ b/src/metabase/query_processor/middleware/add_rows_truncated.clj
@@ -1,23 +1,21 @@
(ns metabase.query-processor.middleware.add-rows-truncated
"Adds `:rows_truncated` to the query results if the results were truncated because of the query's constraints."
- (:refer-clojure :exclude [empty?])
(:require
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
[metabase.query-processor.middleware.limit :as-alias limit]
[metabase.query-processor.schema :as qp.schema]
[metabase.query-processor.settings :as qp.settings]
- [metabase.util.malli :as mu]
- [metabase.util.performance :refer [empty?]]))
+ [metabase.util.malli :as mu]))
(defn- results-limit
[{{:keys [max-results max-results-bare-rows]} :constraints
:as query}]
- (let [{aggregations :aggregation, :keys [limit page], ::limit/keys [original-limit]} (lib/query-stage query -1)]
+ (let [{:keys [limit page], ::limit/keys [original-limit]} (lib/query-stage query -1)]
(or (when (and (or (not limit)
(= original-limit nil))
(not page)
- (empty? aggregations))
+ (not (lib/aggregated-output? query)))
max-results-bare-rows)
max-results
qp.settings/absolute-max-results)))
diff --git a/src/metabase/query_processor/middleware/limit.clj b/src/metabase/query_processor/middleware/limit.clj
index a3df6602e76..ddb5976e894 100644
--- a/src/metabase/query_processor/middleware/limit.clj
+++ b/src/metabase/query_processor/middleware/limit.clj
@@ -1,6 +1,6 @@
(ns metabase.query-processor.middleware.limit
"Middleware that handles limiting the maximum number of rows returned by a query."
- (:refer-clojure :exclude [empty? get-in])
+ (:refer-clojure :exclude [get-in])
(:require
[metabase.lib.core :as lib]
[metabase.lib.schema :as lib.schema]
@@ -9,7 +9,7 @@
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.util :as u]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [empty? get-in]]
+ [metabase.util.performance :refer [get-in]]
[potemkin :as p]))
;;; provided as a convenience since this var used to live here. Prefer using directly from `qp.settings` going forward.
@@ -35,7 +35,7 @@
(and (not (lib/native-stage? query -1))
(not original-limit)
(not (lib/current-page query -1))
- (empty? (lib/aggregations query -1)))
+ (not (lib/aggregated-output? query)))
(lib/limit -1 max-rows))))
(defn determine-query-max-rows
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 4dc654c4a9c..5f538b0b06f 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -598,9 +598,7 @@
" \"json_alias_test\""
" ORDER BY"
" \"json_alias_test\" ASC"
- " ) AS \"__mb_source\""
- "LIMIT"
- " 1048575"]
+ " ) AS \"__mb_source\""]
(str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))))
;;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBC
diff --git a/test/metabase/lib/limit_test.cljc b/test/metabase/lib/limit_test.cljc
index e446629fea8..f2c755567d7 100644
--- a/test/metabase/lib/limit_test.cljc
+++ b/test/metabase/lib/limit_test.cljc
@@ -177,3 +177,32 @@
:limit 10
:aggregation [[:count {:lib/uuid "00000000-0000-0000-0000-000000000000"}]]}]
:constraints {:max-results 15, :max-results-bare-rows 5}})))))
+
+(deftest ^:parallel max-rows-limit-aggregation-then-empty-stage-test
+ (testing "aggregation in an earlier stage + empty (filter-only) final stage uses `:max-results` (#48439)"
+ (is (= 10
+ (lib/max-rows-limit
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ lib/append-stage
+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))
+
+(deftest ^:parallel max-rows-limit-no-aggregation-multi-stage-test
+ (testing "no aggregation in any stage of a multi-stage query uses `:max-results-bare-rows`"
+ (is (= 5
+ (lib/max-rows-limit
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ lib/append-stage
+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))
+
+(deftest ^:parallel max-rows-limit-join-after-aggregation-test
+ (testing "join in a stage after aggregation may re-explode rows, so `:max-results-bare-rows` applies"
+ (is (= 5
+ (lib/max-rows-limit
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/count))
+ lib/append-stage
+ (lib/join (lib/join-clause (meta/table-metadata :products)
+ [(lib/= (meta/field-metadata :orders :product-id)
+ (meta/field-metadata :products :id))]))
+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5})))))))
diff --git a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
index 7098f113032..fc0d2bf61a6 100644
--- a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
+++ b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
@@ -2,6 +2,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.add-rows-truncated-test]}}}}}}
(:require
[clojure.test :refer :all]
+ [metabase.lib.core :as lib]
[metabase.lib.test-metadata :as meta]
[metabase.query-processor.middleware.add-rows-truncated :as add-rows-truncated]
[metabase.query-processor.reducible :as qp.reducible]
@@ -45,6 +46,18 @@
:max-results-bare-rows 5}}
[[1] [1] [1] [1] [1]])))))
+(deftest ^:parallel add-rows-truncated-aggregation-then-empty-stage-test
+ (testing "aggregation in an earlier stage + empty final stage uses `:max-results`, not `:max-results-bare-rows` (#48439)"
+ (is (= {:status :completed
+ :row_count 5
+ :data {:rows [[1] [1] [1] [1] [1]]}}
+ (add-rows-truncated
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ lib/append-stage
+ (assoc :constraints {:max-results 10, :max-results-bare-rows 5}))
+ [[1] [1] [1] [1] [1]])))))
+
(deftest ^:parallel e2e-test
(let [result (qp/process-query
(-> (mt/mbql-query venues {:order-by [[:asc $id]]})
diff --git a/test/metabase/query_processor/middleware/limit_test.clj b/test/metabase/query_processor/middleware/limit_test.clj
index 547e32e1b8c..e72c6ea8ec3 100644
--- a/test/metabase/query_processor/middleware/limit_test.clj
+++ b/test/metabase/query_processor/middleware/limit_test.clj
@@ -42,6 +42,14 @@
:limit qp.settings/absolute-max-results}}
(add-default-limit query))))))
+(deftest ^:parallel aggregation-then-empty-stage-test
+ (testing "do not add a :limit to the appended (filter-only) stage when an earlier stage aggregates (#48439)"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ lib/append-stage)]
+ (is (nil? (lib/current-limit (limit/add-default-limit query) -1))
+ "no :limit should be added to the last stage since the query is aggregated"))))
+
(deftest ^:parallel disable-max-results-test-2
(testing "Don't apply the `absolute-max-results` limit when `disable-max-results` is used."
(let [query (limit/disable-max-results {:database (meta/id)
diff --git a/test/metabase/query_processor/nested_queries_test.clj b/test/metabase/query_processor/nested_queries_test.clj
index 347a65a34ff..4ef8960ac49 100644
--- a/test/metabase/query_processor/nested_queries_test.clj
+++ b/test/metabase/query_processor/nested_queries_test.clj
@@ -1510,8 +1510,7 @@
(when (= driver/*driver* :h2)
(is (= (update q1-native :query (fn [s]
(format (str "SELECT \"__mb_source\".\"count\" AS \"count\" "
- "FROM (%s) AS \"__mb_source\" "
- "LIMIT 1048575")
+ "FROM (%s) AS \"__mb_source\"")
s)))
(qp.compile/compile q2))))
(is (= [[543]]