The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Flip single-query pivot group ordering and row raise limit (#84323)
metambend338b71e3824
diff --git a/.cljfmt.edn b/.cljfmt.ednindex 6818dbf6393..b8f320c148d 100644--- a/.cljfmt.edn+++ b/.cljfmt.edn@@ -238,6 +238,7 @@metabase.query-processor.middleware.process-userland-query-test/with-query-execution [[:block 1]]metabase.query-processor.pipeline-queries-test/mbql5-query [[:block 1]]metabase.query-processor.pipeline-queries-test/run-mbql5-query [[:block 1]]+ metabase.query-processor.pivot/wrapping-pivot-generation-errors [[:block 1]]metabase.query-processor.store/cached [[:block 1]]metabase.query-processor.store/with-metadata-provider [[:inner 0]]metabase.query-processor.streaming/streaming-response [[:block 1]]diff --git a/modules/drivers/hive-like/src/metabase/driver/hive_like.clj b/modules/drivers/hive-like/src/metabase/driver/hive_like.cljindex 2309a0cb2f9..584b09fde61 100644--- a/modules/drivers/hive-like/src/metabase/driver/hive_like.clj+++ b/modules/drivers/hive-like/src/metabase/driver/hive_like.clj@@ -54,7 +54,7 @@;; means the pivot-grouping bitmask varies per row and needs to lead the ORDER BY.multi-set? (some? (nnext (first (:group-by result))))](assoc result :order-by- (into (if multi-set? [[[:inline pivot-grouping-pos] :asc]] [])+ (into (if multi-set? [[[:inline pivot-grouping-pos] :desc]] [])(map (fn [p] [[:inline p] :asc]))breakout-positions))))diff --git a/src/metabase/driver/sql/pivot.clj b/src/metabase/driver/sql/pivot.cljindex 7d01f3714fb..f4246c486a3 100644--- a/src/metabase/driver/sql/pivot.clj+++ b/src/metabase/driver/sql/pivot.clj@@ -188,7 +188,8 @@grouping-sets (into [::grouping-sets] sets-hsql);; With only one grouping set the grouping bitmask is constant; skip the ORDER BY prefix so dialects that;; reject constants in ORDER BY (e.g. SQL Server) don't fail, and to avoid the redundant sort.- prefix-order-by (if (= 1 (count sets-hsql)) [] [[grouping-fn :asc]])]+ ;; `:desc` sorts rollups first so a `:max-results` truncation drops detail rows rather than totals.+ prefix-order-by (if (= 1 (count sets-hsql)) [] [[grouping-fn :desc]])](-> honeysql-form(update :select splice-pivot-grouping-select (count breakout) [grouping-fn lib.pivot/pivot-grouping-column-name])(assoc :group-by [grouping-sets]@@ -369,12 +370,15 @@(when-let [entry (and agg-idx (nth orig-agg-select agg-idx nil))][(alias-ident (alias-of entry)) dir])))))user-order-bys)- ;; `[:pivot-grouping :asc]` sorts combos so grand totals sort last; only meaningful when >1 combo.- ;; User `:order-by` and the breakout tiebreakers apply in every case, so they're always included- ;; when non-empty — dropping them for the single-combo case silently discarded the user's sort.+ ;; `[:pivot-grouping :desc]` sorts combos so grand totals sort first; only meaningful when >1 combo.+ ;; Rollups-first means truncation at `:max-results` drops detail rows rather than totals, so a+ ;; capped pivot still renders with correct subtotals/grand totals (details can be recovered by+ ;; narrowing the question). User `:order-by` and the breakout tiebreakers apply in every case,+ ;; so they're always included when non-empty — dropping them for the single-combo case silently+ ;; discarded the user's sort.outer-order-by (not-empty(into (if (> (count combos) 1)- [[(keyword lib.pivot/pivot-grouping-column-name) :asc]]+ [[(keyword lib.pivot/pivot-grouping-column-name) :desc]][])cat[user-order-by-outerdiff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex 844d91e26f1..887cb0c48ef 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -132,6 +132,7 @@(defmacro ^:private wrapping-pivot-generation-errors"Rewrap any throw from `body` as `\"Error generating pivot queries\"` of type `:qp`, matching theouter error shape every pivot path presents to the QP's error handling."+ {:style/indent 1}[query & body]`(try~@body@@ -212,21 +213,21 @@[query :- ::lib.schema/query{:keys [pivot-rows pivot-cols show-row-totals show-column-totals] :as _pivot-options} :- ::pivot-opts](wrapping-pivot-generation-errors query- (let [all-breakouts (lib/breakouts query)- all-queries (for [breakout-indexes (u/prog1 (breakout-combinations (count all-breakouts)- pivot-rows- pivot-cols- show-row-totals- show-column-totals)- (log/tracef "Using breakout combinations: %s" (pr-str <>)))]- (-> query- (assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)- qp.nest-for-pivot/remove-non-aggregation-order-bys- (keep-breakouts-at-indexes breakout-indexes)))]- (conj (rest (map #(assoc-in % [:info :pivot/result-metadata] :none) all-queries))- (->- (assoc-in (first all-queries) [:info :pivot/original-query] query)- (assoc-in [:info :pivot/result-metadata] (qp.metadata/result-metadata query)))))))+ (let [all-breakouts (lib/breakouts query)+ all-queries (for [breakout-indexes (u/prog1 (breakout-combinations (count all-breakouts)+ pivot-rows+ pivot-cols+ show-row-totals+ show-column-totals)+ (log/tracef "Using breakout combinations: %s" (pr-str <>)))]+ (-> query+ (assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)+ qp.nest-for-pivot/remove-non-aggregation-order-bys+ (keep-breakouts-at-indexes breakout-indexes)))]+ (conj (rest (map #(assoc-in % [:info :pivot/result-metadata] :none) all-queries))+ (->+ (assoc-in (first all-queries) [:info :pivot/original-query] query)+ (assoc-in [:info :pivot/result-metadata] (qp.metadata/result-metadata query)))))))(defn- maybe-userland"Wrap `query` as a userland query when it carries a non-empty `:info` map."@@ -616,20 +617,16 @@:qp.pivot/num-remapped-breakouts num-remapped-breakouts:qp.pivot/remapped-indexes remap)))-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}-(def ^:dynamic ^:private *pivot-max-result-rows*- "Maximum number of result rows for each pivot sub-query. Divided by the number of aggregations since each aggregation- adds a column to the output, so fewer rows are needed to fill the pivot table."- 200000)-(defn- pivot-query-max-rows- "Calculate the per-sub-query row limit for pivot queries: `floor(pivot-max-result-rows / num-aggregations)`.- Falls back to `pivot-max-result-rows` if there are no aggregations (shouldn't happen for pivot queries)."+ "Calculate the per-sub-query row limit for the multi-query pivot path:+ `floor(pivot-max-result-rows / num-aggregations)`. Falls back to the raw setting value if there are no+ aggregations (shouldn't happen for pivot queries)."[query]- (let [num-aggs (count (lib/aggregations query))]+ (let [cap (qp.settings/pivot-max-result-rows)+ num-aggs (count (lib/aggregations query))](if (pos? num-aggs)- (quot *pivot-max-result-rows* num-aggs)- *pivot-max-result-rows*)))+ (quot cap num-aggs)+ cap)))(defn- pivot-opts-from-query"Return `query`'s pivot-options map — as downstream export middleware and streaming writers read it from@@ -759,6 +756,15 @@(not (lib.pivot/has-pivot? query)))(lib.pivot/with-pivot {:rows [] :columns [] :show-row-totals true :show-column-totals true})))+(defn- cap-constraints-at-pivot-setting+ "Cap `:max-results` (and `:max-results-bare-rows`, so it stays ≤ `:max-results`) at the pivot setting.+ A caller-supplied smaller value is still honored."+ [query]+ (let [cap (qp.settings/pivot-max-result-rows)]+ (cond-> (update-in query [:constraints :max-results] u/safe-min cap)+ (get-in query [:constraints :max-results-bare-rows])+ (update-in [:constraints :max-results-bare-rows] min cap))))+(defn- run-sql-pivot-query"Translate `query`'s pivot intent (legacy top-level keys and/or viz-settings) into an MBQL5 `:pivot` clauseon the last stage and submit to the standard QP through `rff`."@@ -767,12 +773,13 @@(get-in query [:info :visualization-settings]))pivot-opts (pivot-opts-from-query query)prepared (wrapping-pivot-generation-errors query- (-> query- apply-legacy-pivot-keys- (apply-pivot-viz-settings viz-settings)- maybe-add-default-pivot-clause- (assoc-in [:middleware :pivot-options] pivot-opts)- maybe-userland))]+ (-> query+ apply-legacy-pivot-keys+ (apply-pivot-viz-settings viz-settings)+ maybe-add-default-pivot-clause+ (assoc-in [:middleware :pivot-options] pivot-opts)+ cap-constraints-at-pivot-setting+ maybe-userland))](qp/process-query prepared rff)))(defn- running-in-clojure-test?diff --git a/src/metabase/query_processor/settings.clj b/src/metabase/query_processor/settings.cljindex 11165a13f3e..b67aaf69684 100644--- a/src/metabase/query_processor/settings.clj+++ b/src/metabase/query_processor/settings.clj@@ -40,6 +40,20 @@:audit :getter:doc "Must be less than 1048575. See also MB_UNAGGREGATED_QUERY_ROW_LIMIT.")+(defsetting pivot-max-result-rows+ (deferred-tru (str "Maximum number of rows a pivot query can return. "+ "Overrides MB_AGGREGATED_QUERY_ROW_LIMIT for pivot queries."))+ :visibility :authenticated+ :export? true+ :type :integer+ :default 200000+ :database-local :allowed+ :audit :getter+ :doc (str "Applies to pivot queries only. When MB_USE_NATIVE_PIVOT_TABLES is on and the "+ "driver supports native pivots, this is a hard cap on the number of rows a "+ "pivot query returns. Otherwise it only approximates the cap, and larger "+ "pivots can exceed it. Must be less than 1048575."))+(defsetting attachment-row-limit(deferred-tru "Row limit in file attachments excluding the header."):visibility :internaldiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 4d0fa86930a..c09af2dfccc 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -682,7 +682,7 @@" GROUPING("" \"__mb_source\".\"region\","" \"__mb_source\".\"__mb_pivot_nfc\""- " ) ASC,"+ " ) DESC,"" \"__mb_source\".\"__mb_pivot_nfc\" ASC,"" \"__mb_source\".\"region\" ASC"](str/split-lines (driver/prettify-native-form :postgres sql))))))))))diff --git a/test/metabase/driver/sql/pivot_test.clj b/test/metabase/driver/sql/pivot_test.cljindex 981bbbf4c41..0ea7e5e542e 100644--- a/test/metabase/driver/sql/pivot_test.clj+++ b/test/metabase/driver/sql/pivot_test.clj@@ -119,11 +119,12 @@(deftest ^:parallel one-breakout-rows-only-test(testing "single breakout, rows-only, both totals: SELECT gets GROUPING(..) AS pivot-grouping; GROUP BY becomes- GROUPING SETS with detail + grand-total; ORDER BY prefixed with GROUPING(..) ASC."+ GROUPING SETS with detail + grand-total; ORDER BY prefixed with GROUPING(..) DESC so rollups+ survive truncation."(is (=? {:select [:b1[[::sql.pivot/grouping-fn some?] "pivot-grouping"]]:group-by [[::sql.pivot/grouping-sets [some?] []]]- :order-by [[[::sql.pivot/grouping-fn some?] :asc]]}+ :order-by [[[::sql.pivot/grouping-fn some?] :desc]]}(compile-grouping-sets [(b1)] {:rows [b1-uuid] :columns []:show-row-totals true :show-column-totals true})))))@@ -172,13 +173,14 @@(#'sql.pivot/compile-grouping-sets-pivot :sql starting stage)))))(deftest ^:parallel order-by-prepends-grouping-test- (testing "GROUPING(...) ASC is prepended as the primary sort; existing :order-by entries pass through untouched- (callers are expected to have already stripped non-aggregation order-bys upstream)"+ (testing "GROUPING(...) DESC is prepended as the primary sort so rollups come first; existing :order-by+ entries pass through untouched (callers are expected to have already stripped non-aggregation+ order-bys upstream)"(let [out (compile-grouping-sets-with-order-by[(b1)]{:rows [b1-uuid] :columns [] :show-row-totals true :show-column-totals true}[[:count :desc]])]- (is (=? [[[::sql.pivot/grouping-fn some?] :asc]+ (is (=? [[[::sql.pivot/grouping-fn some?] :desc][:count :desc]](:order-by out))))))diff --git a/test/metabase/query_processor/pivot/test_util.clj b/test/metabase/query_processor/pivot/test_util.cljindex bc578b2a0e4..2808a807c54 100644--- a/test/metabase/query_processor/pivot/test_util.clj+++ b/test/metabase/query_processor/pivot/test_util.clj@@ -118,15 +118,26 @@`(mt/with-dynamic-fn-redefs [qp.pivot/pivot-outcome-comparator qp.pivot/pivot-cols-equivalent?]~@body))+(defn do-with-pivot-flow-pinned!+ "Pin `run-pivot-query` to one flow by toggling `use-native-pivot-tables` to `single?`."+ [single? thunk]+ (mt/with-temporary-setting-values [use-native-pivot-tables single?]+ ;; disable parity checking so the other flows does't run alongside for comparison+ (binding [qp.pivot/*check-pivot-parity?* false]+ (thunk))))+(defmacro with-multi-query-pivot!- "Force `run-pivot-query` to use the multi-query path inside `body` (by toggling- `use-native-pivot-tables` off), and disable parity checking. Use for tests that assert multi-query-- specific behavior (per-subquery row cap, subquery skipping, etc.) whose semantics don't translate to- the single-query paths.+ "Force `run-pivot-query` to use the multi-query path inside `body`.++ Use for tests that assert multi-query-specific behavior (per-subquery row cap, subquery skipping, etc.) whose+ semantics don't translate to the single-query paths."+ [& body]+ `(do-with-pivot-flow-pinned! false (fn [] ~@body)))++(defmacro with-single-query-pivot!+ "Force `run-pivot-query` to use the single-query (native) path inside `body`.- Not thread-safe — do not use inside `^:parallel` tests; the site-wide setting toggle races other- fixtures. The trailing `!` marks this side effect."+ Use for tests that assert single-query-specific behavior (`:max-results` cap, rollups-first ordering, etc.) whose+ semantics don't translate to the multi-query path."[& body]- `(mt/with-temporary-setting-values [use-native-pivot-tables false]- (binding [qp.pivot/*check-pivot-parity?* false]- ~@body)))+ `(do-with-pivot-flow-pinned! true (fn [] ~@body)))diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 0e5a642df14..6363def8ca7 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -1513,8 +1513,8 @@(mt/test-drivers (qp.pivot.test-util/applicable-drivers);; Use the standard pivot-query but override the pivot limit to be very small.;; The master query (state × source × category) produces ~200+ rows.- ;; With *pivot-max-result-rows* 20, the limit is 20/2=10 per sub-query.- (binding [qp.pivot/*pivot-max-result-rows* 20]+ ;; With pivot-max-result-rows 20, the limit is 20/2=10 per sub-query.+ (mt/with-temporary-setting-values [qp.settings/pivot-max-result-rows 20](let [query (qp.pivot.test-util/pivot-query)results (qp.pivot/run-pivot-query query)rows (mt/rows results)]@@ -1535,38 +1535,101 @@;; Setting unaggregated-query-row-limit to 15 means max-results-bare-rows (15) > max-results (10),;; which violates the constraint schema without the fix.;; Pre-set :constraints like the API layer does (see qp.api/run-query-async+pivot).- (binding [qp.pivot/*pivot-max-result-rows* 20]- (mt/with-temporary-setting-values [qp.settings/unaggregated-query-row-limit 15]- (let [query (-> (qp.pivot.test-util/pivot-query)- (assoc :constraints (qp.constraints/default-query-constraints)))- results (qp.pivot/run-pivot-query query)]- (is (<= 2 (count (get-in query [:query :aggregation])))- "Sanity check: pivot-query should have at least 2 aggregations")- (is (not= :failed (:status results))- "Pivot query should not fail with a constraint violation"))))))))--(deftest ^:parallel native-pivot-honors-max-results-constraint-test- (testing "Native pivot path honors the caller's :constraints :max-results — no special pivot-only cap"- (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)- (let [max-results 50- mp (mt/metadata-provider)- orders (lib.metadata/table mp (mt/id :orders))- product-id (lib.metadata/field mp (mt/id :orders :product_id))- user-id (lib.metadata/field mp (mt/id :orders :user_id))- query (-> (lib/query mp orders)- (lib/breakout product-id)- (lib/breakout user-id)- (lib/aggregate (lib/count))- (merge {:constraints {:max-results max-results}+ (mt/with-temporary-setting-values [qp.settings/pivot-max-result-rows 20+ qp.settings/unaggregated-query-row-limit 15]+ (let [query (-> (qp.pivot.test-util/pivot-query)+ (assoc :constraints (qp.constraints/default-query-constraints)))+ results (qp.pivot/run-pivot-query query)]+ (is (<= 2 (count (get-in query [:query :aggregation])))+ "Sanity check: pivot-query should have at least 2 aggregations")+ (is (not= :failed (:status results))+ "Pivot query should not fail with a constraint violation")))))))++(defn- native-pivot-two-breakout-query+ "An orders breakout product × user × count pivot query, used by the native-pivot limit/ordering tests to+ exercise all four grouping sets `#{0 1 2 3}`."+ []+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ product-id (lib.metadata/field mp (mt/id :orders :product_id))+ user-id (lib.metadata/field mp (mt/id :orders :user_id))]+ (-> (lib/query mp orders)+ (lib/breakout product-id)+ (lib/breakout user-id)+ (lib/aggregate (lib/count)))))++(deftest native-pivot-exceeds-default-aggregated-cap-test+ (testing "Native pivot path returns more than `default-aggregated-query-row-limit` rows when the detail set is bigger — the whole point of the pivot-specific cap"+ (qp.pivot.test-util/with-single-query-pivot!+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)+ ;; `orders` has 18,760 rows, 18,076 unique `(product_id, user_id)` tuples. Add rollups + grand total+ ;; and we're well past the plain aggregated cap of 10,000.+ (let [default-aggregated-max (:max-results (qp.constraints/default-query-constraints))+ query (-> (native-pivot-two-breakout-query)+ (merge {:pivot_rows [0] :pivot_cols [1]}))+ results (qp.pivot/run-pivot-query query)]+ (testing "row count exceeds the default aggregated cap"+ (is (< default-aggregated-max (count (mt/rows results)))))+ (testing "no truncation flag (we're well under `pivot-max-result-rows`)"+ (is (not (contains? (:data results) :pivot_rows_truncated)))))))))++(deftest native-pivot-caps-max-results-at-pivot-cap-test+ (testing "Native pivot path caps `:constraints :max-results` at `pivot-max-result-rows` but honors a smaller caller-supplied value"+ (qp.pivot.test-util/with-single-query-pivot!+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)+ (mt/with-temporary-setting-values [qp.settings/pivot-max-result-rows 100]+ (let [run (fn [caller-max]+ (qp.pivot/run-pivot-query+ (-> (native-pivot-two-breakout-query)+ (cond-> caller-max (assoc :constraints {:max-results caller-max}))+ (assoc :pivot_rows [0] :pivot_cols [1]))))]+ (testing "caller's smaller `:max-results` is honored"+ (let [results (run 10)]+ (is (= 10 (count (mt/rows results))))+ (is (= 10 (get-in results [:data :pivot_rows_truncated])))))+ (testing "caller's larger `:max-results` is clamped to `pivot-max-result-rows`"+ (let [results (run 500)]+ (is (= 100 (count (mt/rows results))))+ (is (= 100 (get-in results [:data :pivot_rows_truncated])))))+ (testing "no caller `:max-results` falls back to `pivot-max-result-rows`"+ (let [results (run nil)]+ (is (= 100 (count (mt/rows results))))+ (is (= 100 (get-in results [:data :pivot_rows_truncated])))))))))))++(deftest native-pivot-caps-max-results-bare-rows-test+ (testing "Native pivot path caps `:max-results-bare-rows` at `pivot-max-result-rows` so it stays ≤ `:max-results`"+ (qp.pivot.test-util/with-single-query-pivot!+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)+ (mt/with-temporary-setting-values [qp.settings/pivot-max-result-rows 10]+ ;; `:max-results-bare-rows` (2000) > `pivot-max-result-rows` (10). Without the cap, the+ ;; constraints middleware rejects the query with+ ;; `max-results-bare-rows must be less than or equal to max-results`.+ (let [query (-> (native-pivot-two-breakout-query)+ (merge {:constraints {:max-results-bare-rows 2000}:pivot_rows [0]- :pivot_cols [1]})- qp.middleware.normalize/normalize-preprocessing-middleware)- results (#'qp.pivot/run-sql-pivot-query query nil)]- (testing "row count matches the requested cap"- (is (= max-results (count (mt/rows results)))))- (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"- (is (= max-results (get-in results [:data :pivot_rows_truncated])))- (is (not (contains? (:data results) :rows_truncated))))))))+ :pivot_cols [1]}))+ results (qp.pivot/run-pivot-query query)]+ (testing "query completes without a constraint-schema violation"+ (is (= :completed (:status results))))+ (testing "row count is capped at `pivot-max-result-rows` and the truncation flag fires"+ (is (= 10 (count (mt/rows results))))+ (is (= 10 (get-in results [:data :pivot_rows_truncated]))))))))))++(deftest native-pivot-emits-rollups-before-details-test+ (testing "Native pivot path returns rows in pivot-grouping DESC order so totals survive `:max-results` truncation"+ (qp.pivot.test-util/with-single-query-pivot!+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)+ (doseq [shape [:native-pivot-query :union-all]]+ (testing (str "forced-shape " shape)+ (let [query (-> (native-pivot-two-breakout-query)+ (merge {:pivot_rows [0] :pivot_cols [1]})+ qp.middleware.normalize/normalize-preprocessing-middleware+ (lib.util/update-query-stage -1 assoc :qp.pivot/forced-shape shape))+ results (qp.pivot/run-pivot-query query)]+ (testing "all four grouping sets present and emitted in rollups-first (DESC) order"+ ;; `long` on each `pivot-grouping` so drivers that return it as BigDecimal (e.g. Oracle) compare equal.+ (is (= [3 2 1 0] (into [] (comp (map #(long (nth % 2))) (distinct))+ (mt/rows results))))))))))))(deftest ^:parallel filters-query-test(testing "Pivot queries with a filter on a breakout column complete successfully"diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yamlindex 2ae8c5cdccf..b16ce24cdad 100644--- a/test_resources/serialization_baseline/settings.yaml+++ b/test_resources/serialization_baseline/settings.yaml@@ -78,6 +78,7 @@ no-object-illustration: nullno-object-illustration-custom: nullnon-table-chart-generated: nullpersisted-models-enabled: null+pivot-max-result-rows: nullreport-timezone: nullreport-timezone-long: nullreport-timezone-short: null