Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove LIMIT in query builder transforms (#69347)

Nicola Mometto09ff1de3635c
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index 586f2a9596a..cb29b4c5f83 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -443,6 +443,17 @@ LIMIT
H.assertQueryBuilderRowCount(3);
});
+ it("should not include absolute-max-results LIMIT in SQL preview for MBQL transforms", () => {
+ createMbqlTransform({ visitTransform: true });
+ H.DataStudio.Transforms.clickEditDefinition();
+ cy.url().should("include", "/edit");
+
+ getQueryEditor().findByLabelText("View SQL").click();
+ H.sidebar()
+ .should("be.visible")
+ .and("not.contain", /\bLIMIT\b/i);
+ });
+
it("should not allow to overwrite an existing table when creating a transform", () => {
cy.log("open the new transform page");
visitTransformListPage();
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 91df2a51615..9432ddc6960 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -275,7 +275,8 @@
metadata-provider (lib-be/application-database-metadata-provider db-id)
table-metadata (lib.metadata/table metadata-provider table-id)
transform (t2/select-one :model/Transform transform-id)]
- (cond-> (lib/query metadata-provider table-metadata)
+ (cond-> (-> (lib/query metadata-provider table-metadata)
+ lib/disable-default-limit)
source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))
limit (lib/limit limit))))
diff --git a/frontend/src/metabase-lib/limit.ts b/frontend/src/metabase-lib/limit.ts
index 8e0d1aba6ce..e8f3cbdf181 100644
--- a/frontend/src/metabase-lib/limit.ts
+++ b/frontend/src/metabase-lib/limit.ts
@@ -14,3 +14,7 @@ export function hasLimit(query: Query, stageIndex: number) {
const limit = currentLimit(query, stageIndex);
return typeof limit === "number" && limit > 0;
}
+
+export function disableDefaultLimit(query: Query): Query {
+ return ML.disable_default_limit(query);
+}
diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx
index 058088b81ef..8505cbf2034 100644
--- a/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx
+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx
@@ -15,6 +15,7 @@ type NativeQueryPreviewSidebarProps = {
convertToNativeButtonLabel?: string;
onConvertToNativeClick: (newQuestion: Question) => void;
readOnly?: boolean;
+ disableDefaultLimit?: boolean;
};
export function NativeQueryPreviewSidebar({
@@ -23,6 +24,7 @@ export function NativeQueryPreviewSidebar({
convertToNativeButtonLabel,
onConvertToNativeClick,
readOnly,
+ disableDefaultLimit,
}: NativeQueryPreviewSidebarProps) {
const { width: windowWidth } = useWindowSize();
const minSidebarWidth = 428;
@@ -45,6 +47,7 @@ export function NativeQueryPreviewSidebar({
buttonTitle={convertToNativeButtonLabel}
onConvertClick={onConvertToNativeClick}
readOnly={readOnly}
+ disableDefaultLimit={disableDefaultLimit}
/>
</ResizableBox>
);
diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
index 4de7dba4e4c..00b26ebb1d9 100644
--- a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
@@ -178,6 +178,7 @@ export function QueryEditor({
convertToNativeButtonLabel={uiOptions?.convertToNativeButtonLabel}
onConvertToNativeClick={convertToNative}
readOnly={uiOptions?.readOnly}
+ disableDefaultLimit={uiOptions?.disableDefaultLimit}
/>
)}
</Flex>
diff --git a/frontend/src/metabase/querying/editor/types.ts b/frontend/src/metabase/querying/editor/types.ts
index c31e0c2adf3..27b7211f7e7 100644
--- a/frontend/src/metabase/querying/editor/types.ts
+++ b/frontend/src/metabase/querying/editor/types.ts
@@ -48,6 +48,7 @@ export type QueryEditorUiOptions = {
canConvertToNative?: boolean;
convertToNativeTitle?: string;
convertToNativeButtonLabel?: string;
+ disableDefaultLimit?: boolean;
shouldDisableDataPickerItem?: (item: QueryEditorDataPickerItem) => boolean;
shouldDisableDatabasePickerItem?: (
item: QueryEditorDatabasePickerItem,
diff --git a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx
index 8b90bd561d0..d73513896f5 100644
--- a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx
+++ b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx
@@ -35,6 +35,7 @@ type NotebookNativePreviewProps = {
buttonTitle?: string;
onConvertClick: (newQuestion: Question) => void;
readOnly?: boolean;
+ disableDefaultLimit?: boolean;
};
export const NotebookNativePreview = ({
@@ -43,6 +44,7 @@ export const NotebookNativePreview = ({
buttonTitle,
onConvertClick,
readOnly,
+ disableDefaultLimit,
}: NotebookNativePreviewProps) => {
const database = question.database();
const engine = database?.engine;
@@ -50,7 +52,10 @@ export const NotebookNativePreview = ({
const sourceQuery = question.query();
const canRun = Lib.canRun(sourceQuery, question.type());
- const payload = Lib.toJsQuery(sourceQuery);
+ const queryForPayload = disableDefaultLimit
+ ? Lib.disableDefaultLimit(sourceQuery)
+ : sourceQuery;
+ const payload = Lib.toJsQuery(queryForPayload);
const { data, error, isFetching } = useGetNativeDatasetQuery(payload);
const showLoader = isFetching;
diff --git a/frontend/src/metabase/transforms/components/TransformEditor/utils.ts b/frontend/src/metabase/transforms/components/TransformEditor/utils.ts
index 7d750d044f1..5381843969d 100644
--- a/frontend/src/metabase/transforms/components/TransformEditor/utils.ts
+++ b/frontend/src/metabase/transforms/components/TransformEditor/utils.ts
@@ -13,6 +13,7 @@ export function getEditorOptions(
canConvertToNative: true,
convertToNativeTitle: t`SQL for this transform`,
convertToNativeButtonLabel: t`Convert this transform to SQL`,
+ disableDefaultLimit: true,
shouldDisableDatabasePickerItem: (item) => {
const database = databases.find((database) => database.id === item.id);
return !doesDatabaseSupportTransforms(database);
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 0151980a190..6b272934562 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -601,7 +601,8 @@
{:sql sql :driver driver}))))
(defn- execute-statement-or-prepared-statement! ^ResultSet [driver ^Statement stmt max-rows params sql]
- (let [st (doto stmt (.setMaxRows max-rows))]
+ (let [max-rows (or max-rows 0) ; 0 means no limit
+ st (doto stmt (.setMaxRows max-rows))]
(if (use-statement? driver params)
(execute-statement! driver st sql)
(execute-prepared-statement! driver st))))
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index ee691e5cd2f..e619ac644fb 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -348,6 +348,7 @@
available-metrics]
[lib.limit
current-limit
+ disable-default-limit
limit
max-rows-limit]
[metabase.lib.metadata
diff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljc
index d7df729d79f..b70ea6fa7ea 100644
--- a/src/metabase/lib/limit.cljc
+++ b/src/metabase/lib/limit.cljc
@@ -35,6 +35,12 @@
stage-number :- :int]
(:limit (lib.util/query-stage query stage-number))))
+(defn ^:export disable-default-limit
+ "Sets the `disable-max-results?` middleware option on `query`, which disables the default limit on
+ query results. Used by transforms to allow unlimited result rows."
+ [query]
+ (assoc-in query [:middleware :disable-max-results?] true))
+
(mu/defn max-rows-limit :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]
"Calculate the absolute maximum number of results that should be returned by this query (MBQL or native), useful for
doing the equivalent of
diff --git a/src/metabase/query_processor/middleware/limit.clj b/src/metabase/query_processor/middleware/limit.clj
index 589f8e3b32a..a3df6602e76 100644
--- a/src/metabase/query_processor/middleware/limit.clj
+++ b/src/metabase/query_processor/middleware/limit.clj
@@ -99,6 +99,7 @@
"Post-processing middleware. Limit the maximum number of rows that are returned in post-processing."
[query :- ::lib.schema/query
rff :- ::qp.schema/rff]
- (let [max-rows (determine-query-max-rows query)]
+ (if-let [max-rows (determine-query-max-rows query)]
(fn limit-result-rows-rff* [metadata]
- (limit-xform max-rows (rff metadata)))))
+ (limit-xform max-rows (rff metadata)))
+ rff))
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index 548a845d562..c11f7da982b 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -272,7 +272,9 @@
(defn massage-sql-query
"Adjusts mbql query for use in a transform."
[query]
- (remap/disable-remaps query))
+ (-> query
+ remap/disable-remaps
+ lib/disable-default-limit))
(defn- checkpoint-incremental?
"Returns true if `source` uses checkpoint-based incremental strategy."
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index 4e215fabc27..23c8745c133 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -555,6 +555,16 @@
(assoc (mt/mbql-query venues {:fields [$id $name]})
:pretty false)))))))
+(deftest ^:parallel compile-disable-max-results-test
+ (testing "POST /api/dataset/native"
+ (testing "\nWith disable-max-results? the compiled SQL should not include a LIMIT clause"
+ (let [query (-> (mt/mbql-query venues {:fields [$id $name]})
+ (assoc-in [:middleware :disable-max-results?] true)
+ (assoc :pretty false))
+ result (mt/user-http-request :crowberto :post 200 "dataset/native" query)]
+ (is (not (re-find #"(?i)\bLIMIT\b" (:query result)))
+ (str "Expected no LIMIT in SQL, got: " (:query result)))))))
+
(deftest ^:parallel compile-test-2
(testing "POST /api/dataset/native"
(testing "\nCan we fetch a native version of an MBQL query?"
@@ -960,6 +970,16 @@
(driver/prettify-native-form :h2)
str/split-lines))))))))))
+(deftest ^:parallel mlv2-query-convert-to-native-disable-default-limit-test
+ (testing "POST /api/dataset/native"
+ (testing "MLv2 query with disable-default-limit should compile to SQL without a LIMIT clause"
+ (let [metadata-provider (mt/metadata-provider)
+ venues (lib.metadata/table metadata-provider (mt/id :venues))
+ query (-> (lib/query metadata-provider venues)
+ lib/disable-default-limit)]
+ (is (not (re-find #"(?i)\bLIMIT\b" (:query (mt/user-http-request :crowberto :post 200 "dataset/native" query))))
+ "Expected no LIMIT in SQL for query with disable-default-limit")))))
+
(deftest ^:parallel format-export-middleware-test
(testing "The `:format-export?` query processor middleware has the intended effect on file exports."
(let [q {:database (mt/id)
diff --git a/test/metabase/query_processor/middleware/limit_test.clj b/test/metabase/query_processor/middleware/limit_test.clj
index 6abd3ae9dd4..1584aaa7f1d 100644
--- a/test/metabase/query_processor/middleware/limit_test.clj
+++ b/test/metabase/query_processor/middleware/limit_test.clj
@@ -50,6 +50,18 @@
(is (= query
(add-default-limit query))))))
+(deftest disable-max-results-post-processing-test
+ (testing "limit-result-rows passes through all rows when `disable-max-results` is used"
+ (with-redefs [qp.settings/absolute-max-results test-max-results]
+ (let [query (-> (lib/query meta/metadata-provider {:type :native, :native {:query "SELECT 1;"}})
+ limit/disable-max-results)
+ num-rows (inc test-max-results)
+ rff (limit/limit-result-rows query qp.reducible/default-rff)
+ rf (rff {})
+ result (transduce identity rf (repeat num-rows [:ok]))]
+ (is (= num-rows
+ (-> result mt/rows count)))))))
+
(deftest max-results-constraint-test
(testing "Apply an arbitrary max-results on the query and ensure our results size is appropriately constrained"
(is (= 1234
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 4ee72b294af..a86444bb11a 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
+ [metabase.lib.core :as lib]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
@@ -351,3 +352,21 @@
(mt/with-temp [:model/Table table {:transform_id nil}]
(let [hydrated (t2/hydrate table :transform)]
(is (nil? (:transform hydrated))))))))
+
+(deftest ^:parallel massage-sql-query-test
+ (testing "massage-sql-query sets disable-remaps? and disable-max-results?"
+ (let [query {:database 1, :type :query, :query {:source-table 1}}
+ massaged (transforms.util/massage-sql-query query)]
+ (is (true? (get-in massaged [:middleware :disable-remaps?])))
+ (is (true? (get-in massaged [:middleware :disable-max-results?]))))))
+
+(deftest compile-source-no-limit-test
+ (testing "compile-source produces SQL without a LIMIT clause"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/with-premium-features #{:transforms}
+ (let [transform {:source {:type "query"
+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
+ {:keys [query]} (transforms.util/compile-source transform)]
+ (is (string? query))
+ (is (not (re-find #"(?i)\bLIMIT\b" query))
+ (str "Expected no LIMIT clause in compiled SQL, got: " query)))))))