The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove LIMIT in query builder transforms (#69347)
Nicola Mometto09ff1de3635c
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex 586f2a9596a..cb29b4c5f83 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -443,6 +443,17 @@ LIMITH.assertQueryBuilderRowCount(3);});+ it("should not include absolute-max-results LIMIT in SQL preview for MBQL transforms", () => {+ createMbqlTransform({ visitTransform: true });+ H.DataStudio.Transforms.clickEditDefinition();+ cy.url().should("include", "/edit");++ getQueryEditor().findByLabelText("View SQL").click();+ H.sidebar()+ .should("be.visible")+ .and("not.contain", /\bLIMIT\b/i);+ });+it("should not allow to overwrite an existing table when creating a transform", () => {cy.log("open the new transform page");visitTransformListPage();diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 91df2a51615..9432ddc6960 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -275,7 +275,8 @@metadata-provider (lib-be/application-database-metadata-provider db-id)table-metadata (lib.metadata/table metadata-provider table-id)transform (t2/select-one :model/Transform transform-id)]- (cond-> (lib/query metadata-provider table-metadata)+ (cond-> (-> (lib/query metadata-provider table-metadata)+ lib/disable-default-limit)source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))limit (lib/limit limit))))diff --git a/frontend/src/metabase-lib/limit.ts b/frontend/src/metabase-lib/limit.tsindex 8e0d1aba6ce..e8f3cbdf181 100644--- a/frontend/src/metabase-lib/limit.ts+++ b/frontend/src/metabase-lib/limit.ts@@ -14,3 +14,7 @@ export function hasLimit(query: Query, stageIndex: number) {const limit = currentLimit(query, stageIndex);return typeof limit === "number" && limit > 0;}++export function disableDefaultLimit(query: Query): Query {+ return ML.disable_default_limit(query);+}diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsxindex 058088b81ef..8505cbf2034 100644--- a/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/NativeQueryPreviewSidebar/NativeQueryPreviewSidebar.tsx@@ -15,6 +15,7 @@ type NativeQueryPreviewSidebarProps = {convertToNativeButtonLabel?: string;onConvertToNativeClick: (newQuestion: Question) => void;readOnly?: boolean;+ disableDefaultLimit?: boolean;};export function NativeQueryPreviewSidebar({@@ -23,6 +24,7 @@ export function NativeQueryPreviewSidebar({convertToNativeButtonLabel,onConvertToNativeClick,readOnly,+ disableDefaultLimit,}: NativeQueryPreviewSidebarProps) {const { width: windowWidth } = useWindowSize();const minSidebarWidth = 428;@@ -45,6 +47,7 @@ export function NativeQueryPreviewSidebar({buttonTitle={convertToNativeButtonLabel}onConvertClick={onConvertToNativeClick}readOnly={readOnly}+ disableDefaultLimit={disableDefaultLimit}/></ResizableBox>);diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsxindex 4de7dba4e4c..00b26ebb1d9 100644--- a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx@@ -178,6 +178,7 @@ export function QueryEditor({convertToNativeButtonLabel={uiOptions?.convertToNativeButtonLabel}onConvertToNativeClick={convertToNative}readOnly={uiOptions?.readOnly}+ disableDefaultLimit={uiOptions?.disableDefaultLimit}/>)}</Flex>diff --git a/frontend/src/metabase/querying/editor/types.ts b/frontend/src/metabase/querying/editor/types.tsindex c31e0c2adf3..27b7211f7e7 100644--- a/frontend/src/metabase/querying/editor/types.ts+++ b/frontend/src/metabase/querying/editor/types.ts@@ -48,6 +48,7 @@ export type QueryEditorUiOptions = {canConvertToNative?: boolean;convertToNativeTitle?: string;convertToNativeButtonLabel?: string;+ disableDefaultLimit?: boolean;shouldDisableDataPickerItem?: (item: QueryEditorDataPickerItem) => boolean;shouldDisableDatabasePickerItem?: (item: QueryEditorDatabasePickerItem,diff --git a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsxindex 8b90bd561d0..d73513896f5 100644--- a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx+++ b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx@@ -35,6 +35,7 @@ type NotebookNativePreviewProps = {buttonTitle?: string;onConvertClick: (newQuestion: Question) => void;readOnly?: boolean;+ disableDefaultLimit?: boolean;};export const NotebookNativePreview = ({@@ -43,6 +44,7 @@ export const NotebookNativePreview = ({buttonTitle,onConvertClick,readOnly,+ disableDefaultLimit,}: NotebookNativePreviewProps) => {const database = question.database();const engine = database?.engine;@@ -50,7 +52,10 @@ export const NotebookNativePreview = ({const sourceQuery = question.query();const canRun = Lib.canRun(sourceQuery, question.type());- const payload = Lib.toJsQuery(sourceQuery);+ const queryForPayload = disableDefaultLimit+ ? Lib.disableDefaultLimit(sourceQuery)+ : sourceQuery;+ const payload = Lib.toJsQuery(queryForPayload);const { data, error, isFetching } = useGetNativeDatasetQuery(payload);const showLoader = isFetching;diff --git a/frontend/src/metabase/transforms/components/TransformEditor/utils.ts b/frontend/src/metabase/transforms/components/TransformEditor/utils.tsindex 7d750d044f1..5381843969d 100644--- a/frontend/src/metabase/transforms/components/TransformEditor/utils.ts+++ b/frontend/src/metabase/transforms/components/TransformEditor/utils.ts@@ -13,6 +13,7 @@ export function getEditorOptions(canConvertToNative: true,convertToNativeTitle: t`SQL for this transform`,convertToNativeButtonLabel: t`Convert this transform to SQL`,+ disableDefaultLimit: true,shouldDisableDatabasePickerItem: (item) => {const database = databases.find((database) => database.id === item.id);return !doesDatabaseSupportTransforms(database);diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex 0151980a190..6b272934562 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -601,7 +601,8 @@{:sql sql :driver driver}))))(defn- execute-statement-or-prepared-statement! ^ResultSet [driver ^Statement stmt max-rows params sql]- (let [st (doto stmt (.setMaxRows max-rows))]+ (let [max-rows (or max-rows 0) ; 0 means no limit+ st (doto stmt (.setMaxRows max-rows))](if (use-statement? driver params)(execute-statement! driver st sql)(execute-prepared-statement! driver st))))diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex ee691e5cd2f..e619ac644fb 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -348,6 +348,7 @@available-metrics][lib.limitcurrent-limit+ disable-default-limitlimitmax-rows-limit][metabase.lib.metadatadiff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljcindex d7df729d79f..b70ea6fa7ea 100644--- a/src/metabase/lib/limit.cljc+++ b/src/metabase/lib/limit.cljc@@ -35,6 +35,12 @@stage-number :- :int](:limit (lib.util/query-stage query stage-number))))+(defn ^:export disable-default-limit+ "Sets the `disable-max-results?` middleware option on `query`, which disables the default limit on+ query results. Used by transforms to allow unlimited result rows."+ [query]+ (assoc-in query [:middleware :disable-max-results?] true))+(mu/defn max-rows-limit :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]"Calculate the absolute maximum number of results that should be returned by this query (MBQL or native), useful fordoing the equivalent ofdiff --git a/src/metabase/query_processor/middleware/limit.clj b/src/metabase/query_processor/middleware/limit.cljindex 589f8e3b32a..a3df6602e76 100644--- a/src/metabase/query_processor/middleware/limit.clj+++ b/src/metabase/query_processor/middleware/limit.clj@@ -99,6 +99,7 @@"Post-processing middleware. Limit the maximum number of rows that are returned in post-processing."[query :- ::lib.schema/queryrff :- ::qp.schema/rff]- (let [max-rows (determine-query-max-rows query)]+ (if-let [max-rows (determine-query-max-rows query)](fn limit-result-rows-rff* [metadata]- (limit-xform max-rows (rff metadata)))))+ (limit-xform max-rows (rff metadata)))+ rff))diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex 548a845d562..c11f7da982b 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -272,7 +272,9 @@(defn massage-sql-query"Adjusts mbql query for use in a transform."[query]- (remap/disable-remaps query))+ (-> query+ remap/disable-remaps+ lib/disable-default-limit))(defn- checkpoint-incremental?"Returns true if `source` uses checkpoint-based incremental strategy."diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 4e215fabc27..23c8745c133 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -555,6 +555,16 @@(assoc (mt/mbql-query venues {:fields [$id $name]}):pretty false)))))))+(deftest ^:parallel compile-disable-max-results-test+ (testing "POST /api/dataset/native"+ (testing "\nWith disable-max-results? the compiled SQL should not include a LIMIT clause"+ (let [query (-> (mt/mbql-query venues {:fields [$id $name]})+ (assoc-in [:middleware :disable-max-results?] true)+ (assoc :pretty false))+ result (mt/user-http-request :crowberto :post 200 "dataset/native" query)]+ (is (not (re-find #"(?i)\bLIMIT\b" (:query result)))+ (str "Expected no LIMIT in SQL, got: " (:query result)))))))+(deftest ^:parallel compile-test-2(testing "POST /api/dataset/native"(testing "\nCan we fetch a native version of an MBQL query?"@@ -960,6 +970,16 @@(driver/prettify-native-form :h2)str/split-lines))))))))))+(deftest ^:parallel mlv2-query-convert-to-native-disable-default-limit-test+ (testing "POST /api/dataset/native"+ (testing "MLv2 query with disable-default-limit should compile to SQL without a LIMIT clause"+ (let [metadata-provider (mt/metadata-provider)+ venues (lib.metadata/table metadata-provider (mt/id :venues))+ query (-> (lib/query metadata-provider venues)+ lib/disable-default-limit)]+ (is (not (re-find #"(?i)\bLIMIT\b" (:query (mt/user-http-request :crowberto :post 200 "dataset/native" query))))+ "Expected no LIMIT in SQL for query with disable-default-limit")))))+(deftest ^:parallel format-export-middleware-test(testing "The `:format-export?` query processor middleware has the intended effect on file exports."(let [q {:database (mt/id)diff --git a/test/metabase/query_processor/middleware/limit_test.clj b/test/metabase/query_processor/middleware/limit_test.cljindex 6abd3ae9dd4..1584aaa7f1d 100644--- a/test/metabase/query_processor/middleware/limit_test.clj+++ b/test/metabase/query_processor/middleware/limit_test.clj@@ -50,6 +50,18 @@(is (= query(add-default-limit query))))))+(deftest disable-max-results-post-processing-test+ (testing "limit-result-rows passes through all rows when `disable-max-results` is used"+ (with-redefs [qp.settings/absolute-max-results test-max-results]+ (let [query (-> (lib/query meta/metadata-provider {:type :native, :native {:query "SELECT 1;"}})+ limit/disable-max-results)+ num-rows (inc test-max-results)+ rff (limit/limit-result-rows query qp.reducible/default-rff)+ rf (rff {})+ result (transduce identity rf (repeat num-rows [:ok]))]+ (is (= num-rows+ (-> result mt/rows count)))))))+(deftest max-results-constraint-test(testing "Apply an arbitrary max-results on the query and ensure our results size is appropriately constrained"(is (= 1234diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex 4ee72b294af..a86444bb11a 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.events.core :as events]+ [metabase.lib.core :as lib][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt]@@ -351,3 +352,21 @@(mt/with-temp [:model/Table table {:transform_id nil}](let [hydrated (t2/hydrate table :transform)](is (nil? (:transform hydrated))))))))++(deftest ^:parallel massage-sql-query-test+ (testing "massage-sql-query sets disable-remaps? and disable-max-results?"+ (let [query {:database 1, :type :query, :query {:source-table 1}}+ massaged (transforms.util/massage-sql-query query)]+ (is (true? (get-in massaged [:middleware :disable-remaps?])))+ (is (true? (get-in massaged [:middleware :disable-max-results?]))))))++(deftest compile-source-no-limit-test+ (testing "compile-source produces SQL without a LIMIT clause"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/with-premium-features #{:transforms}+ (let [transform {:source {:type "query"+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}+ {:keys [query]} (transforms.util/compile-source transform)]+ (is (string? query))+ (is (not (re-find #"(?i)\bLIMIT\b" query))+ (str "Expected no LIMIT clause in compiled SQL, got: " query)))))))