Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-1419 warn when incremental edits break checkpoint auto selection (#67933)

Egor Grushin83ca22073086
diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.ts
index a364520f52f..5ada007c03c 100644
--- a/e2e/support/helpers/e2e-transform-helpers.ts
+++ b/e2e/support/helpers/e2e-transform-helpers.ts
@@ -6,6 +6,7 @@ import type {
PythonTransformTableAliases,
TransformId,
TransformRun,
+ TransformSourceCheckpointStrategy,
TransformTagId,
} from "metabase-types/api";
@@ -117,12 +118,14 @@ export function createSqlTransform({
targetSchema,
tagIds,
visitTransform,
+ sourceCheckpointStrategy,
}: {
sourceQuery: string;
targetTable: string;
targetSchema: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
+ sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;
}) {
return createTransform(
{
@@ -136,6 +139,7 @@ export function createSqlTransform({
query: sourceQuery,
},
},
+ "source-incremental-strategy": sourceCheckpointStrategy,
},
target: {
type: "table",
diff --git a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
index fc151d59316..f8a42b736af 100644
--- a/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms.cy.spec.ts
@@ -7,6 +7,7 @@ import type {
CardType,
CollectionId,
PythonTransformTableAliases,
+ TransformSourceCheckpointStrategy,
TransformTagId,
} from "metabase-types/api";
@@ -1535,6 +1536,98 @@ LIMIT
H.queryBuilderHeader().findByText(DB_NAME).should("be.visible");
H.assertQueryBuilderRowCount(1);
});
+
+ describe("query complexity warning", () => {
+ it("should show complexity warning modal when saving a complex SQL query", () => {
+ cy.log("create a simple SQL transform");
+ createSqlTransform({
+ sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
+ visitTransform: true,
+ sourceCheckpointStrategy: { type: "checkpoint" },
+ });
+
+ cy.log("visit edit mode and change to a complex query with LIMIT");
+ H.DataStudio.Transforms.editDefinition().click();
+ cy.url().should("include", "/edit");
+
+ H.NativeEditor.type(" LIMIT 10");
+ getQueryEditor().button("Save").click();
+
+ handleQueryComplexityWarningModal("cancel");
+ cy.log("verify modal is closed and still in edit mode");
+ H.modal().should("not.exist");
+ cy.url().should("include", "/edit");
+ cy.get("@updateTransform.all").should("have.length", 0);
+
+ cy.log("Save anyway");
+ getQueryEditor().button("Save").click();
+ handleQueryComplexityWarningModal("save");
+
+ cy.wait("@updateTransform");
+ cy.url().should("not.include", "/edit");
+ });
+
+ it("should confirm incremental settings change if query is complex", () => {
+ createSqlTransform({
+ sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,
+ sourceCheckpointStrategy: { type: "checkpoint" },
+ visitTransform: true,
+ });
+ H.DataStudio.Transforms.settingsTab().click();
+
+ cy.log("Toggle incremental on");
+ isIncrementalSwitchDisabled();
+ getIncrementalSwitch().click();
+
+ handleQueryComplexityWarningModal("cancel");
+
+ cy.log("Verify that the switch is still off");
+ isIncrementalSwitchDisabled();
+
+ cy.log("Toggle incremental on");
+ getIncrementalSwitch().click();
+ handleQueryComplexityWarningModal("save");
+
+ cy.wait("@updateTransform");
+ isIncrementalSwitchEnabled();
+ H.undoToast().should(
+ "contain.text",
+ "Incremental transformation settings updated",
+ );
+ });
+
+ it("should show complexity warning with danger button in create transform modal when enabling incremental with complex query", () => {
+ cy.log("create a new SQL transform with a complex query");
+ visitTransformListPage();
+ cy.button("Create a transform").click();
+ H.popover().findByText("SQL query").click();
+ H.popover().findByText(DB_NAME).click();
+
+ H.NativeEditor.type(
+ `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,
+ );
+
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("Complex SQL transform");
+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+
+ cy.log("Enable incremental transformation");
+ getIncrementalSwitch().click();
+
+ cy.log("Verify complexity warning appears inline");
+ cy.findByTestId("query-complexity-warning")
+ .scrollIntoView()
+ .should("be.visible");
+
+ cy.log("Verify the submit button is styled as danger (red)");
+ cy.findByRole("button", { name: "Save anyway" })
+ .scrollIntoView()
+ .should("have.css", "background-color", "rgb(209, 44, 41)");
+ });
+ });
+ });
});
describe("runs", () => {
@@ -3259,6 +3352,18 @@ function isIncrementalSwitchDisabled() {
return getIncrementalSwitch().findByRole("switch").should("not.be.checked");
}
+function handleQueryComplexityWarningModal(action: "cancel" | "save") {
+ cy.log(`Verify complexity warning modal appears and ${action} it`);
+ return H.modal().within(() => {
+ cy.findByTestId("query-complexity-warning").should("be.visible");
+ if (action === "save") {
+ cy.button("Save anyway").click();
+ } else {
+ cy.button("Cancel").click();
+ }
+ });
+}
+
function getContentTable() {
return cy.findByTestId("admin-content-table");
}
@@ -3339,13 +3444,13 @@ function createMbqlTransform(
...opts,
});
}
-
function createSqlTransform(opts: {
sourceQuery: string;
targetTable?: string;
targetSchema?: string;
tagIds?: TransformTagId[];
visitTransform?: boolean;
+ sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;
}) {
return H.createSqlTransform({
targetTable: TARGET_TABLE,
diff --git a/enterprise/frontend/src/metabase-enterprise/api/transform.ts b/enterprise/frontend/src/metabase-enterprise/api/transform.ts
index 091a1ad3201..5f02a4761b7 100644
--- a/enterprise/frontend/src/metabase-enterprise/api/transform.ts
+++ b/enterprise/frontend/src/metabase-enterprise/api/transform.ts
@@ -1,13 +1,13 @@
import { isResourceNotFoundError } from "metabase/lib/errors";
import type {
CheckQueryComplexityRequest,
- CheckQueryComplexityResponse,
CreateTransformRequest,
ExtractColumnsFromQueryRequest,
ExtractColumnsFromQueryResponse,
ListTransformRunsRequest,
ListTransformRunsResponse,
ListTransformsRequest,
+ QueryComplexity,
RunTransformResponse,
Transform,
TransformId,
@@ -213,7 +213,7 @@ export const transformApi = EnterpriseApi.injectEndpoints({
}),
}),
checkQueryComplexity: builder.query<
- CheckQueryComplexityResponse,
+ QueryComplexity,
CheckQueryComplexityRequest
>({
query: (queryString) => ({
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index 7dc7a7ce06a..1d35f314380 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -1,20 +1,19 @@
import { useFormikContext } from "formik";
-import { useEffect, useMemo, useState } from "react";
+import { match } from "ts-pattern";
import { t } from "ttag";
-import { FormSelect, FormSwitch } from "metabase/forms";
+import { FormSelect } from "metabase/forms";
import { useSelector } from "metabase/lib/redux";
import { getMetadata } from "metabase/selectors/metadata";
-import { Alert, Box, Divider, Group, Stack, Text } from "metabase/ui";
-import { useLazyCheckQueryComplexityQuery } from "metabase-enterprise/api";
+import { getShowMetabaseLinks } from "metabase/selectors/whitelabel";
+import { Anchor, Box, Divider, Group, Stack, Switch, Text } from "metabase/ui";
import { TitleSection } from "metabase-enterprise/transforms/components/TitleSection";
import {
- CHECKPOINT_TEMPLATE_TAG,
SOURCE_STRATEGY_OPTIONS,
TARGET_STRATEGY_OPTIONS,
} from "metabase-enterprise/transforms/constants";
-import * as Lib from "metabase-lib";
-import Question from "metabase-lib/v1/Question";
+import { getLibQuery, isMbqlQuery } from "metabase-enterprise/transforms/utils";
+import type * as Lib from "metabase-lib";
import type { TransformSource } from "metabase-types/api";
import {
@@ -26,49 +25,20 @@ import type { IncrementalSettingsFormValues } from "./form";
type IncrementalTransformSettingsProps = {
source: TransformSource;
- checkOnMount?: boolean;
+ incremental: boolean;
+ onIncrementalChange: (value: boolean) => void;
variant?: "embedded" | "standalone";
};
export const IncrementalTransformSettings = ({
source,
- checkOnMount,
+ incremental,
+ onIncrementalChange,
variant = "embedded",
}: IncrementalTransformSettingsProps) => {
const metadata = useSelector(getMetadata);
- const { values } = useFormikContext<IncrementalSettingsFormValues>();
- // Convert DatasetQuery to Lib.Query via Question
- const libQuery = useMemo(() => {
- if (source.type !== "query") {
- return null;
- }
-
- try {
- const question = Question.create({
- dataset_query: source.query,
- metadata,
- });
- return question.query();
- } catch (error) {
- console.error("SourceStrategyFields: Error creating question", error);
- return null;
- }
- }, [source, metadata]);
-
- // Check if this is an MBQL query (not native SQL or Python)
- const isMbqlQuery = useMemo(() => {
- if (!libQuery) {
- return false;
- }
-
- try {
- const queryDisplayInfo = Lib.queryDisplayInfo(libQuery);
- return !queryDisplayInfo.isNative;
- } catch (error) {
- console.error("Error checking query type", error);
- return false;
- }
- }, [libQuery]);
+ const libQuery = getLibQuery(source, metadata);
+ const showMetabaseLinks = useSelector(getShowMetabaseLinks);
// Check if this is a Python transform with exactly one source table
// Incremental transforms are only supported for single-table Python transforms
@@ -76,118 +46,60 @@ export const IncrementalTransformSettings = ({
const isMultiTablePythonTransform =
isPythonTransform && Object.keys(source["source-tables"]).length > 1;
- const [checkQueryComplexity, { data: complexity }] =
- useLazyCheckQueryComplexityQuery();
- const [showComplexityWarning, setShowComplexityWarning] = useState(false);
-
- const { transformType, query } = useMemo(() => {
- if (isMbqlQuery) {
- return { transformType: "query" as const, query: libQuery };
- }
- if (isPythonTransform) {
- return { transformType: "python" as const, query: null };
- }
- return { transformType: "native" as const, query: libQuery };
- }, [isMbqlQuery, isPythonTransform, libQuery]);
-
- const hasCheckpointTag = useMemo(() => {
- if (!libQuery || transformType !== "native") {
- return false;
- }
- const tags = Lib.templateTags(libQuery);
- return CHECKPOINT_TEMPLATE_TAG in tags;
- }, [libQuery, transformType]);
-
- useEffect(() => {
- async function checkExistingQueryComplexity() {
- if (
- checkOnMount &&
- transformType === "native" &&
- libQuery &&
- !hasCheckpointTag &&
- "source-incremental-strategy" in source
- ) {
- const { is_simple } = await checkQueryComplexity(
- Lib.rawNativeQuery(libQuery),
- true,
- ).unwrap();
- setShowComplexityWarning(!is_simple);
- }
- }
- checkExistingQueryComplexity();
- }, [
- checkOnMount,
- checkQueryComplexity,
- libQuery,
- transformType,
- source,
- hasCheckpointTag,
- ]);
+ const transformType = match({
+ isMbqlQuery: isMbqlQuery(source, metadata),
+ isPythonTransform,
+ })
+ .with({ isMbqlQuery: true }, () => "query" as const)
+ .with({ isPythonTransform: true }, () => "python" as const)
+ .otherwise(() => "native" as const);
const renderIncrementalSwitch = () => (
- <FormSwitch
+ <Switch
disabled={isMultiTablePythonTransform}
- name="incremental"
+ checked={incremental}
size="sm"
label={
isMultiTablePythonTransform
? t`Incremental transforms are only supported for single data source transforms.`
: t`Only process new and changed data`
}
- onChange={async (e) => {
- if (
- e.target.checked &&
- transformType === "native" &&
- !hasCheckpointTag &&
- query
- ) {
- const complexity = await checkQueryComplexity(
- Lib.rawNativeQuery(query),
- true,
- ).unwrap();
- setShowComplexityWarning(complexity?.is_simple === false);
- }
- }}
wrapperProps={{
"data-testid": "incremental-switch",
}}
+ onChange={(e) => onIncrementalChange(e.target.checked)}
/>
);
- const complexityWarningAlert = showComplexityWarning && (
- <Alert variant="info" icon="info">
- <Stack gap="xs">
- <span>
- {t`This query is too complex to allow automatic checkpoint column selection. You may need to explicitly add a conditional filter in your query, for example:`}
- </span>
- <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>
- <span>
- {t`Reason: `}
- <strong>{complexity?.reason}</strong>
- </span>
- </Stack>
- </Alert>
- );
-
const label = t`Incremental transformation`;
- const description = t`If you don't need to reprocess everything, incremental transforms can be faster.`;
+ const renderDescription = () => {
+ const description = t`If you don’t need to reprocess all the data, incremental transforms can be faster. To use this, your transform definition should have a stable schema. `;
+ return (
+ <>
+ {description}
+ {showMetabaseLinks && (
+ <Anchor
+ href="https://www.metabase.com/docs/latest/"
+ target="_blank"
+ td="underline"
+ c="inherit"
+ size="sm"
+ >{t`Learn more.`}</Anchor>
+ )}{" "}
+ </>
+ );
+ };
if (variant === "standalone") {
return (
- <TitleSection label={label} description={description}>
+ <TitleSection label={label} description={renderDescription()}>
<Group p="lg">{renderIncrementalSwitch()}</Group>
- {values?.incremental && (
+ {incremental && (
<>
- {complexityWarningAlert && (
- <>
- <Divider />
- <Group p="lg">{complexityWarningAlert}</Group>
- </>
- )}
<Divider />
<Group p="lg">
<SourceStrategyFields
source={source}
- query={query}
+ query={libQuery}
type={transformType}
/>
</Group>
@@ -199,26 +111,25 @@ export const IncrementalTransformSettings = ({
}
return (
- <Group gap="lg">
+ <Stack gap="lg">
<Box>
<Text fw="bold">{label}</Text>
- <Text size="sm" c="text-secondary" mb="sm">
- {description}
+ <Text size="sm" lh="1rem" mb="sm">
+ {renderDescription()}
</Text>
{renderIncrementalSwitch()}
</Box>
- {values?.incremental && (
+ {incremental && (
<>
- {complexityWarningAlert}
<SourceStrategyFields
source={source}
- query={query}
+ query={libQuery}
type={transformType}
/>
<TargetStrategyFields variant={variant} />
</>
)}
- </Group>
+ </Stack>
);
};
@@ -284,7 +195,7 @@ function SourceStrategyFields({
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
description={t`Pick the field that we should scan to determine which records are new or changed`}
- descriptionProps={{ c: "text-secondary", mb: "xs" }}
+ descriptionProps={{ lh: "1rem" }}
query={query}
/>
)}
@@ -294,7 +205,7 @@ function SourceStrategyFields({
label={t`Column to check for new values`}
placeholder={t`Pick a column`}
description={t`Pick the column that we should scan to determine which records are new or changed`}
- descriptionProps={{ c: "text-secondary", mb: "xs" }}
+ descriptionProps={{ lh: "1rem" }}
query={query}
/>
)}
@@ -304,7 +215,7 @@ function SourceStrategyFields({
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
description={t`Pick the field that we should scan to determine which records are new or changed`}
- descriptionProps={{ c: "text-secondary", mb: "xs" }}
+ descriptionProps={{ lh: "1rem" }}
sourceTables={source["source-tables"]}
/>
)}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css
new file mode 100644
index 00000000000..86f3a93b537
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css
@@ -0,0 +1,6 @@
+.bordered {
+ background: var(--mb-color-background-secondary);
+ border-radius: var(--mantine-radius-md);
+ padding: var(--mantine-spacing-md);
+ border: 1px solid var(--mb-color-border);
+}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx
new file mode 100644
index 00000000000..ed5d7743ff4
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx
@@ -0,0 +1,51 @@
+import cx from "classnames";
+import { c, t } from "ttag";
+
+import { uncapitalize } from "metabase/lib/formatting/strings";
+import { Stack, Text } from "metabase/ui";
+import { CHECKPOINT_TEMPLATE_TAG } from "metabase-enterprise/transforms/constants";
+import type { QueryComplexity } from "metabase-types/api";
+
+import S from "./QueryComplexityWarning.module.css";
+
+type QueryComplexityWarningProps = {
+ variant?: "default" | "standout";
+ complexity: QueryComplexity;
+};
+
+export const QueryComplexityWarning = ({
+ variant = "default",
+ complexity,
+}: QueryComplexityWarningProps) => {
+ const renderParagraph = () => {
+ const reason = uncapitalize(complexity.reason);
+ return c(
+ "{0} is a reason phrase like 'contains a LIMIT', 'contains an OFFSET', 'contains a CTE', or 'is not a simple SELECT'",
+ )
+ .t`Because this query ${reason}, we can't automatically select a checkpoint column. You should either remove the LIMIT, turn off incremental transformation, or add an explicit conditional filter to the query, like this:`;
+ };
+
+ return (
+ <Stack
+ gap={variant === "standout" ? "sm" : "md"}
+ className={cx({ [S.bordered]: variant === "standout" })}
+ data-testid="query-complexity-warning"
+ >
+ {variant === "standout" && (
+ <Text fw="bold">{t`We couldn’t add the check automatically.`}</Text>
+ )}
+ <Text lh="1.25rem">{renderParagraph()}</Text>
+ <Text
+ lh="1.25rem"
+ py="md"
+ className={cx({ [S.bordered]: variant === "default" })}
+ >
+ <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>
+ </Text>
+ <Text
+ lh="1.25rem"
+ c="saturated-red"
+ >{t`If you save anyway, this transform will likely not process new data, or might only process partial quantities of new data.`}</Text>
+ </Stack>
+ );
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/index.ts b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/index.ts
new file mode 100644
index 00000000000..638c5b5972a
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/index.ts
@@ -0,0 +1,2 @@
+export { QueryComplexityWarning } from "./QueryComplexityWarning";
+export { useQueryComplexityChecks } from "./use-query-complexity-checks";
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx
new file mode 100644
index 00000000000..e4dca972dd7
--- /dev/null
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx
@@ -0,0 +1,78 @@
+import { useCallback } from "react";
+import { t } from "ttag";
+
+import { useConfirmation } from "metabase/common/hooks";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
+import { useLazyCheckQueryComplexityQuery } from "metabase-enterprise/api";
+import { CHECKPOINT_TEMPLATE_TAG } from "metabase-enterprise/transforms/constants";
+import { getLibQuery, isMbqlQuery } from "metabase-enterprise/transforms/utils";
+import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
+import type { DraftTransformSource } from "metabase-types/api";
+
+import { QueryComplexityWarning } from "./QueryComplexityWarning";
+
+const getQueryForComplexityCheck = (
+ source: DraftTransformSource,
+ metadata: Metadata,
+): Lib.Query | undefined => {
+ const query = getLibQuery(source, metadata);
+ if (!query || isMbqlQuery(source, metadata)) {
+ return;
+ }
+
+ const tags = Lib.templateTags(query);
+ if (CHECKPOINT_TEMPLATE_TAG in tags) {
+ return;
+ }
+
+ return query;
+};
+
+export const useQueryComplexityChecks = () => {
+ const [checkQueryComplexity] = useLazyCheckQueryComplexityQuery();
+ const metadata = useSelector(getMetadata);
+ const { modalContent: modal, show } = useConfirmation();
+
+ const checkComplexity = useCallback(
+ async (source: DraftTransformSource) => {
+ const query = getQueryForComplexityCheck(source, metadata);
+ if (!query) {
+ return;
+ }
+ const rawQuery = Lib.rawNativeQuery(query);
+ const complexity = await checkQueryComplexity(rawQuery, true).unwrap();
+ if (complexity.is_simple) {
+ return;
+ }
+ return complexity;
+ },
+ [metadata, checkQueryComplexity],
+ );
+
+ const confirmIfQueryIsComplex = useCallback(
+ async (
+ source: DraftTransformSource,
+ confirmButtonText: string = t`Save anyway`,
+ ) => {
+ const complexity = await checkComplexity(source);
+ if (!complexity) {
+ return true;
+ }
+ return new Promise<boolean>((resolve) => {
+ show({
+ title: t`Can't automatically run this transform incrementally`,
+ message: <QueryComplexityWarning complexity={complexity} />,
+ onConfirm: () => resolve(true),
+ onCancel: () => resolve(false),
+ confirmButtonText,
+ confirmButtonProps: { color: "saturated-red", variant: "filled" },
+ });
+ });
+ },
+ [checkComplexity, show],
+ );
+
+ return { confirmIfQueryIsComplex, checkComplexity, modal };
+};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TitleSection/TitleSection.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TitleSection/TitleSection.tsx
index 9f014e8ffd1..e245178be82 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TitleSection/TitleSection.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TitleSection/TitleSection.tsx
@@ -4,7 +4,7 @@ import { Card, Group, Stack, Text, Title } from "metabase/ui";
type TitleSectionProps = {
label: string;
- description?: string;
+ description?: ReactNode;
children?: ReactNode;
};
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
index d53bb17435c..0166c8d0cbe 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/components/TransformEditor/TransformEditor.tsx
@@ -25,6 +25,7 @@ type TransformEditorProps = {
onChangeUiState: (state: QueryEditorUiState) => void;
onAcceptProposed: () => void;
onRejectProposed: () => void;
+ onBlur?: () => void;
readOnly?: boolean;
transformId?: TransformId;
};
@@ -38,6 +39,7 @@ export function TransformEditor({
onChangeUiState,
onAcceptProposed,
onRejectProposed,
+ onBlur,
readOnly,
transformId,
}: TransformEditorProps) {
@@ -78,6 +80,7 @@ export function TransformEditor({
onChangeUiState={onChangeUiState}
onAcceptProposed={onAcceptProposed}
onRejectProposed={onRejectProposed}
+ onBlur={onBlur}
topBarInnerContent={
readOnly &&
!!transformId && (
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
index ea7fb67f0fd..8f021113307 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
@@ -1,3 +1,5 @@
+import { useFormikContext } from "formik";
+import { useState } from "react";
import { t } from "ttag";
import { hasFeature } from "metabase/admin/databases/utils";
@@ -17,7 +19,15 @@ import {
} from "metabase/forms";
import { Box, Button, Group, Modal, Stack } from "metabase/ui";
import { IncrementalTransformSettings } from "metabase-enterprise/transforms/components/IncrementalTransform/IncrementalTransformSettings";
-import type { Transform, TransformSource } from "metabase-types/api";
+import {
+ QueryComplexityWarning,
+ useQueryComplexityChecks,
+} from "metabase-enterprise/transforms/components/QueryComplexityWarning";
+import type {
+ QueryComplexity,
+ Transform,
+ TransformSource,
+} from "metabase-types/api";
import { SchemaFormSelect } from "../../../components/SchemaFormSelect";
@@ -38,31 +48,6 @@ export function CreateTransformModal({
onCreate,
onClose,
}: CreateTransformModalProps) {
- return (
- <Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
- <CreateTransformForm
- source={source}
- defaultValues={defaultValues}
- onCreate={onCreate}
- onClose={onClose}
- />
- </Modal>
- );
-}
-
-type CreateTransformFormProps = {
- source: TransformSource;
- defaultValues: Partial<NewTransformValues>;
- onCreate: (transform: Transform) => void;
- onClose: () => void;
-};
-
-function CreateTransformForm({
- source,
- defaultValues,
- onCreate,
- onClose,
-}: CreateTransformFormProps) {
const databaseId =
source.type === "query" ? source.query.database : source["source-database"];
@@ -101,43 +86,93 @@ function CreateTransformForm({
};
return (
- <FormProvider
- initialValues={initialValues}
- validationSchema={validationSchema}
- onSubmit={handleSubmit}
- >
- <Form>
- <Stack gap="lg" mt="sm">
- <FormTextInput
- name="name"
- label={t`Name`}
- placeholder={t`My Great Transform`}
- data-autofocus
+ <Modal title={t`Save your transform`} opened padding="xl" onClose={onClose}>
+ <FormProvider
+ initialValues={initialValues}
+ validationSchema={validationSchema}
+ onSubmit={handleSubmit}
+ >
+ <CreateTransformForm
+ source={source}
+ supportsSchemas={supportsSchemas}
+ schemas={schemas}
+ onClose={onClose}
+ />
+ </FormProvider>
+ </Modal>
+ );
+}
+
+type CreateTransformFormFieldsProps = {
+ source: TransformSource;
+ supportsSchemas: boolean | undefined;
+ schemas: string[];
+ onClose: () => void;
+};
+
+function CreateTransformForm({
+ source,
+ supportsSchemas,
+ schemas,
+ onClose,
+}: CreateTransformFormFieldsProps) {
+ const { values, setFieldValue } = useFormikContext<NewTransformValues>();
+ const { checkComplexity } = useQueryComplexityChecks();
+ const [complexity, setComplexity] = useState<QueryComplexity | undefined>();
+
+ const handleIncrementalChange = async (value: boolean) => {
+ setFieldValue("incremental", value);
+ if (value) {
+ const complexity = await checkComplexity(source);
+ setComplexity(complexity);
+ } else {
+ setComplexity(undefined);
+ }
+ };
+
+ return (
+ <Form>
+ <Stack gap="lg" mt="sm">
+ <FormTextInput
+ name="name"
+ label={t`Name`}
+ placeholder={t`My Great Transform`}
+ data-autofocus
+ />
+ {supportsSchemas && (
+ <SchemaFormSelect
+ name="targetSchema"
+ label={t`Schema`}
+ data={schemas}
/>
- {supportsSchemas && (
- <SchemaFormSelect
- name="targetSchema"
- label={t`Schema`}
- data={schemas}
- />
- )}
- <TargetNameInput />
- <FormCollectionPicker
- name="collection_id"
- title={t`Collection`}
- type="transform-collections"
- style={{ marginBottom: 0 }}
+ )}
+ <TargetNameInput />
+ <FormCollectionPicker
+ name="collection_id"
+ title={t`Collection`}
+ type="transform-collections"
+ style={{ marginBottom: 0 }}
+ />
+ <IncrementalTransformSettings
+ source={source}
+ incremental={values.incremental}
+ onIncrementalChange={handleIncrementalChange}
+ />
+ {complexity && (
+ <QueryComplexityWarning complexity={complexity} variant="standout" />
+ )}
+ <Group>
+ <Box flex={1}>
+ <FormErrorMessage />
+ </Box>
+ <Button onClick={onClose}>{t`Back`}</Button>
+ <FormSubmitButton
+ label={complexity ? t`Save anyway` : t`Save`}
+ variant="filled"
+ color={complexity ? "saturated-red" : undefined}
/>
- <IncrementalTransformSettings source={source} />
- <Group>
- <Box flex={1}>
- <FormErrorMessage />
- </Box>
- <Button onClick={onClose}>{t`Back`}</Button>
- <FormSubmitButton label={t`Save`} variant="filled" />
- </Group>
- </Stack>
- </Form>
- </FormProvider>
+ </Group>
+ </Stack>
+ </Form>
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx
index 242a847b6a4..d1819cb5f94 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx
@@ -35,7 +35,7 @@ import { TransformEditor } from "../../components/TransformEditor";
import { NAME_MAX_LENGTH } from "../../constants";
import { useRegisterMetabotTransformContext } from "../../hooks/use-register-transform-metabot-context";
import { useSourceState } from "../../hooks/use-source-state";
-import { getValidationResult, isNotDraftSource } from "../../utils";
+import { getValidationResult, isCompleteSource } from "../../utils";
import { CreateTransformModal } from "./CreateTransformModal";
import {
@@ -186,7 +186,7 @@ function NewTransformPageBody({
)}
</Box>
</PageContainer>
- {isModalOpened && isNotDraftSource(source) && (
+ {isModalOpened && isCompleteSource(source) && (
<CreateTransformModal
source={source}
defaultValues={getDefaultValues(name, suggestedTransform)}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index 656bd636200..65755927ce8 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -23,11 +23,12 @@ import {
import { PageContainer } from "metabase-enterprise/data-studio/common/components/PageContainer";
import type { Database, Transform } from "metabase-types/api";
+import { useQueryComplexityChecks } from "../../components/QueryComplexityWarning";
import { TransformEditor } from "../../components/TransformEditor";
import { TransformHeader } from "../../components/TransformHeader";
import { useRegisterMetabotTransformContext } from "../../hooks/use-register-transform-metabot-context";
import { useSourceState } from "../../hooks/use-source-state";
-import { isNotDraftSource } from "../../utils";
+import { isCompleteSource } from "../../utils";
import { TransformPaneHeaderActions } from "./TransformPaneHeaderActions";
@@ -103,6 +104,8 @@ function TransformQueryPageBody({
const isEditMode = !!route.path?.includes("/edit");
useRegisterMetabotTransformContext(transform, source);
+ const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();
+
const {
checkData,
isCheckingDependencies,
@@ -137,16 +140,20 @@ function TransformQueryPageBody({
}, [source.type, isEditMode, setSourceAndRejectProposed, transform.source]);
const handleSave = async () => {
- if (isNotDraftSource(source)) {
- await handleInitialSave({
- id: transform.id,
- source,
- });
-
- if (isEditMode) {
- dispatch(push(Urls.transform(transform.id)));
+ if (!isCompleteSource(source)) {
+ return;
+ }
+ if ("source-incremental-strategy" in source) {
+ const confirmed = await confirmIfQueryIsComplex(source);
+ if (!confirmed) {
+ return;
}
}
+ await handleInitialSave({ id: transform.id, source });
+
+ if (isEditMode) {
+ dispatch(push(Urls.transform(transform.id)));
+ }
};
const handleCancel = () => {
@@ -228,6 +235,7 @@ function TransformQueryPageBody({
isEnabled={isDirty && !isSaving && !isCheckingDependencies}
onConfirm={rejectProposed}
/>
+ {modal}
</>
);
}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
index e210ab2595f..b4d36394c9c 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
@@ -1,25 +1,55 @@
+import { useFormikContext } from "formik";
import { t } from "ttag";
import _ from "underscore";
import { Form, FormInlineUpdater, FormProvider } from "metabase/forms";
import { useMetadataToasts } from "metabase/metadata/hooks";
-import type { Transform } from "metabase-types/api";
-
+import type { IncrementalSettingsFormValues } from "metabase-enterprise/transforms/components/IncrementalTransform";
import {
IncrementalTransformSettings,
useUpdateIncrementalSettings,
-} from "../../../components/IncrementalTransform";
+} from "metabase-enterprise/transforms/components/IncrementalTransform";
+import { useQueryComplexityChecks } from "metabase-enterprise/transforms/components/QueryComplexityWarning";
+import type { Transform } from "metabase-types/api";
type UpdateIncrementalSettingsProps = {
transform: Transform;
};
+const IncrementalTransformSettingsWrapper = ({
+ transform,
+}: UpdateIncrementalSettingsProps) => {
+ const { values, setFieldValue } =
+ useFormikContext<IncrementalSettingsFormValues>();
+ const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();
+
+ const handleIncrementalChange = async (value: boolean) => {
+ if (value) {
+ const confirmed = await confirmIfQueryIsComplex(transform.source);
+ if (!confirmed) {
+ return;
+ }
+ }
+ setFieldValue("incremental", value);
+ };
+
+ return (
+ <>
+ <IncrementalTransformSettings
+ source={transform.source}
+ incremental={values.incremental}
+ onIncrementalChange={handleIncrementalChange}
+ variant="standalone"
+ />
+ {modal}
+ </>
+ );
+};
+
export const UpdateIncrementalSettings = ({
transform,
}: UpdateIncrementalSettingsProps) => {
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
- const { initialValues, validationSchema, updateIncrementalSettings } =
- useUpdateIncrementalSettings(transform);
const showSuccessToast = () =>
sendSuccessToast(t`Incremental transformation settings updated`);
@@ -27,6 +57,9 @@ export const UpdateIncrementalSettings = ({
const showErrorToast = () =>
sendErrorToast(t`Failed to update incremental transformation settings`);
+ const { initialValues, validationSchema, updateIncrementalSettings } =
+ useUpdateIncrementalSettings(transform);
+
return (
<FormProvider
initialValues={initialValues}
@@ -40,11 +73,7 @@ export const UpdateIncrementalSettings = ({
onSuccess={showSuccessToast}
onError={showErrorToast}
/>
- <IncrementalTransformSettings
- source={transform.source}
- checkOnMount
- variant="standalone"
- />
+ <IncrementalTransformSettingsWrapper transform={transform} />
</Form>
</FormProvider>
);
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
index e8e381f5ddd..0e10f6d5675 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms/utils.ts
@@ -5,6 +5,7 @@ import { hasFeature } from "metabase/admin/databases/utils";
import { parseTimestamp } from "metabase/lib/time-dayjs";
import { isNotNull } from "metabase/lib/types";
import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
Database,
DatabaseId,
@@ -170,7 +171,7 @@ export function isSameSource(
return false;
}
-export function isNotDraftSource(
+export function isCompleteSource(
source: DraftTransformSource,
): source is TransformSource {
return source.type !== "python" || source["source-database"] != null;
@@ -204,3 +205,25 @@ export function getValidationResult(query: Lib.Query): ValidationResult {
return { isValid: Lib.canSave(query, "question") };
}
+
+export const getLibQuery = (
+ source: DraftTransformSource,
+ metadata: Metadata,
+) => {
+ if (source.type !== "query") {
+ return null;
+ }
+ return Lib.fromJsQueryAndMetadata(metadata, source.query);
+};
+
+// Check if this is an MBQL query (not native SQL or Python)
+export const isMbqlQuery = (
+ source: DraftTransformSource,
+ metadata: Metadata,
+) => {
+ const query = getLibQuery(source, metadata);
+ if (!query) {
+ return false;
+ }
+ return !Lib.queryDisplayInfo(query).isNative;
+};
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index ef8fb7e5e78..4b53c41a959 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -256,7 +256,7 @@ export type ExtractColumnsFromQueryResponse = {
export type CheckQueryComplexityRequest = string;
-export type CheckQueryComplexityResponse = {
+export type QueryComplexity = {
is_simple: boolean;
reason: string;
};
diff --git a/frontend/src/metabase/lib/formatting/strings.ts b/frontend/src/metabase/lib/formatting/strings.ts
index 1d156d79c59..247e5563def 100644
--- a/frontend/src/metabase/lib/formatting/strings.ts
+++ b/frontend/src/metabase/lib/formatting/strings.ts
@@ -17,6 +17,10 @@ export function capitalize(str: string, { lowercase = true } = {}) {
return firstChar + rest;
}
+export function uncapitalize(str: string) {
+ return str.charAt(0).toLowerCase() + str.slice(1);
+}
+
export function inflect(
str: string,
count: number,
diff --git a/frontend/src/metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.tsx b/frontend/src/metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.tsx
index 67c5d0a20a9..8888af18c39 100644
--- a/frontend/src/metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.tsx
+++ b/frontend/src/metabase/query_builder/components/NativeQueryEditor/CodeMirrorEditor/CodeMirrorEditor.tsx
@@ -41,6 +41,7 @@ export type CodeMirrorEditorProps = {
onCursorMoveOverCardTag?: (id: CardId) => void;
onRightClickSelection?: () => void;
onSelectionChange?: (range: SelectionRange[]) => void;
+ onBlur?: () => void;
};
export interface CodeMirrorEditorRef {
@@ -65,6 +66,7 @@ export const CodeMirrorEditor = forwardRef<
onRightClickSelection,
onCursorMoveOverCardTag,
onFormatQuery,
+ onBlur,
},
ref,
) {
@@ -165,6 +167,7 @@ export const CodeMirrorEditor = forwardRef<
placeholder={placeholder}
highlightRanges={highlightedRanges}
onFormat={onFormatQuery}
+ onBlur={onBlur}
/>
);
});
diff --git a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
index d71a9721fdf..22c9c0fb9b2 100644
--- a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
+++ b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditor.tsx
@@ -117,6 +117,7 @@ type OwnProps = {
onSetDatabaseId?: (id: DatabaseId) => void;
databaseIsDisabled?: (database: Database) => boolean;
topBarInnerContent?: ReactNode;
+ onBlur?: () => void;
};
interface ExplicitSizeProps {
@@ -261,6 +262,7 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {
highlightedLineNumbers,
placeholder,
extensions,
+ onBlur,
} = this.props;
const dragHandle = resizable ? (
@@ -363,6 +365,7 @@ class NativeQueryEditor extends Component<Props, NativeQueryEditorState> {
onFormatQuery={
canFormatQuery ? this.handleFormatQuery : undefined
}
+ onBlur={onBlur}
/>
<Stack m="1rem" gap="md" mt="auto">
diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
index 27ff145ba9a..0955ca79b2b 100644
--- a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditor.tsx
@@ -26,6 +26,7 @@ type QueryEditorProps = {
onChangeUiState: (newUiState: QueryEditorUiState) => void;
onAcceptProposed?: () => void;
onRejectProposed?: () => void;
+ onBlur?: () => void;
topBarInnerContent?: ReactNode;
};
@@ -38,6 +39,7 @@ export function QueryEditor({
onChangeUiState,
onAcceptProposed,
onRejectProposed,
+ onBlur,
topBarInnerContent,
}: QueryEditorProps) {
const {
@@ -112,6 +114,7 @@ export function QueryEditor({
onChangeNativeEditorSelection={setSelectionRange}
onAcceptProposed={onAcceptProposed}
onRejectProposed={onRejectProposed}
+ onBlur={onBlur}
topBarInnerContent={topBarInnerContent}
/>
<QueryEditorVisualization
diff --git a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditorBody/QueryEditorBody.tsx b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditorBody/QueryEditorBody.tsx
index c22ae328175..fe52fad41a5 100644
--- a/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditorBody/QueryEditorBody.tsx
+++ b/frontend/src/metabase/querying/editor/components/QueryEditor/QueryEditorBody/QueryEditorBody.tsx
@@ -67,6 +67,7 @@ type QueryEditorBodyProps = {
onOpenModal: (type: QueryModalType) => void;
onAcceptProposed?: () => void;
onRejectProposed?: () => void;
+ onBlur?: () => void;
topBarInnerContent?: ReactNode;
};
@@ -96,6 +97,7 @@ export function QueryEditorBody({
onOpenModal,
onAcceptProposed,
onRejectProposed,
+ onBlur,
topBarInnerContent,
}: QueryEditorBodyProps) {
const [isResizing, setIsResizing] = useState(false);
@@ -161,6 +163,7 @@ export function QueryEditorBody({
onOpenModal={onOpenModal}
onAcceptProposed={onAcceptProposed}
onRejectProposed={onRejectProposed}
+ onBlur={onBlur}
topBarInnerContent={topBarInnerContent}
/>
) : (