Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

prevent removing table tag from existing incremental transforms (#75101)

Nicola Mometto3d6b88a2a104
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index d635e274342..0ddb5c69dd8 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2330,6 +2330,7 @@
;; :model/TransformRun directly. Remove once the module is agnostic to global
;; versus workspace transforms. See https://linear.app/metabase/issue/BOT-1143
:model-imports #{:model/Database
+ :model/Field
:model/Table
:model/Transform
:model/TransformRun}
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index bca04ac97ea..8daa0919694 100644
--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -1,3 +1,4 @@
+import { useDisclosure } from "@mantine/hooks";
import { useEffect, useLayoutEffect, useMemo, useState } from "react";
import type { Route, RouteProps } from "react-router";
import { push } from "react-router-redux";
@@ -10,6 +11,7 @@ import {
useUpdateTransformMutation,
} from "metabase/api";
import { getErrorMessage } from "metabase/api/utils";
+import { ConfirmModal } from "metabase/common/components/ConfirmModal";
import { EmptyState } from "metabase/common/components/EmptyState/EmptyState";
import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
@@ -22,6 +24,7 @@ import {
} from "metabase/plugins";
import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";
import { useDispatch, useSelector } from "metabase/redux";
+import { getMetadata } from "metabase/selectors/metadata";
import { useRegisterMetabotTransformContext } from "metabase/transforms/hooks/use-register-transform-metabot-context";
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
import { Box, Center, Group, Icon } from "metabase/ui";
@@ -33,6 +36,11 @@ import type {
Transform,
} from "metabase-types/api";
+import {
+ buildIncrementalSource,
+ buildIncrementalTarget,
+ getInitialValues,
+} from "../../components/IncrementalTransform/form";
import {
TransformEditor,
type TransformEditorProps,
@@ -42,6 +50,7 @@ import { useSourceState } from "../../hooks/use-source-state";
import { isCompleteSource } from "../../utils";
import { TransformPaneHeaderActions } from "./TransformPaneHeaderActions";
+import { isMissingIncrementalTableTag } from "./utils";
type TransformQueryPageParams = {
transformId: string;
@@ -110,6 +119,7 @@ function TransformQueryPageBody({
initialSource: transform.source,
});
const dispatch = useDispatch();
+ const metadata = useSelector(getMetadata);
const isRemoteSyncReadOnly = useSelector(
PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,
);
@@ -118,6 +128,10 @@ function TransformQueryPageBody({
useUpdateTransformMutation();
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
const isEditMode = !readOnly && !!route.path?.includes("/edit");
+ const [
+ isTurnOffIncrementalShown,
+ { open: openTurnOffIncremental, close: closeTurnOffIncremental },
+ ] = useDisclosure(false);
const lastRunError = useMemo(() => {
if (!transform.last_run) {
@@ -187,9 +201,29 @@ function TransformQueryPageBody({
if (!isCompleteSource(source)) {
return;
}
+ // Editing the SQL of an existing incremental transform to drop the table variable
+ // would leave it in a broken state (and the backend rejects it). Warn first, and on
+ // confirmation turn off incremental processing as part of the save.
+ if (isMissingIncrementalTableTag(transform, source, metadata)) {
+ openTurnOffIncremental();
+ return;
+ }
await handleInitialSave({ id: transform.id, source });
};
+ const handleConfirmTurnOffIncremental = async () => {
+ if (!isCompleteSource(source)) {
+ return;
+ }
+ closeTurnOffIncremental();
+ const values = getInitialValues({ incremental: false });
+ await handleInitialSave({
+ id: transform.id,
+ source: buildIncrementalSource(source, values),
+ target: buildIncrementalTarget(transform.target, values),
+ });
+ };
+
const handleCancel = () => {
if (isEditMode) {
dispatch(push(Urls.transform(transform.id)));
@@ -279,6 +313,14 @@ function TransformQueryPageBody({
onClose={handleCloseConfirmation}
/>
)}
+ <ConfirmModal
+ opened={isTurnOffIncrementalShown}
+ title={t`Turn off incremental processing?`}
+ message={t`Removing the table variable used by the incremental filter will turn off incremental processing for this transform, so it will reprocess all rows on every run.`}
+ confirmButtonText={t`Turn off incremental processing`}
+ onConfirm={handleConfirmTurnOffIncremental}
+ onClose={closeTurnOffIncremental}
+ />
<LeaveRouteConfirmModal
route={route as Route}
isEnabled={isDirty && !isSaving && !isCheckingDependencies}
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.ts
new file mode 100644
index 00000000000..72a0dae6e09
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.ts
@@ -0,0 +1,35 @@
+import { getLibQuery } from "metabase/transforms/utils";
+import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
+import type { DraftTransformSource, Transform } from "metabase-types/api";
+
+/**
+ * True when saving `source` would leave an existing incremental (table-incremental)
+ * native transform without the table variable its incremental filter relies on.
+ *
+ * Mirrors the backend validation in `metabase.transforms.crud`: a table-incremental
+ * native transform requires a table template tag for the incremental range filter to be
+ * injected into. Removing it leaves the transform in a broken state, so we warn the user
+ * and offer to turn off incremental processing instead.
+ */
+export function isMissingIncrementalTableTag(
+ transform: Transform,
+ source: DraftTransformSource,
+ metadata: Metadata,
+): boolean {
+ if (transform.target.type !== "table-incremental") {
+ return false;
+ }
+ if (transform.source_type !== "native" || source.type !== "query") {
+ return false;
+ }
+
+ const query = getLibQuery(source, metadata);
+ const hasTableTag = query
+ ? Object.values(Lib.templateTags(query)).some(
+ (tag) => tag.type === "table" && tag["table-id"] != null,
+ )
+ : false;
+
+ return !hasTableTag;
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.ts
new file mode 100644
index 00000000000..a25c72986b3
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.ts
@@ -0,0 +1,112 @@
+import { createMockMetadata } from "__support__/metadata";
+import type { QueryTransformSource, TemplateTags } from "metabase-types/api";
+import {
+ createMockNativeDatasetQuery,
+ createMockNativeQuery,
+ createMockStructuredDatasetQuery,
+ createMockTemplateTag,
+ createMockTransform,
+ createMockTransformTarget,
+} from "metabase-types/api/mocks";
+import {
+ SAMPLE_DB_ID,
+ createSampleDatabase,
+} from "metabase-types/api/mocks/presets";
+
+import { isMissingIncrementalTableTag } from "./utils";
+
+const metadata = createMockMetadata({ databases: [createSampleDatabase()] });
+
+const mbqlSource: QueryTransformSource = {
+ type: "query",
+ query: createMockStructuredDatasetQuery({ database: SAMPLE_DB_ID }),
+};
+
+function createNativeSource(templateTags?: TemplateTags): QueryTransformSource {
+ return {
+ type: "query",
+ query: createMockNativeDatasetQuery({
+ database: SAMPLE_DB_ID,
+ native: createMockNativeQuery({
+ query: "SELECT * FROM table",
+ "template-tags": templateTags,
+ }),
+ }),
+ };
+}
+
+describe("isMissingIncrementalTableTag", () => {
+ it("returns false for a non-incremental transform", () => {
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table" }),
+ });
+
+ expect(
+ isMissingIncrementalTableTag(transform, createNativeSource(), metadata),
+ ).toBe(false);
+ });
+
+ it("returns false for an mbql incremental transform", () => {
+ const transform = createMockTransform({
+ source_type: "mbql",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(isMissingIncrementalTableTag(transform, mbqlSource, metadata)).toBe(
+ false,
+ );
+ });
+
+ it("returns true when the edited native source has a non-table variable", () => {
+ const source = createNativeSource({
+ snippet1: createMockTemplateTag({
+ id: "1",
+ name: "snippet1",
+ type: "snippet",
+ }),
+ });
+
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(
+ true,
+ );
+ });
+
+ it("returns true when the edited native source has no variables", () => {
+ const source = createNativeSource();
+
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(
+ true,
+ );
+ });
+
+ it("returns false when the edited native source still has a table variable", () => {
+ const source = createNativeSource({
+ my_table: createMockTemplateTag({
+ id: "1",
+ name: "my_table",
+ type: "table",
+ "table-id": 42,
+ }),
+ });
+
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(
+ false,
+ );
+ });
+});
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index c2f125651d1..269b27c03c9 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -77,6 +77,19 @@
(:name field)
(pr-str (:base_type field))))))))
+(defn validate-incremental-table-tag!
+ "Reject a table-incremental native-query transform whose source query has no table template tag for
+ the incremental range filter to target.
+
+ Without that table variable the incremental filter has nowhere to be injected, so the transform is
+ in a broken state. This surfaces the error eagerly at create/update time instead of only when the
+ transform is run."
+ [{:keys [target] :as transform}]
+ (when (and (= :table-incremental (keyword (:type target)))
+ (transforms-base.u/native-query-transform? transform))
+ (api/check-400 (transforms-base.u/incremental-table-tag-name transform)
+ (deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))))
+
(defn get-transforms
"Get a list of transforms."
[& {:keys [last-run-start-time last-run-statuses tag-ids database-id]}]
@@ -114,6 +127,7 @@
(when (transforms-base.u/query-transform? body)
(validate-transform-query! body))
(validate-target-schema! body)
+ (validate-incremental-table-tag! body)
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
(let [tag-ids (:tag_ids body)
@@ -151,6 +165,8 @@
(validate-target-schema! new))
(when (contains? body :source)
(validate-incremental-column-type! new))
+ (when (or (contains? body :source) (contains? body :target))
+ (validate-incremental-table-tag! new))
(when (transforms-base.u/query-transform? old)
(validate-transform-query! new)
(when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 4be275657db..700bb367e7c 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -64,6 +64,32 @@
(let [query (-> transform :source :query)]
(lib/native-only-query? query))))
+(defn table-template-tag-name
+ "Return the name (key) of the table template tag in `query` that refers to `table-id`, or nil.
+
+ This is the table variable the incremental range filter is injected into (see
+ `inject-filters-into-table-tag`). When `table-id` is nil, any table template tag qualifies."
+ [query table-id]
+ (some (fn [[k v]]
+ (when (and (#{:table "table"} (:type v))
+ (or (nil? table-id) (= table-id (:table-id v))))
+ k))
+ (lib/template-tags query)))
+
+(defn incremental-table-tag-name
+ "Return the name of the table template tag the incremental range filter targets in `transform`, or
+ nil when the source query has no suitable table variable.
+
+ The range filter is injected into the table variable referring to the checkpoint field's table
+ (see `inject-filters-into-table-tag`); when no checkpoint field has been selected yet, any table
+ variable qualifies. A nil result for a table-incremental native transform is a broken state — there
+ is no table variable to filter on. This can happen when the SQL is edited to drop the table tag
+ after the transform was made incremental."
+ [{:keys [source]}]
+ (let [field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
+ table-id (when field-id (t2/select-one-fn :table_id :model/Field field-id))]
+ (table-template-tag-name (:query source) table-id)))
+
(defn python-transform?
"Check if this is a Python transform."
[transform]
@@ -262,16 +288,11 @@
[query source-range-params]
(let [{:keys [checkpoint-filter-field-id lo hi column]} source-range-params
table-id (:table-id column)
- tags (lib/template-tags query)
- tag-name (some (fn [[k v]]
- (when (and (#{:table "table"} (:type v))
- (= table-id (:table-id v)))
- k))
- tags)
+ tag-name (table-template-tag-name query table-id)
_ (when-not tag-name
(throw (ex-info "No table variable found for checkpoint field's table"
{:checkpoint-table-id table-id
- :template-tags tags})))
+ :template-tags (lib/template-tags query)})))
filters (cond-> []
lo (conj {:field-id checkpoint-filter-field-id :op :> :value (:value lo)})
hi (conj {:field-id checkpoint-filter-field-id :op :<= :value (:value hi)}))]
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 7eda140b249..13b1e59b671 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1676,37 +1676,132 @@
(deftest update-tag-ids-with-stale-checkpoint-field-test
(testing "Updating only tag_ids succeeds even when the checkpoint field has since been deleted (GDGT-2410)"
(mt/with-premium-features #{}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)]
+ (with-transform-cleanup! [table-name "stale_checkpoint_test"]
+ ;; Insert directly to simulate a legacy broken transform whose checkpoint field was
+ ;; later deleted (e.g. its source table was re-synced), as happened with production
+ ;; transform 2344. A non-existent field id is equivalent to a deleted field: the
+ ;; validator's (select-one :model/Field id) returns nil either way. Such transforms
+ ;; predate the table-tag/checkpoint validation and can no longer be created via the API.
+ (mt/with-temp [:model/TransformTag {tag-id :id} {:name "stale-checkpoint-tag"}
+ :model/Transform {transform-id :id}
+ {:name "Stale Checkpoint Transform"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name, category FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id Integer/MAX_VALUE}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}}]
+ (transform.model/update-transform-tags! transform-id [tag-id])
+ (testing "removing the tag (tag_ids-only update) should not re-validate the stale checkpoint field"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (format "transform/%d" transform-id)
+ {:tag_ids []})]
+ (is (= [] (:tag_ids updated)))))))))))))
+
+(defn- native-query-with-table-tag
+ "Build a native query map with a `source_table` table template tag referring to `table-id`."
+ [sql table-id]
+ {:database (mt/id)
+ :type :native
+ :native {:query sql
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}})
+
+(deftest incremental-table-tag-validated-on-create-test
+ (mt/with-premium-features #{}
+ (testing "POST /api/transform rejects a table-incremental native query with no table template tag (GDGT-2524)"
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
(let [schema (get-test-schema)
id-field-id (mt/id :transforms_products :id)]
- (mt/with-temp [:model/TransformTag {tag-id :id} {:name "stale-checkpoint-tag"}]
- (with-transform-cleanup! [table-name "stale_checkpoint_test"]
- (let [created (mt/user-http-request :crowberto :post 200 "transform"
- {:name "Stale Checkpoint Transform"
- :tag_ids [tag-id]
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, category FROM transforms_products")
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter-field-id id-field-id}}
- :target {:type "table-incremental"
- :schema schema
- :name table-name
- :target-incremental-strategy {:type "append"}}})
- source (t2/select-one-fn :source :model/Transform (:id created))]
- ;; Simulate the checkpoint field being deleted out from under the transform
- ;; (e.g. its source table was re-synced), as happened with production transform 2344.
- ;; A non-existent field id is equivalent to a deleted field: the validator's
- ;; (select-one :model/Field id) returns nil either way.
- (t2/update! :model/Transform (:id created)
- {:source (assoc-in source [:source-incremental-strategy :checkpoint-filter-field-id]
- Integer/MAX_VALUE)})
- (testing "removing the tag (tag_ids-only update) should not re-validate the stale checkpoint field"
- (let [updated (mt/user-http-request :crowberto :put 200
- (format "transform/%d" (:id created))
- {:tag_ids []})]
- (is (= [] (:tag_ids updated))))))))))))))
+ (testing "with a checkpoint field selected"
+ (let [response (mt/user-http-request :crowberto :post 400 "transform"
+ {:name "No Table Tag"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id id-field-id}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name "no_table_tag"
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"requires a table variable" response))))
+ (testing "even without a checkpoint field selected"
+ (let [response (mt/user-http-request :crowberto :post 400 "transform"
+ {:name "No Table Tag No Checkpoint"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name FROM transforms_products")}
+ :target {:type "table-incremental"
+ :schema schema
+ :name "no_table_tag_no_checkpoint"
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"requires a table variable" response))))))))))
+
+(deftest incremental-table-tag-validated-on-update-test
+ (mt/with-premium-features #{}
+ (testing "PUT /api/transform rejects removing the table tag from an incremental transform (GDGT-2524)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)
+ id-field-id (mt/id :transforms_products :id)]
+ (testing "removing the table tag from the source query of an existing incremental transform"
+ (with-transform-cleanup! [table-name "remove_table_tag_test"]
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Has Table Tag"
+ :source {:type "query"
+ :query (native-query-with-table-tag
+ "SELECT id, name FROM {{source_table}}"
+ (mt/id :transforms_products))
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id id-field-id}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}}]
+ (let [response (mt/user-http-request :crowberto :put 400
+ (format "transform/%d" transform-id)
+ {:source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id id-field-id}}})]
+ (is (string? response))
+ (is (re-find #"requires a table variable" response))))))
+ (testing "switching a non-incremental transform to incremental without a table tag"
+ (with-transform-cleanup! [table-name "switch_no_tag_test"]
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Plain Transform"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name FROM transforms_products")}
+ :target {:type "table" :schema schema :name table-name}}]
+ (let [response (mt/user-http-request :crowberto :put 400
+ (format "transform/%d" transform-id)
+ {:source {:type "query"
+ :query (lib/native-query (mt/metadata-provider)
+ "SELECT id, name FROM transforms_products")
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id id-field-id}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}})]
+ (is (string? response))
+ (is (re-find #"requires a table variable" response))))))))))))
(deftest search-filters-transform-source-types-test
(mt/with-premium-features #{}