The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
prevent removing table tag from existing incremental transforms (#75101)
Nicola Mometto3d6b88a2a104
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex d635e274342..0ddb5c69dd8 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2330,6 +2330,7 @@;; :model/TransformRun directly. Remove once the module is agnostic to global;; versus workspace transforms. See https://linear.app/metabase/issue/BOT-1143:model-imports #{:model/Database+ :model/Field:model/Table:model/Transform:model/TransformRun}diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsxindex bca04ac97ea..8daa0919694 100644--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx@@ -1,3 +1,4 @@+import { useDisclosure } from "@mantine/hooks";import { useEffect, useLayoutEffect, useMemo, useState } from "react";import type { Route, RouteProps } from "react-router";import { push } from "react-router-redux";@@ -10,6 +11,7 @@ import {useUpdateTransformMutation,} from "metabase/api";import { getErrorMessage } from "metabase/api/utils";+import { ConfirmModal } from "metabase/common/components/ConfirmModal";import { EmptyState } from "metabase/common/components/EmptyState/EmptyState";import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";@@ -22,6 +24,7 @@ import {} from "metabase/plugins";import { getInitialUiState } from "metabase/querying/editor/components/QueryEditor";import { useDispatch, useSelector } from "metabase/redux";+import { getMetadata } from "metabase/selectors/metadata";import { useRegisterMetabotTransformContext } from "metabase/transforms/hooks/use-register-transform-metabot-context";import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";import { Box, Center, Group, Icon } from "metabase/ui";@@ -33,6 +36,11 @@ import type {Transform,} from "metabase-types/api";+import {+ buildIncrementalSource,+ buildIncrementalTarget,+ getInitialValues,+} from "../../components/IncrementalTransform/form";import {TransformEditor,type TransformEditorProps,@@ -42,6 +50,7 @@ import { useSourceState } from "../../hooks/use-source-state";import { isCompleteSource } from "../../utils";import { TransformPaneHeaderActions } from "./TransformPaneHeaderActions";+import { isMissingIncrementalTableTag } from "./utils";type TransformQueryPageParams = {transformId: string;@@ -110,6 +119,7 @@ function TransformQueryPageBody({initialSource: transform.source,});const dispatch = useDispatch();+ const metadata = useSelector(getMetadata);const isRemoteSyncReadOnly = useSelector(PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,);@@ -118,6 +128,10 @@ function TransformQueryPageBody({useUpdateTransformMutation();const { sendSuccessToast, sendErrorToast } = useMetadataToasts();const isEditMode = !readOnly && !!route.path?.includes("/edit");+ const [+ isTurnOffIncrementalShown,+ { open: openTurnOffIncremental, close: closeTurnOffIncremental },+ ] = useDisclosure(false);const lastRunError = useMemo(() => {if (!transform.last_run) {@@ -187,9 +201,29 @@ function TransformQueryPageBody({if (!isCompleteSource(source)) {return;}+ // Editing the SQL of an existing incremental transform to drop the table variable+ // would leave it in a broken state (and the backend rejects it). Warn first, and on+ // confirmation turn off incremental processing as part of the save.+ if (isMissingIncrementalTableTag(transform, source, metadata)) {+ openTurnOffIncremental();+ return;+ }await handleInitialSave({ id: transform.id, source });};+ const handleConfirmTurnOffIncremental = async () => {+ if (!isCompleteSource(source)) {+ return;+ }+ closeTurnOffIncremental();+ const values = getInitialValues({ incremental: false });+ await handleInitialSave({+ id: transform.id,+ source: buildIncrementalSource(source, values),+ target: buildIncrementalTarget(transform.target, values),+ });+ };+const handleCancel = () => {if (isEditMode) {dispatch(push(Urls.transform(transform.id)));@@ -279,6 +313,14 @@ function TransformQueryPageBody({onClose={handleCloseConfirmation}/>)}+ <ConfirmModal+ opened={isTurnOffIncrementalShown}+ title={t`Turn off incremental processing?`}+ message={t`Removing the table variable used by the incremental filter will turn off incremental processing for this transform, so it will reprocess all rows on every run.`}+ confirmButtonText={t`Turn off incremental processing`}+ onConfirm={handleConfirmTurnOffIncremental}+ onClose={closeTurnOffIncremental}+ /><LeaveRouteConfirmModalroute={route as Route}isEnabled={isDirty && !isSaving && !isCheckingDependencies}diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.tsnew file mode 100644index 00000000000..72a0dae6e09--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.ts@@ -0,0 +1,35 @@+import { getLibQuery } from "metabase/transforms/utils";+import * as Lib from "metabase-lib";+import type Metadata from "metabase-lib/v1/metadata/Metadata";+import type { DraftTransformSource, Transform } from "metabase-types/api";++/**+ * True when saving `source` would leave an existing incremental (table-incremental)+ * native transform without the table variable its incremental filter relies on.+ *+ * Mirrors the backend validation in `metabase.transforms.crud`: a table-incremental+ * native transform requires a table template tag for the incremental range filter to be+ * injected into. Removing it leaves the transform in a broken state, so we warn the user+ * and offer to turn off incremental processing instead.+ */+export function isMissingIncrementalTableTag(+ transform: Transform,+ source: DraftTransformSource,+ metadata: Metadata,+): boolean {+ if (transform.target.type !== "table-incremental") {+ return false;+ }+ if (transform.source_type !== "native" || source.type !== "query") {+ return false;+ }++ const query = getLibQuery(source, metadata);+ const hasTableTag = query+ ? Object.values(Lib.templateTags(query)).some(+ (tag) => tag.type === "table" && tag["table-id"] != null,+ )+ : false;++ return !hasTableTag;+}diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.tsnew file mode 100644index 00000000000..a25c72986b3--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/utils.unit.spec.ts@@ -0,0 +1,112 @@+import { createMockMetadata } from "__support__/metadata";+import type { QueryTransformSource, TemplateTags } from "metabase-types/api";+import {+ createMockNativeDatasetQuery,+ createMockNativeQuery,+ createMockStructuredDatasetQuery,+ createMockTemplateTag,+ createMockTransform,+ createMockTransformTarget,+} from "metabase-types/api/mocks";+import {+ SAMPLE_DB_ID,+ createSampleDatabase,+} from "metabase-types/api/mocks/presets";++import { isMissingIncrementalTableTag } from "./utils";++const metadata = createMockMetadata({ databases: [createSampleDatabase()] });++const mbqlSource: QueryTransformSource = {+ type: "query",+ query: createMockStructuredDatasetQuery({ database: SAMPLE_DB_ID }),+};++function createNativeSource(templateTags?: TemplateTags): QueryTransformSource {+ return {+ type: "query",+ query: createMockNativeDatasetQuery({+ database: SAMPLE_DB_ID,+ native: createMockNativeQuery({+ query: "SELECT * FROM table",+ "template-tags": templateTags,+ }),+ }),+ };+}++describe("isMissingIncrementalTableTag", () => {+ it("returns false for a non-incremental transform", () => {+ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table" }),+ });++ expect(+ isMissingIncrementalTableTag(transform, createNativeSource(), metadata),+ ).toBe(false);+ });++ it("returns false for an mbql incremental transform", () => {+ const transform = createMockTransform({+ source_type: "mbql",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(isMissingIncrementalTableTag(transform, mbqlSource, metadata)).toBe(+ false,+ );+ });++ it("returns true when the edited native source has a non-table variable", () => {+ const source = createNativeSource({+ snippet1: createMockTemplateTag({+ id: "1",+ name: "snippet1",+ type: "snippet",+ }),+ });++ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(+ true,+ );+ });++ it("returns true when the edited native source has no variables", () => {+ const source = createNativeSource();++ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(+ true,+ );+ });++ it("returns false when the edited native source still has a table variable", () => {+ const source = createNativeSource({+ my_table: createMockTemplateTag({+ id: "1",+ name: "my_table",+ type: "table",+ "table-id": 42,+ }),+ });++ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(isMissingIncrementalTableTag(transform, source, metadata)).toBe(+ false,+ );+ });+});diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex c2f125651d1..269b27c03c9 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -77,6 +77,19 @@(:name field)(pr-str (:base_type field))))))))+(defn validate-incremental-table-tag!+ "Reject a table-incremental native-query transform whose source query has no table template tag for+ the incremental range filter to target.++ Without that table variable the incremental filter has nowhere to be injected, so the transform is+ in a broken state. This surfaces the error eagerly at create/update time instead of only when the+ transform is run."+ [{:keys [target] :as transform}]+ (when (and (= :table-incremental (keyword (:type target)))+ (transforms-base.u/native-query-transform? transform))+ (api/check-400 (transforms-base.u/incremental-table-tag-name transform)+ (deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))))+(defn get-transforms"Get a list of transforms."[& {:keys [last-run-start-time last-run-statuses tag-ids database-id]}]@@ -114,6 +127,7 @@(when (transforms-base.u/query-transform? body)(validate-transform-query! body))(validate-target-schema! body)+ (validate-incremental-table-tag! body)(let [creator-id (or creator-id api/*current-user-id*)transform (t2/with-transaction [_](let [tag-ids (:tag_ids body)@@ -151,6 +165,8 @@(validate-target-schema! new))(when (contains? body :source)(validate-incremental-column-type! new))+ (when (or (contains? body :source) (contains? body :target))+ (validate-incremental-table-tag! new))(when (transforms-base.u/query-transform? old)(validate-transform-query! new)(when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 4be275657db..700bb367e7c 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -64,6 +64,32 @@(let [query (-> transform :source :query)](lib/native-only-query? query))))+(defn table-template-tag-name+ "Return the name (key) of the table template tag in `query` that refers to `table-id`, or nil.++ This is the table variable the incremental range filter is injected into (see+ `inject-filters-into-table-tag`). When `table-id` is nil, any table template tag qualifies."+ [query table-id]+ (some (fn [[k v]]+ (when (and (#{:table "table"} (:type v))+ (or (nil? table-id) (= table-id (:table-id v))))+ k))+ (lib/template-tags query)))++(defn incremental-table-tag-name+ "Return the name of the table template tag the incremental range filter targets in `transform`, or+ nil when the source query has no suitable table variable.++ The range filter is injected into the table variable referring to the checkpoint field's table+ (see `inject-filters-into-table-tag`); when no checkpoint field has been selected yet, any table+ variable qualifies. A nil result for a table-incremental native transform is a broken state — there+ is no table variable to filter on. This can happen when the SQL is edited to drop the table tag+ after the transform was made incremental."+ [{:keys [source]}]+ (let [field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])+ table-id (when field-id (t2/select-one-fn :table_id :model/Field field-id))]+ (table-template-tag-name (:query source) table-id)))+(defn python-transform?"Check if this is a Python transform."[transform]@@ -262,16 +288,11 @@[query source-range-params](let [{:keys [checkpoint-filter-field-id lo hi column]} source-range-paramstable-id (:table-id column)- tags (lib/template-tags query)- tag-name (some (fn [[k v]]- (when (and (#{:table "table"} (:type v))- (= table-id (:table-id v)))- k))- tags)+ tag-name (table-template-tag-name query table-id)_ (when-not tag-name(throw (ex-info "No table variable found for checkpoint field's table"{:checkpoint-table-id table-id- :template-tags tags})))+ :template-tags (lib/template-tags query)})))filters (cond-> []lo (conj {:field-id checkpoint-filter-field-id :op :> :value (:value lo)})hi (conj {:field-id checkpoint-filter-field-id :op :<= :value (:value hi)}))]diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 7eda140b249..13b1e59b671 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1676,37 +1676,132 @@(deftest update-tag-ids-with-stale-checkpoint-field-test(testing "Updating only tag_ids succeeds even when the checkpoint field has since been deleted (GDGT-2410)"(mt/with-premium-features #{}+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)]+ (with-transform-cleanup! [table-name "stale_checkpoint_test"]+ ;; Insert directly to simulate a legacy broken transform whose checkpoint field was+ ;; later deleted (e.g. its source table was re-synced), as happened with production+ ;; transform 2344. A non-existent field id is equivalent to a deleted field: the+ ;; validator's (select-one :model/Field id) returns nil either way. Such transforms+ ;; predate the table-tag/checkpoint validation and can no longer be created via the API.+ (mt/with-temp [:model/TransformTag {tag-id :id} {:name "stale-checkpoint-tag"}+ :model/Transform {transform-id :id}+ {:name "Stale Checkpoint Transform"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name, category FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id Integer/MAX_VALUE}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}}]+ (transform.model/update-transform-tags! transform-id [tag-id])+ (testing "removing the tag (tag_ids-only update) should not re-validate the stale checkpoint field"+ (let [updated (mt/user-http-request :crowberto :put 200+ (format "transform/%d" transform-id)+ {:tag_ids []})]+ (is (= [] (:tag_ids updated)))))))))))))++(defn- native-query-with-table-tag+ "Build a native query map with a `source_table` table template tag referring to `table-id`."+ [sql table-id]+ {:database (mt/id)+ :type :native+ :native {:query sql+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}})++(deftest incremental-table-tag-validated-on-create-test+ (mt/with-premium-features #{}+ (testing "POST /api/transform rejects a table-incremental native query with no table template tag (GDGT-2524)"(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/dataset transforms-dataset/transforms-test(let [schema (get-test-schema)id-field-id (mt/id :transforms_products :id)]- (mt/with-temp [:model/TransformTag {tag-id :id} {:name "stale-checkpoint-tag"}]- (with-transform-cleanup! [table-name "stale_checkpoint_test"]- (let [created (mt/user-http-request :crowberto :post 200 "transform"- {:name "Stale Checkpoint Transform"- :tag_ids [tag-id]- :source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, category FROM transforms_products")- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter-field-id id-field-id}}- :target {:type "table-incremental"- :schema schema- :name table-name- :target-incremental-strategy {:type "append"}}})- source (t2/select-one-fn :source :model/Transform (:id created))]- ;; Simulate the checkpoint field being deleted out from under the transform- ;; (e.g. its source table was re-synced), as happened with production transform 2344.- ;; A non-existent field id is equivalent to a deleted field: the validator's- ;; (select-one :model/Field id) returns nil either way.- (t2/update! :model/Transform (:id created)- {:source (assoc-in source [:source-incremental-strategy :checkpoint-filter-field-id]- Integer/MAX_VALUE)})- (testing "removing the tag (tag_ids-only update) should not re-validate the stale checkpoint field"- (let [updated (mt/user-http-request :crowberto :put 200- (format "transform/%d" (:id created))- {:tag_ids []})]- (is (= [] (:tag_ids updated))))))))))))))+ (testing "with a checkpoint field selected"+ (let [response (mt/user-http-request :crowberto :post 400 "transform"+ {:name "No Table Tag"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id id-field-id}}+ :target {:type "table-incremental"+ :schema schema+ :name "no_table_tag"+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"requires a table variable" response))))+ (testing "even without a checkpoint field selected"+ (let [response (mt/user-http-request :crowberto :post 400 "transform"+ {:name "No Table Tag No Checkpoint"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name FROM transforms_products")}+ :target {:type "table-incremental"+ :schema schema+ :name "no_table_tag_no_checkpoint"+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"requires a table variable" response))))))))))++(deftest incremental-table-tag-validated-on-update-test+ (mt/with-premium-features #{}+ (testing "PUT /api/transform rejects removing the table tag from an incremental transform (GDGT-2524)"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)+ id-field-id (mt/id :transforms_products :id)]+ (testing "removing the table tag from the source query of an existing incremental transform"+ (with-transform-cleanup! [table-name "remove_table_tag_test"]+ (mt/with-temp [:model/Transform {transform-id :id}+ {:name "Has Table Tag"+ :source {:type "query"+ :query (native-query-with-table-tag+ "SELECT id, name FROM {{source_table}}"+ (mt/id :transforms_products))+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id id-field-id}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}}]+ (let [response (mt/user-http-request :crowberto :put 400+ (format "transform/%d" transform-id)+ {:source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id id-field-id}}})]+ (is (string? response))+ (is (re-find #"requires a table variable" response))))))+ (testing "switching a non-incremental transform to incremental without a table tag"+ (with-transform-cleanup! [table-name "switch_no_tag_test"]+ (mt/with-temp [:model/Transform {transform-id :id}+ {:name "Plain Transform"+ :source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name FROM transforms_products")}+ :target {:type "table" :schema schema :name table-name}}]+ (let [response (mt/user-http-request :crowberto :put 400+ (format "transform/%d" transform-id)+ {:source {:type "query"+ :query (lib/native-query (mt/metadata-provider)+ "SELECT id, name FROM transforms_products")+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id id-field-id}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}})]+ (is (string? response))+ (is (re-find #"requires a table variable" response))))))))))))(deftest search-filters-transform-source-types-test(mt/with-premium-features #{}