The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Incremental transforms merge (#76663)
Nicola Mometto03b95225511e
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 03fa2bbac71..cc1a0045274 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -83,12 +83,12 @@(defmethod calculate-deps* :transform[_ {{:keys [query]} :source :as transform}]- (let [source-type (transforms-base.u/transform-type transform)]- (case source-type- :query (upstream-deps:query query)- :python (upstream-deps:python-transform transform)- (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)- {}))))+ (cond+ (transforms-base.u/query-transform? transform) (upstream-deps:query query)+ (transforms-base.u/python-transform? transform) (upstream-deps:python-transform transform)+ :else (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'"+ (:id transform) (-> transform :source :type))+ {})))(defmethod calculate-deps* :snippet[_ {:keys [template_tags] :as _snippet}]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex ef47624f43f..31cd603c628 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -12,6 +12,8 @@[metabase-enterprise.transforms-python.s3 :as s3][metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase.driver :as driver]+ [metabase.driver.sql :as driver.sql]+ [metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.schema :as transforms-base.schema]@@ -160,6 +162,23 @@(log/error e "Failed to transfer data using drop-create fallback strategy")(throw e))))+(defn- upsert-with-merge-strategy!+ "Upsert the staged data into `table-name`, keyed on `unique-key` (a vector of column names)."+ [driver db table-name metadata data-source unique-key]+ (let [db-id (:id db)+ staging (transforms-base.u/temp-table-name driver (namespace table-name))]+ (transforms-base.u/validate-merge-unique-key! unique-key (mapv :name (:fields metadata)))+ (create-table-and-insert-data! driver db-id (table-schema staging metadata) data-source)+ (try+ (let [[sql & params] (sql.qp/format-honeysql driver {:select [:*], :from [staging]})+ select {:query sql, :params (vec params)}+ merge-spec {:unique-key unique-key+ :columns (mapv :name (:fields metadata))}]+ (driver/execute-raw-queries! driver (driver/connection-spec driver db)+ (driver.sql/compile-merge driver table-name select merge-spec)))+ (finally+ (transforms-base.u/drop-table! driver db-id staging)))))+(defmulti ^:private transfer-file-to-db{:arglists '([driver db transform metadata temp-file])}(fn [_ _ transform _ _] (-> transform :target :type keyword)))@@ -172,15 +191,22 @@;; like non-incremental and drop-and-recreate rather than appending.(if (transforms-base.u/full-incremental-run? transform)((get-method transfer-file-to-db :table) driver db transform metadata temp-file)- ;; Normal incremental: append if table exists, create if it doesn't+ ;; Normal incremental: append (or merge) if table exists, create if it doesn't(let [table-name (transforms-base.u/qualified-table-name driver target)table-exists? (transforms-base.u/target-table-exists? transform)data-source {:type :jsonl-file- :file temp-file}]- (if (not table-exists?)+ :file temp-file}+ unique-key (transforms-base.u/merge-target-unique-key transform)]+ (cond+ (not table-exists?)(do(log/info "New table")(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))++ unique-key+ (upsert-with-merge-strategy! driver db table-name metadata data-source unique-key)++ :else(insert-data! driver db-id (table-schema table-name metadata) data-source)))))(defmethod transfer-file-to-db :tablediff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 76fccf49dc3..8315836fff5 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -113,7 +113,7 @@(tracing/with-span :tasks "task.transform.python"{:transform/id transform-id:transform/target-type (name target-type)- :transform/incremental (= :table-incremental target-type)+ :transform/incremental (transforms-base.u/incremental-target? transform):transform/full-incremental-run (transforms-base.u/full-incremental-run? transform):db/id (:id db):db/engine (name driver)}diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.cljnew file mode 100644index 00000000000..06ff70b776c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj@@ -0,0 +1,209 @@+(ns ^:mb/driver-tests metabase-enterprise.transforms.merge-test+ "Tests for the key-based `merge` incremental transform target strategy (upsert/restate)."+ (:require+ [clojure.string :as str]+ [clojure.test :refer :all]+ [metabase.driver :as driver]+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]+ [metabase.driver.sql.util :as sql.u]+ [metabase.query-processor :as qp]+ [metabase.test :as mt]+ [metabase.transforms.execute :as transforms.execute]+ [metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;; An append-only status log: `order_id` repeats across rows, `change_seq` is the checkpoint column.+(mt/defdataset merge-test+ "A tiny event-log dataset for testing the merge strategy. `order_id` is the merge key and repeats+ across rows; `change_seq` is the incremental checkpoint column."+ [["order_status"+ [{:field-name "order_id" :base-type :type/Integer}+ {:field-name "status" :base-type :type/Text}+ {:field-name "change_seq" :base-type :type/Integer}]+ [[1 "created" 1]+ [2 "created" 2]+ [3 "created" 3]]]])++;; Two key columns (order_id, region) for testing a composite merge key.+(mt/defdataset merge-composite-test+ "An event log keyed on (order_id, region), for testing a two-column merge key."+ [["order_region_status"+ [{:field-name "order_id" :base-type :type/Integer}+ {:field-name "region" :base-type :type/Text}+ {:field-name "status" :base-type :type/Text}+ {:field-name "change_seq" :base-type :type/Integer}]+ [[1 "us" "created" 1]+ [1 "eu" "created" 2]+ [2 "us" "created" 3]]]])++(defn- test-drivers []+ (mt/normal-drivers-with-feature :transforms/table))++(defn- target-table-gen [prefix]+ {:type :table+ :name prefix+ :schema (t2/select-one-fn :schema :model/Table (mt/id :order_status))+ :database (mt/id)})++(defn- merge-source-query []+ {:database (mt/id)+ :type :native+ :native {:query (format "SELECT %s, %s FROM {{order_status}} AS %s"+ (sql.u/quote-name driver/*driver* :field "order_id")+ (sql.u/quote-name driver/*driver* :field "status")+ (sql.u/quote-name driver/*driver* :field "order_status"))+ :template-tags {"order_status" {:id "order_status"+ :name "order_status"+ :display-name "Order Status"+ :type "table"+ :table-id (mt/id :order_status)+ :required true}}}})++(defn- merge-transform-payload [target-table & {:keys [validate?] :or {validate? true}}]+ (let [seq-field-id (t2/select-one-pk :model/Field :name "change_seq" :table_id (mt/id :order_status))]+ {:name "Merge Transform"+ :source_database_id (mt/id)+ :source {:type "query"+ :query (merge-source-query)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id seq-field-id}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy (cond-> {:type "merge"+ :unique-key [{:name "order_id"}]}+ (not validate?)+ (assoc :validate-unique-key? false))})}))++(defn- insert-order-status!+ "Append `rows` (vectors of `[order-id status change-seq]`) to the source table."+ [rows]+ (let [[schema table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :order_status))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ values (str/join ", " (map (fn [[oid st sq]] (format "(%d, '%s', %d)" oid st sq)) rows))+ sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema table-name)+ (str/join "," (map #(sql.u/quote-name driver/*driver* :field %)+ ["order_id" "status" "change_seq"]))+ values)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))++(defn- target-status-by-order+ "Return the target table contents as a map of `order_id` -> `status`."+ [{:keys [schema name]}]+ (let [sql (format "SELECT %s, %s FROM %s"+ (sql.u/quote-name driver/*driver* :field "order_id")+ (sql.u/quote-name driver/*driver* :field "status")+ (sql.u/quote-name driver/*driver* :table schema name))+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})+ :data :rows)]+ (into {} (map (fn [[oid st]] [(long oid) st])) rows)))++(deftest merge-builds-then-upserts-test+ (testing "a merge transform builds the full target, then upserts by key on subsequent runs"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-test+ (with-transform-cleanup! [target-table (target-table-gen "merge_upsert")]+ (mt/with-temp [:model/Transform transform (merge-transform-payload target-table)]+ (testing "first run builds the full target"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (is (= {1 "created", 2 "created", 3 "created"}+ (target-status-by-order target-table))))+ (testing "new events update existing keys in place and insert new ones"+ (insert-order-status! [[1 "shipped" 4] [4 "created" 5]])+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {1 "shipped", 2 "created", 3 "created", 4 "created"}+ (target-status-by-order target-table))+ "order 1 updated (paid->shipped), order 4 inserted")))))))))++(deftest merge-resolves-field-ids-test+ (testing "after the first build the stored unique key gets its field-ids resolved"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-test+ (with-transform-cleanup! [target-table (target-table-gen "merge_fieldids")]+ (mt/with-temp [:model/Transform transform (merge-transform-payload target-table)]+ (is (nil? (-> transform :target :target-incremental-strategy :unique-key first :field-id))+ "no field-id before the first run — the target table doesn't exist yet")+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (let [reloaded (t2/select-one :model/Transform (:id transform))+ unique-key (-> reloaded :target :target-incremental-strategy :unique-key)+ target-tbl-id (:target_table_id reloaded)+ expected (t2/select-one-pk :model/Field :name "order_id"+ :table_id target-tbl-id :active true)]+ (is (= ["order_id"] (mapv :name unique-key)))+ (is (= expected (:field-id (first unique-key)))+ "field-id resolved to the target's order_id field")))))))))++(deftest merge-composite-key-test+ (testing "a merge transform can upsert on a two-column key"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-composite-test+ (with-transform-cleanup! [target-table {:type :table+ :name "merge_composite"+ :schema (t2/select-one-fn :schema :model/Table+ (mt/id :order_region_status))+ :database (mt/id)}]+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))+ seq-field-id (t2/select-one-pk :model/Field :name "change_seq"+ :table_id (mt/id :order_region_status))+ source-query {:database (mt/id)+ :type :native+ :native {:query (format "SELECT %s, %s, %s FROM {{t}} AS %s"+ (q "order_id") (q "region") (q "status") (q "t"))+ :template-tags {"t" {:id "t"+ :name "t"+ :display-name "T"+ :type "table"+ :table-id (mt/id :order_region_status)+ :required true}}}}+ payload {:name "Composite Merge"+ :source_database_id (mt/id)+ :source {:type "query"+ :query source-query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id seq-field-id}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "merge"+ :unique-key [{:name "order_id"}+ {:name "region"}]}})}+ insert! (fn [rows]+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table+ (mt/id :order_region_status))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ values (str/join ", " (map (fn [[oid rg st sq]]+ (format "(%d, '%s', '%s', %d)" oid rg st sq))+ rows))+ sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema tbl)+ (str/join "," (map q ["order_id" "region" "status" "change_seq"]))+ values)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))+ read-target (fn []+ (let [sql (format "SELECT %s, %s, %s FROM %s"+ (q "order_id") (q "region") (q "status")+ (sql.u/quote-name driver/*driver* :table+ (:schema target-table) (:name target-table)))+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})+ :data :rows)]+ (into {} (map (fn [[oid rg st]] [[(long oid) rg] st])) rows)))]+ (mt/with-temp [:model/Transform transform payload]+ (testing "first run builds the full target keyed on (order_id, region)"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (is (= {[1 "us"] "created", [1 "eu"] "created", [2 "us"] "created"}+ (read-target))))+ (testing "new events upsert one composite key and insert another"+ (insert! [[1 "us" "shipped" 4] [3 "us" "created" 5]])+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {[1 "us"] "shipped", [1 "eu"] "created", [2 "us"] "created", [3 "us"] "created"}+ (read-target))+ "(1,us) updated, (3,us) inserted; (1,eu) and (2,us) untouched"))))))))))diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex df4491ff7d9..bcadba83497 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -128,6 +128,16 @@ export type TransformSource = QueryTransformSource | PythonTransformSource;export type TransformTargetAppendStrategy = {type: "append";};++export type MergeKeyColumn = {+ name?: string;+ "field-id"?: number | null;+};++export type TransformTargetMergeStrategy = {+ type: "merge";+ "unique-key": MergeKeyColumn[];+};export type DraftTransformSource =| Transform["source"]| PythonTransformSourceDraft;@@ -136,7 +146,9 @@ export type DraftTransform = Partial<Pick<Transform, "id" | "name" | "description" | "target">> & { source: DraftTransformSource };-export type TargetIncrementalStrategy = TransformTargetAppendStrategy;+export type TargetIncrementalStrategy =+ | TransformTargetAppendStrategy+ | TransformTargetMergeStrategy;export type TransformTargetType = "table" | "table-incremental";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsxindex 010b13baa87..77a2058d135 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx@@ -7,10 +7,7 @@ import { FormSelect } from "metabase/forms";import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";import { useSelector } from "metabase/redux";import { getMetadata } from "metabase/selectors/metadata";-import {- SOURCE_STRATEGY_OPTIONS,- TARGET_STRATEGY_OPTIONS,-} from "metabase/transforms/constants";+import { SOURCE_STRATEGY_OPTIONS } from "metabase/transforms/constants";import { getLibQuery } from "metabase/transforms/utils";import {Anchor,@@ -23,7 +20,7 @@ import {Tooltip,} from "metabase/ui";import * as Lib from "metabase-lib";-import type { TransformSource } from "metabase-types/api";+import type { TableId, TransformSource } from "metabase-types/api";import type { TransformType } from "metabase-types/api/transform";import {@@ -31,6 +28,7 @@ import {NativeQueryTableTagFieldSelect,PythonKeysetColumnSelect,} from "./KeysetColumnSelect";+import { UniqueKeyField } from "./UniqueKeyField";import type { IncrementalSettingsFormValues } from "./form";import { useHasCheckpointOptions } from "./useHasCheckpointOptions";@@ -41,6 +39,8 @@ type IncrementalTransformSettingsProps = {variant?: "embedded" | "standalone";readOnly?: boolean;extraActions?: React.ReactNode;+ // When the target table already exists, its id powers a column picker for the unique key.+ targetTableId?: TableId;};export const IncrementalTransformSettings = ({@@ -50,6 +50,7 @@ export const IncrementalTransformSettings = ({variant = "embedded",readOnly,extraActions,+ targetTableId,}: IncrementalTransformSettingsProps) => {const metadata = useSelector(getMetadata);const libQuery = getLibQuery(source, metadata);@@ -165,7 +166,11 @@ export const IncrementalTransformSettings = ({<Group p="lg">{extraActions}</Group></>)}- <TargetStrategyFields variant={variant} />+ <TargetStrategyFields+ variant={variant}+ targetTableId={targetTableId}+ readOnly={readOnly}+ /></>)}</TitleSection>@@ -188,7 +193,11 @@ export const IncrementalTransformSettings = ({query={libQuery}transformType={transformType}/>- <TargetStrategyFields variant={variant} />+ <TargetStrategyFields+ variant={variant}+ targetTableId={targetTableId}+ readOnly={readOnly}+ /></>)}</Stack>@@ -197,19 +206,18 @@ export const IncrementalTransformSettings = ({function TargetStrategyFields({variant,+ targetTableId,+ readOnly,}: {variant: "embedded" | "standalone";+ targetTableId?: TableId;+ readOnly?: boolean;}) {- const content = TARGET_STRATEGY_OPTIONS.length > 1 && (+ // No explicit strategy picker: leaving the unique key empty appends; setting it upserts+ // (merge/restate) matching rows in place.+ const content = (<Stack>- <FormSelect- name="targetStrategy"- label={t`Target Strategy`}- description={t`How to update the target table`}- data={TARGET_STRATEGY_OPTIONS}- />- {/* Append strategy has no additional fields */}- {/* Future strategies like "merge" could add fields here */}+ <UniqueKeyField targetTableId={targetTableId} readOnly={readOnly} /></Stack>);@@ -218,12 +226,10 @@ function TargetStrategyFields({}return (- content && (- <>- <Divider />- <Group p="lg">{content}</Group>- </>- )+ <>+ <Divider />+ <Group p="lg">{content}</Group>+ </>);}@@ -260,7 +266,7 @@ function SourceStrategyFields({name="checkpointFilterFieldId"label={t`Field to check for new values`}placeholder={t`Pick a field`}- description={t`Pick the field that we should scan to determine which records are new or changed`}+ description={t`Pick the input field we should scan to determine which records are new or changed`}descriptionProps={{ lh: "1rem" }}query={query}source={source}@@ -272,7 +278,7 @@ function SourceStrategyFields({name="checkpointFilterFieldId"label={t`Field to check for new values`}placeholder={t`Pick a field`}- description={t`Pick the field that we should scan to determine which records are new or changed`}+ description={t`Pick the input field we should scan to determine which records are new or changed`}descriptionProps={{ lh: "1rem" }}query={query}disabled={readOnly}@@ -283,7 +289,7 @@ function SourceStrategyFields({name="checkpointFilterFieldId"label={t`Field to check for new values`}placeholder={t`Pick a field`}- description={t`Pick the field that we should scan to determine which records are new or changed`}+ description={t`Pick the input field we should scan to determine which records are new or changed`}descriptionProps={{ lh: "1rem" }}sourceTables={source["source-tables"]}disabled={readOnly}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsxindex aeb92dcb302..7d63635eac8 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx@@ -108,4 +108,14 @@ describe("IncrementalTransformSettings", () => {expect(getSwitch()).toBeEnabled();});++ it("shows the merge key field when incremental is enabled", () => {+ setup({ incremental: true });+ expect(screen.getByText("Merge key")).toBeInTheDocument();+ });++ it("does not show the merge key field when incremental is disabled", () => {+ setup({ incremental: false });+ expect(screen.queryByText("Merge key")).not.toBeInTheDocument();+ });});diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsxnew file mode 100644index 00000000000..33fd09532dc--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsx@@ -0,0 +1,88 @@+import { useField } from "formik";+import { t } from "ttag";++import { skipToken, useGetTableQueryMetadataQuery } from "metabase/api";+import { MultiSelect, TagsInput } from "metabase/ui";+import type { TableId } from "metabase-types/api";++// Form value is a comma-separated string of physical column names; this adapter presents it as a+// column multi-select. Labels use the column display name (matching the checkpoint field select),+// while the stored value stays the physical name the merge SQL needs.+function UniqueKeyColumnSelect({+ options,+ readOnly,+}: {+ options: { value: string; label: string }[];+ readOnly?: boolean;+}) {+ const [field, , helpers] = useField<string>("uniqueKey");+ const value = field.value+ ? field.value+ .split(",")+ .map((name) => name.trim())+ .filter((name) => name.length > 0)+ : [];++ return (+ <MultiSelect+ label={t`Merge key`}+ description={t`Optional. Output columns to upsert on.`}+ placeholder={t`Pick columns`}+ data={options}+ value={value}+ onChange={(names) => helpers.setValue(names.join(", "))}+ disabled={readOnly}+ searchable+ clearable+ />+ );+}++// Used before the target table exists, so we can't offer a column picker. Free-text entry that turns+// each comma- or Enter-separated name into a chip, like the column selector once the table exists.+function MergeKeyTagsInput({ readOnly }: { readOnly?: boolean }) {+ const [field, , helpers] = useField<string>("uniqueKey");+ const value = field.value+ ? field.value+ .split(",")+ .map((name) => name.trim())+ .filter((name) => name.length > 0)+ : [];++ return (+ <TagsInput+ label={t`Merge key`}+ description={t`Optional. Output columns to upsert on. Type a column name and press comma or enter.`}+ placeholder={t`e.g. id`}+ value={value}+ onChange={(names) => helpers.setValue(names.join(", "))}+ splitChars={[","]}+ disabled={readOnly}+ clearable+ />+ );+}++// The merge key input: a column picker once the target table exists, otherwise free-text chips.+export function UniqueKeyField({+ targetTableId,+ readOnly,+}: {+ targetTableId?: TableId;+ readOnly?: boolean;+}) {+ const { data: table } = useGetTableQueryMetadataQuery(+ targetTableId != null ? { id: targetTableId } : skipToken,+ );+ const options =+ table?.fields?.map((fieldObj) => ({+ value: fieldObj.name,+ label: fieldObj.display_name ?? fieldObj.name,+ })) ?? [];++ if (options.length > 0) {+ return <UniqueKeyColumnSelect options={options} readOnly={readOnly} />;+ }++ return <MergeKeyTagsInput readOnly={readOnly} />;+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.tsindex 2b7a9be0e55..0a9c68d969f 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts@@ -12,7 +12,9 @@ export type IncrementalSettingsFormValues = {incremental: boolean;sourceStrategy: "checkpoint";checkpointFilterFieldId: string | null; // String because Mantine Select requires string values- targetStrategy: "append";+ // Comma-separated target column names. Empty → append. Non-empty → the target is upserted+ // (merge/restate) on these columns instead of appended.+ uniqueKey: string;};export const VALIDATION_SCHEMA = Yup.object({@@ -27,7 +29,7 @@ export const VALIDATION_SCHEMA = Yup.object({then: (schema) => schema.required(Errors.required),otherwise: (schema) => schema.nullable().defined(),}),- targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),+ uniqueKey: Yup.string().default(""),});export const getInitialValues = (@@ -36,10 +38,17 @@ export const getInitialValues = (incremental: false,sourceStrategy: "checkpoint",checkpointFilterFieldId: null,- targetStrategy: "append",+ uniqueKey: "",...defaults,});+const parseUniqueKey = (uniqueKey: string) =>+ uniqueKey+ .split(",")+ .map((name) => name.trim())+ .filter((name) => name.length > 0)+ .map((name) => ({ name }));+export const getIncrementalSettingsFromTransform = (transform: Transform,): IncrementalSettingsFormValues => {@@ -51,6 +60,11 @@ export const getIncrementalSettingsFromTransform = (? (strategy["checkpoint-filter-field-id"] ?? null): null;+ const targetStrategy =+ transform.target.type === "table-incremental"+ ? transform.target["target-incremental-strategy"]+ : undefined;+return {incremental: isIncremental,sourceStrategy: "checkpoint",@@ -58,7 +72,13 @@ export const getIncrementalSettingsFromTransform = (isIncremental && checkpointFilterFieldId != null? String(checkpointFilterFieldId): null,- targetStrategy: "append",+ uniqueKey:+ targetStrategy?.type === "merge"+ ? targetStrategy["unique-key"]+ .map((column) => column.name)+ .filter((name): name is string => name != null)+ .join(", ")+ : "",};};@@ -114,11 +134,13 @@ export const buildIncrementalTarget = (if (!values.incremental) {return base;}+ const uniqueKey = parseUniqueKey(values.uniqueKey);return {...base,type: "table-incremental",- "target-incremental-strategy": {- type: values.targetStrategy,- },+ "target-incremental-strategy":+ uniqueKey.length > 0+ ? { type: "merge", "unique-key": uniqueKey }+ : { type: "append" },};};diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.tsindex 27d91e8eccf..5a1939a746c 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts@@ -1,4 +1,11 @@-import { type IncrementalSettingsFormValues, VALIDATION_SCHEMA } from "./form";+import { createMockTransform } from "metabase-types/api/mocks";++import {+ type IncrementalSettingsFormValues,+ VALIDATION_SCHEMA,+ buildIncrementalTarget,+ getIncrementalSettingsFromTransform,+} from "./form";const getValues = (values: Partial<IncrementalSettingsFormValues> = {},@@ -6,10 +13,12 @@ const getValues = (incremental: true,sourceStrategy: "checkpoint",checkpointFilterFieldId: "123",- targetStrategy: "append",+ uniqueKey: "",...values,});+const BASE_TARGET = { name: "orders_latest", schema: "public", database: 1 };+describe("IncrementalTransform form validation", () => {it("requires checkpoint field for incremental transforms", async () => {const values = getValues({ checkpointFilterFieldId: null });@@ -34,3 +43,85 @@ describe("IncrementalTransform form validation", () => {await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);});});++describe("buildIncrementalTarget (merge)", () => {+ it("produces an append target when no unique key is set", () => {+ const target = buildIncrementalTarget(+ BASE_TARGET,+ getValues({ uniqueKey: "" }),+ );+ expect(target).toMatchObject({+ type: "table-incremental",+ "target-incremental-strategy": { type: "append" },+ });+ });++ it("produces a merge target keyed on the unique key column", () => {+ const target = buildIncrementalTarget(+ BASE_TARGET,+ getValues({ uniqueKey: "order_id" }),+ );+ expect(target).toMatchObject({+ type: "table-incremental",+ "target-incremental-strategy": {+ type: "merge",+ "unique-key": [{ name: "order_id" }],+ },+ });+ });++ it("splits a comma-separated unique key and trims whitespace", () => {+ const target = buildIncrementalTarget(+ BASE_TARGET,+ getValues({ uniqueKey: " order_id , region " }),+ );+ expect(target).toMatchObject({+ "target-incremental-strategy": {+ type: "merge",+ "unique-key": [{ name: "order_id" }, { name: "region" }],+ },+ });+ });++ it("produces a plain table target when incremental is off", () => {+ const target = buildIncrementalTarget(+ BASE_TARGET,+ getValues({ incremental: false, uniqueKey: "order_id" }),+ );+ expect(target.type).toBe("table");+ });+});++describe("getIncrementalSettingsFromTransform (merge)", () => {+ it("reads the merge unique key into a comma-separated string", () => {+ const transform = createMockTransform({+ target: {+ type: "table-incremental",+ name: "t",+ schema: "public",+ database: 1,+ "target-incremental-strategy": {+ type: "merge",+ "unique-key": [{ name: "order_id" }, { name: "region" }],+ },+ },+ });+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({+ incremental: true,+ uniqueKey: "order_id, region",+ });+ });++ it("leaves the unique key empty for an append target", () => {+ const transform = createMockTransform({+ target: {+ type: "table-incremental",+ name: "t",+ schema: "public",+ database: 1,+ "target-incremental-strategy": { type: "append" },+ },+ });+ expect(getIncrementalSettingsFromTransform(transform).uniqueKey).toBe("");+ });+});diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.tsindex 0524111d9a9..31c4aa2a3fc 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts@@ -9,7 +9,7 @@ const baseValues = {incremental: false,sourceStrategy: "checkpoint" as const,checkpointFilterFieldId: null,- targetStrategy: "append" as const,+ uniqueKey: "",};describe("CreateTransformModal VALIDATION_SCHEMA (GDGT-2144)", () => {diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsxindex e8c1a02bc5d..bac4560ff0f 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx@@ -44,6 +44,7 @@ const IncrementalTransformSettingsWrapper = ({onIncrementalChange={handleIncrementalChange}variant="standalone"readOnly={readOnly}+ targetTableId={transform.table?.id}extraActions={!readOnly && transform.last_checkpoint_value != null ? (<ResetCheckpointSection transform={transform} />diff --git a/frontend/src/metabase/ui/components/inputs/TagsInput/index.ts b/frontend/src/metabase/ui/components/inputs/TagsInput/index.tsnew file mode 100644index 00000000000..842b483ea5b--- /dev/null+++ b/frontend/src/metabase/ui/components/inputs/TagsInput/index.ts@@ -0,0 +1,2 @@+export { TagsInput } from "@mantine/core";+export type { TagsInputProps } from "@mantine/core";diff --git a/frontend/src/metabase/ui/components/inputs/index.ts b/frontend/src/metabase/ui/components/inputs/index.tsindex 96e99f43959..a63284fa0b4 100644--- a/frontend/src/metabase/ui/components/inputs/index.ts+++ b/frontend/src/metabase/ui/components/inputs/index.ts@@ -21,6 +21,7 @@ export * from "./SegmentedControl";export * from "./Select";export * from "./Slider";export * from "./Switch";+export * from "./TagsInput";export * from "./Textarea";export * from "./TextareaBlurChange";export * from "./TextInput";diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex 35182cfe266..de73a343ef8 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -1114,6 +1114,15 @@^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))][(format "INSERT INTO %s %s" table-name sql-query) sql-params]))+(defmethod driver/run-transform! [:sqlserver :table-incremental]+ [driver transform-details opts]+ ;; SQL Server has no row-valued `IN (...)`, so the shared `:sql` merge can't express a composite-key+ ;; delete the default way. Ask it to use a correlated `EXISTS` instead by threading `:delete-strategy`+ ;; through `merge-opts`; only inject it when a merge is actually happening (leave append/create alone).+ (let [opts (cond-> opts+ (:merge opts) (assoc-in [:merge :delete-strategy] :exists))]+ ((get-method driver/run-transform! [:sql :table-incremental]) driver transform-details opts)))+(defmethod driver/table-exists? :sqlserver[driver database {:keys [schema name] :as _table}](sql-jdbc.execute/do-with-connection-with-optionsdiff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 3ef12bdc9c4..a17349b02e9 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -238,15 +238,63 @@:else(run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))+(defn- merge-delete-query+ "DELETE the rows of `target` whose `unique-key` matches a row in `temp`.++ `delete-strategy` selects how the match is expressed:+ - `:in` — uncorrelated `WHERE (k…) IN (SELECT k… FROM temp)`+ - `:exists` — correlated `WHERE EXISTS (SELECT 1 FROM temp WHERE temp.k = target.k …)"+ [driver target temp unique-key delete-strategy]+ (let [honeysql (case delete-strategy+ :exists (let [temp-col (fn [c] (keyword (name temp) c))+ target-col (fn [c] (keyword (name target) c))]+ {:delete-from target+ :where [:exists {:select [[[:inline 1]]]+ :from [temp]+ :where (into [:and]+ (map (fn [c] [:= (temp-col c) (target-col c)]))+ unique-key)}]})+ :in (let [cols (mapv keyword unique-key)+ lhs (if (= 1 (count cols)) (first cols) (into [:composite] cols))]+ {:delete-from target+ :where [:in lhs {:select cols, :from [temp]}]}))]+ (sql.qp/format-honeysql driver honeysql)))++(defn- merge-insert-query+ "INSERT every row of `temp` into `target`."+ [driver target temp]+ (sql.qp/format-honeysql driver+ {:insert-into [target {:select [:*] :from [temp]}]}))++(defn compile-merge+ "Returns `[sql params]` queries that upsert the rows of `select` (a compiled `{:query sql :params}`)+ into `target`. `merge-spec` is `{:unique-key [col-names] :columns [all-col-names] :delete-strategy k}`."+ [driver target select {:keys [unique-key delete-strategy] :or {delete-strategy :in}}]+ (let [temp (driver.u/temp-table-name driver target)]+ [(driver/compile-transform driver {:query select, :output-table temp})+ (merge-delete-query driver target temp unique-key delete-strategy)+ (merge-insert-query driver target temp)+ (driver/compile-drop-table driver temp)]))+(mu/defmethod driver/run-transform! [:sql :table-incremental] :- ::driver/run-transform-result- [driver {:keys [conn-spec database output-table] :as transform-details} _opts]- (let [queries (if (driver/table-exists? driver database {:schema (namespace output-table)- :name (name output-table)})- (driver/compile-insert driver transform-details)- (driver/compile-transform driver transform-details))]- (log/tracef "Executing incremental transform queries: %s" (pr-str queries))- ;; `execute-raw-queries!` already yields `{:rows-affected N}` maps; take the last as-is.- (last (driver/execute-raw-queries! driver conn-spec [queries]))))+ [driver {:keys [conn-spec database output-table query] :as transform-details} {merge-opts :merge}]+ (let [table-exists? (driver/table-exists? driver database {:schema (namespace output-table)+ :name (name output-table)})]+ (cond+ ;; First run — no target yet — create it.+ (not table-exists?)+ (last (driver/execute-raw-queries! driver conn-spec [(driver/compile-transform driver transform-details)]))++ ;; Key-based upsert+ merge-opts+ (let [results (driver/execute-raw-queries!+ driver conn-spec+ (compile-merge driver output-table query merge-opts))]+ {:rows-affected (reduce max 0 (keep :rows-affected results))})++ ;; Append.+ :else+ (last (driver/execute-raw-queries! driver conn-spec [(driver/compile-insert driver transform-details)])))))(defn qualified-name"Return the name of the target table of a transform as a possibly qualified symbol."diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 64e6c172653..b1db52505e2 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -33,8 +33,7 @@python-transform?query-transform?transform-source-database- transform-source-type- transform-type]+ transform-source-type][metabase.transforms.utiladd-source-readableis-temp-transform-table?]diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex a7cd2dc383c..1496843e6c9 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -66,16 +66,15 @@Resolves the field by ID and checks that its base-type is numeric or temporal.Throws a 400 error if the field cannot be found or its type is not supported."- [{:keys [source]}]- (when-let [{:keys [checkpoint-filter-field-id] strategy-type :type}- (:source-incremental-strategy source)]- (when (= "checkpoint" strategy-type)- (let [field (t2/select-one :model/Field checkpoint-filter-field-id)]- (api/check-400 field (deferred-tru "Checkpoint field not found."))- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."- (:name field)- (pr-str (:base_type field))))))))+ [{:keys [source] :as transform}]+ (when (transforms-base.u/checkpoint-source? transform)+ (let [checkpoint-filter-field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])+ field (t2/select-one :model/Field checkpoint-filter-field-id)]+ (api/check-400 field (deferred-tru "Checkpoint field not found."))+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."+ (:name field)+ (pr-str (:base_type field)))))))(defn validate-incremental-table-tag!"Reject a table-incremental native-query transform whose source query has no table template tag for@@ -84,8 +83,8 @@Without that table variable the incremental filter has nowhere to be injected, so the transform isin a broken state. This surfaces the error eagerly at create/update time instead of only when thetransform is run."- [{:keys [target] :as transform}]- (when (and (= :table-incremental (keyword (:type target)))+ [transform]+ (when (and (transforms-base.u/incremental-target? transform)(transforms-base.u/native-query-transform? transform))(api/check-400 (transforms-base.u/incremental-table-tag-name transform)(deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))))diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex e219359ddf9..0fd4827321b 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -146,6 +146,23 @@:target_db_id (when valid-db-id? target-db-id):source_database_id (or source_database_id (transforms-base.i/source-db-id transform))))))+(defn- resolve-merge-key-field-ids+ "Fill `:field-id` on a merge target's unique-key columns from the synced target table's fields, once+ the table exists and any are still by-name only."+ [transform]+ (let [target (or (:target transform) (:target (t2/original transform)))+ table-id (or (:target_table_id transform) (:target_table_id (t2/original transform)))+ unique-key (get-in target [:target-incremental-strategy :unique-key])]+ (if (and (transforms-base.u/merge-target? {:target target})+ table-id+ (some #(nil? (:field-id %)) unique-key))+ (let [name->id (t2/select-fn->fn :name :id [:model/Field :name :id] :table_id table-id :active true)]+ (assoc transform :target+ (assoc-in target [:target-incremental-strategy :unique-key]+ (mapv (fn [e] (cond-> e (:name e) (assoc :field-id (name->id (:name e)))))+ unique-key))))+ transform)))+(t2/define-before-update :model/Transform[{:keys [source source_database_id] :as transform}](when-let [new-collection (:collection_id (t2/changes transform))]@@ -174,7 +191,10 @@(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])](and old-field-id (not= old-field-id new-field-id)))- (assoc :last_checkpoint_value nil))))+ (assoc :last_checkpoint_value nil)++ true+ resolve-merge-key-field-ids)))(t2/define-after-select :model/Transform[{:keys [source] :as transform}]diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex 6c7089509e4..69749c90078 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -51,7 +51,7 @@(tracing/with-span :tasks "task.transform.query"{:transform/id id:transform/target-type (name target-type)- :transform/incremental (= :table-incremental target-type)+ :transform/incremental (transforms-base.u/incremental-target? transform):transform/full-incremental-run (transforms-base.u/full-incremental-run? transform):db/id (:id db):db/engine (name driver)}diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex f38427e8e42..51a45bf63af 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -35,9 +35,22 @@(mr/def ::append-config[:map [:type [:= "append"]]])+(mr/def ::merge-key-column+ "One column of a merge unique key. Carries a resolved `:field-id` when the target column is known,+ degrading to a `:name` ref when the target table doesn't exist yet (mirrors `::source-table-entry`)."+ [:map+ [:name {:optional true} ms/NonBlankString]+ [:field-id {:optional true} [:maybe ::lib.schema.id/field]]])++(mr/def ::merge-config+ [:map+ [:type [:= "merge"]]+ [:unique-key [:sequential ::merge-key-column]]])+(mr/def ::target-incremental-strategy[:multi {:dispatch :type}- ["append" ::append-config]])+ ["append" ::append-config]+ ["merge" ::merge-config]])(mr/def ::table-target[:mapdiff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.cljindex a6dea66bfe6..23abb773d37 100644--- a/src/metabase/transforms_base/query.clj+++ b/src/metabase/transforms_base/query.clj@@ -49,11 +49,16 @@;;; ------------------------------------------------- Helpers --------------------------------------------------(defn- transform-opts [{:keys [transform-type]}]+(defn- transform-opts [transform-type transform](case transform-type:table {:overwrite? true}- ;; once we have more than just append, dispatch on :target-incremental-strategy- :table-incremental {}))+ :table-incremental+ (if (transforms-base.u/merge-target? transform)+ (let [unique-key (transforms-base.u/merge-target-unique-key transform)+ columns (transforms-base.u/target-column-names transform)]+ (transforms-base.u/validate-merge-unique-key! unique-key columns)+ {:merge {:unique-key unique-key, :columns columns}})+ {})));;; ------------------------------------------------- Base Execution -------------------------------------------------@@ -124,7 +129,7 @@;; `metabase.driver.sql.query-processor/compile-transform :sql`.:output-db (:db target):output-table (transforms-base.u/qualified-table-name driver target)}- opts (transform-opts transform-details)+ opts (transform-opts effective-transform-type transform)features (transforms-base.u/required-database-features transform)](when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)(throw (ex-info "The database does not support the requested transform target type."diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 79c22cfa523..a68091a0f28 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -46,15 +46,15 @@;;; ------------------------------------------------- Transform Type Predicates --------------------------------------------------(defn transform-type- "Get the type of a transform"- [transform]- (-> transform :source :type keyword))+(defn- type-is?+ "True if `m`'s keywordized `:type` equals `t`."+ [m t]+ (= t (keyword (:type m))))(defn query-transform?"Check if this is a query transform: native query / mbql query."[transform]- (= :query (transform-type transform)))+ (type-is? (:source transform) :query))(defn native-query-transform?"Check if this is a native query transform.@@ -64,6 +64,45 @@(let [query (-> transform :source :query)](lib/native-only-query? query))))+(defn python-transform?+ "Check if this is a Python transform."+ [transform]+ (type-is? (:source transform) :python))++(defn incremental-target?+ "True if `transform` writes to an incremental table."+ [transform]+ (type-is? (:target transform) :table-incremental))++(defn merge-target?+ "True if `transform`'s target uses the merge strategy."+ [transform]+ (type-is? (get-in transform [:target :target-incremental-strategy]) :merge))++(defn checkpoint-source?+ "True if `transform`'s source uses the checkpoint strategy."+ [transform]+ (type-is? (get-in transform [:source :source-incremental-strategy]) :checkpoint))++(defn transform-source-database+ "Get the source database from a transform"+ [transform]+ (cond+ (query-transform? transform) (-> transform :source :query :database)+ (python-transform? transform) (-> transform :source :source-database)))++(defn full-incremental-run?+ "True when an incremental transform should drop-and-recreate the target rather than append.+ Fires whenever `last_checkpoint_value` is nil — covers the literal first run, and any run after the+ `:model/Transform` before-update hook clears the watermark on a `checkpoint-filter-field-id`+ change. Stays true across failed attempts since the predicate only inspects+ `:last_checkpoint_value`."+ [transform]+ (and (incremental-target? transform)+ (nil? (:last_checkpoint_value transform))))++;;; ------------------------------------------------- Table Template Tags -------------------------------------------------+(defn table-template-tag-name"Return the name (key) of the table template tag in `query` that refers to `table-id`, or nil.@@ -90,18 +129,6 @@table-id (when field-id (t2/select-one-fn :table_id :model/Field field-id))](table-template-tag-name (:query source) table-id)))-(defn python-transform?- "Check if this is a Python transform."- [transform]- (= :python (transform-type transform)))--(defn transform-source-database- "Get the source database from a transform"- [transform]- (case (transform-type transform)- :query (-> transform :source :query :database)- :python (-> transform :source :source-database)))-;;; ------------------------------------------------- Transform Normalization -------------------------------------------------(defn normalize-transform@@ -109,7 +136,7 @@This should be called on transforms before processing them to ensure queries are in the expected format."[transform](if (and (map? transform)- (= (:type transform) "transform")+ (type-is? transform :transform)(get-in transform [:source :query]))(update-in transform [:source :query] lib-be/normalize-query)transform))@@ -119,13 +146,13 @@Throws if the source type cannot be detected.Note: The transform should be normalized (via `normalize-transform`) before calling this function."[source]- (case (keyword (:type source))- :python :python- :query (if (lib/native-only-query? (:query source))- :native- :mbql)- (throw (ex-info (str "Unknown transform source type: " (:type source))- {:source source}))))+ (cond+ (type-is? source :python) :python+ (type-is? source :query) (if (lib/native-only-query? (:query source))+ :native+ :mbql)+ :else (throw (ex-info (str "Unknown transform source type: " (:type source))+ {:source source}))));;; ------------------------------------------------- Feature Checks -------------------------------------------------@@ -198,16 +225,6 @@(or (isa? base-type :type/Temporal)(isa? base-type :type/Number)))-(defn full-incremental-run?- "True when an incremental transform should drop-and-recreate the target rather than append.- Fires whenever `last_checkpoint_value` is nil — covers the literal first run, and any run after the- `:model/Transform` before-update hook clears the watermark on a `checkpoint-filter-field-id`- change. Stays true across failed attempts since the predicate only inspects- `:last_checkpoint_value`."- [{:keys [target last_checkpoint_value]}]- (and (= :table-incremental (keyword (:type target)))- (nil? last_checkpoint_value)))-(defn- encode-checkpoint-value [v](if (number? v)(str v)@@ -310,17 +327,17 @@:lo values in the source table must be > this :value.:hi values in the source table must be <= this :value.:rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."- [{:keys [source target] :as transform}]+ [{:keys [source] :as transform}](let [{:keys [source-incremental-strategy]} source{:keys [checkpoint-filter-field-id]} source-incremental-strategy]- (when (and (= "table-incremental" (:type target))+ (when (and (incremental-target? transform)(native-query-transform? transform)(not (some (fn [[_k v]] (#{:table "table"} (:type v)))(lib/template-tags (:query source)))))(let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. ""Please add a table variable to the query and update the checkpoint field."))](throw (ex-info msg {:transform-message msg}))))- (when (and (= "table-incremental" (:type target))+ (when (and (incremental-target? transform)(not checkpoint-filter-field-id))(let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. ""Please select a checkpoint field in the transform settings."))]@@ -614,6 +631,28 @@[transform](t2/select-one :model/Table :transform_id (:id transform)))+(defn merge-target-unique-key+ "Physical column names of a transform's `merge` unique key, or nil when the target isn't a merge target."+ [transform]+ (when (merge-target? transform)+ (not-empty (mapv :name (get-in transform [:target :target-incremental-strategy :unique-key])))))++(defn target-column-names+ "Physical column names of a transform's synced target table, ordered by position, or nil."+ [transform]+ (when-let [table (output-table transform)]+ (not-empty (t2/select-fn-vec :name [:model/Field :name :position]+ :table_id (:id table) :active true+ {:order-by [[:position :asc]]}))))++(defn validate-merge-unique-key!+ "Throws if any of `unique-key` is not present in `columns`. Returns `unique-key`."+ [unique-key columns]+ (when-let [missing (not-empty (remove (set columns) unique-key))]+ (let [msg (i18n/tru "Merge unique key references columns not present in the target: {0}"+ (str/join ", " missing))]+ (throw (ex-info msg {:transform-message msg}))))+ unique-key);;; ------------------------------------------------- Source Table Schemas -------------------------------------------------;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.cljindex 86be4dcf9c3..88ad39bcfa9 100644--- a/test/metabase/transforms_base/util_test.clj+++ b/test/metabase/transforms_base/util_test.clj@@ -9,6 +9,48 @@(set! *warn-on-reflection* true)+;;; ------------------------------------------------- Merge target helpers -------------------------------------------------++(deftest merge-target?-test+ (testing "true only for a table-incremental target with a merge strategy"+ (is (transforms-base.u/merge-target?+ {:target {:type "table-incremental"+ :target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}}))+ (is (not (transforms-base.u/merge-target?+ {:target {:type "table-incremental"+ :target-incremental-strategy {:type "append"}}})))+ (is (not (transforms-base.u/merge-target? {:target {:type "table"}})))))++(deftest merge-target-unique-key-test+ (testing "returns the physical column names of the merge key"+ (is (= ["id"]+ (transforms-base.u/merge-target-unique-key+ {:target {:target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}})))+ (is (= ["order_id" "region"]+ (transforms-base.u/merge-target-unique-key+ {:target {:target-incremental-strategy+ {:type "merge" :unique-key [{:name "order_id"} {:name "region"}]}}}))))+ (testing "nil when the target isn't a merge target"+ (is (nil? (transforms-base.u/merge-target-unique-key+ {:target {:target-incremental-strategy {:type "append"}}})))))++(deftest validate-merge-unique-key!-test+ (testing "returns the key when every column is present in the target columns"+ (is (= ["id"] (transforms-base.u/validate-merge-unique-key! ["id"] ["id" "status"])))+ (is (= ["a" "b"] (transforms-base.u/validate-merge-unique-key! ["a" "b"] ["a" "b" "c"]))))+ (testing "throws when a key column is not among the target columns"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"not present in the target"+ (transforms-base.u/validate-merge-unique-key! ["id" "ghost"] ["id" "status"])))+ (testing "the thrown error carries a user-facing :transform-message"+ (is (-> (try+ (transforms-base.u/validate-merge-unique-key! ["ghost"] ["id"])+ (catch clojure.lang.ExceptionInfo e+ (ex-data e)))+ :transform-message+ some?)))))+(deftest throw-if-db-routing-enabled!-oss-test(testing "on OSS (no :database-routing premium feature) the check is a no-op"(mt/with-premium-features #{}