Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Incremental transforms merge (#76663)

Nicola Mometto03b95225511e
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index 03fa2bbac71..cc1a0045274 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -83,12 +83,12 @@
(defmethod calculate-deps* :transform
[_ {{:keys [query]} :source :as transform}]
- (let [source-type (transforms-base.u/transform-type transform)]
- (case source-type
- :query (upstream-deps:query query)
- :python (upstream-deps:python-transform transform)
- (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
- {}))))
+ (cond
+ (transforms-base.u/query-transform? transform) (upstream-deps:query query)
+ (transforms-base.u/python-transform? transform) (upstream-deps:python-transform transform)
+ :else (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'"
+ (:id transform) (-> transform :source :type))
+ {})))
(defmethod calculate-deps* :snippet
[_ {:keys [template_tags] :as _snippet}]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index ef47624f43f..31cd603c628 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -12,6 +12,8 @@
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase.driver :as driver]
+ [metabase.driver.sql :as driver.sql]
+ [metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.schema :as transforms-base.schema]
@@ -160,6 +162,23 @@
(log/error e "Failed to transfer data using drop-create fallback strategy")
(throw e))))
+(defn- upsert-with-merge-strategy!
+ "Upsert the staged data into `table-name`, keyed on `unique-key` (a vector of column names)."
+ [driver db table-name metadata data-source unique-key]
+ (let [db-id (:id db)
+ staging (transforms-base.u/temp-table-name driver (namespace table-name))]
+ (transforms-base.u/validate-merge-unique-key! unique-key (mapv :name (:fields metadata)))
+ (create-table-and-insert-data! driver db-id (table-schema staging metadata) data-source)
+ (try
+ (let [[sql & params] (sql.qp/format-honeysql driver {:select [:*], :from [staging]})
+ select {:query sql, :params (vec params)}
+ merge-spec {:unique-key unique-key
+ :columns (mapv :name (:fields metadata))}]
+ (driver/execute-raw-queries! driver (driver/connection-spec driver db)
+ (driver.sql/compile-merge driver table-name select merge-spec)))
+ (finally
+ (transforms-base.u/drop-table! driver db-id staging)))))
+
(defmulti ^:private transfer-file-to-db
{:arglists '([driver db transform metadata temp-file])}
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
@@ -172,15 +191,22 @@
;; like non-incremental and drop-and-recreate rather than appending.
(if (transforms-base.u/full-incremental-run? transform)
((get-method transfer-file-to-db :table) driver db transform metadata temp-file)
- ;; Normal incremental: append if table exists, create if it doesn't
+ ;; Normal incremental: append (or merge) if table exists, create if it doesn't
(let [table-name (transforms-base.u/qualified-table-name driver target)
table-exists? (transforms-base.u/target-table-exists? transform)
data-source {:type :jsonl-file
- :file temp-file}]
- (if (not table-exists?)
+ :file temp-file}
+ unique-key (transforms-base.u/merge-target-unique-key transform)]
+ (cond
+ (not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+
+ unique-key
+ (upsert-with-merge-strategy! driver db table-name metadata data-source unique-key)
+
+ :else
(insert-data! driver db-id (table-schema table-name metadata) data-source)))))
(defmethod transfer-file-to-db :table
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 76fccf49dc3..8315836fff5 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -113,7 +113,7 @@
(tracing/with-span :tasks "task.transform.python"
{:transform/id transform-id
:transform/target-type (name target-type)
- :transform/incremental (= :table-incremental target-type)
+ :transform/incremental (transforms-base.u/incremental-target? transform)
:transform/full-incremental-run (transforms-base.u/full-incremental-run? transform)
:db/id (:id db)
:db/engine (name driver)}
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
new file mode 100644
index 00000000000..06ff70b776c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
@@ -0,0 +1,209 @@
+(ns ^:mb/driver-tests metabase-enterprise.transforms.merge-test
+ "Tests for the key-based `merge` incremental transform target strategy (upsert/restate)."
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer :all]
+ [metabase.driver :as driver]
+ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
+ [metabase.driver.sql.util :as sql.u]
+ [metabase.query-processor :as qp]
+ [metabase.test :as mt]
+ [metabase.transforms.execute :as transforms.execute]
+ [metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;; An append-only status log: `order_id` repeats across rows, `change_seq` is the checkpoint column.
+(mt/defdataset merge-test
+ "A tiny event-log dataset for testing the merge strategy. `order_id` is the merge key and repeats
+ across rows; `change_seq` is the incremental checkpoint column."
+ [["order_status"
+ [{:field-name "order_id" :base-type :type/Integer}
+ {:field-name "status" :base-type :type/Text}
+ {:field-name "change_seq" :base-type :type/Integer}]
+ [[1 "created" 1]
+ [2 "created" 2]
+ [3 "created" 3]]]])
+
+;; Two key columns (order_id, region) for testing a composite merge key.
+(mt/defdataset merge-composite-test
+ "An event log keyed on (order_id, region), for testing a two-column merge key."
+ [["order_region_status"
+ [{:field-name "order_id" :base-type :type/Integer}
+ {:field-name "region" :base-type :type/Text}
+ {:field-name "status" :base-type :type/Text}
+ {:field-name "change_seq" :base-type :type/Integer}]
+ [[1 "us" "created" 1]
+ [1 "eu" "created" 2]
+ [2 "us" "created" 3]]]])
+
+(defn- test-drivers []
+ (mt/normal-drivers-with-feature :transforms/table))
+
+(defn- target-table-gen [prefix]
+ {:type :table
+ :name prefix
+ :schema (t2/select-one-fn :schema :model/Table (mt/id :order_status))
+ :database (mt/id)})
+
+(defn- merge-source-query []
+ {:database (mt/id)
+ :type :native
+ :native {:query (format "SELECT %s, %s FROM {{order_status}} AS %s"
+ (sql.u/quote-name driver/*driver* :field "order_id")
+ (sql.u/quote-name driver/*driver* :field "status")
+ (sql.u/quote-name driver/*driver* :field "order_status"))
+ :template-tags {"order_status" {:id "order_status"
+ :name "order_status"
+ :display-name "Order Status"
+ :type "table"
+ :table-id (mt/id :order_status)
+ :required true}}}})
+
+(defn- merge-transform-payload [target-table & {:keys [validate?] :or {validate? true}}]
+ (let [seq-field-id (t2/select-one-pk :model/Field :name "change_seq" :table_id (mt/id :order_status))]
+ {:name "Merge Transform"
+ :source_database_id (mt/id)
+ :source {:type "query"
+ :query (merge-source-query)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id seq-field-id}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy (cond-> {:type "merge"
+ :unique-key [{:name "order_id"}]}
+ (not validate?)
+ (assoc :validate-unique-key? false))})}))
+
+(defn- insert-order-status!
+ "Append `rows` (vectors of `[order-id status change-seq]`) to the source table."
+ [rows]
+ (let [[schema table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :order_status))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ values (str/join ", " (map (fn [[oid st sq]] (format "(%d, '%s', %d)" oid st sq)) rows))
+ sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema table-name)
+ (str/join "," (map #(sql.u/quote-name driver/*driver* :field %)
+ ["order_id" "status" "change_seq"]))
+ values)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+
+(defn- target-status-by-order
+ "Return the target table contents as a map of `order_id` -> `status`."
+ [{:keys [schema name]}]
+ (let [sql (format "SELECT %s, %s FROM %s"
+ (sql.u/quote-name driver/*driver* :field "order_id")
+ (sql.u/quote-name driver/*driver* :field "status")
+ (sql.u/quote-name driver/*driver* :table schema name))
+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})
+ :data :rows)]
+ (into {} (map (fn [[oid st]] [(long oid) st])) rows)))
+
+(deftest merge-builds-then-upserts-test
+ (testing "a merge transform builds the full target, then upserts by key on subsequent runs"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-test
+ (with-transform-cleanup! [target-table (target-table-gen "merge_upsert")]
+ (mt/with-temp [:model/Transform transform (merge-transform-payload target-table)]
+ (testing "first run builds the full target"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (is (= {1 "created", 2 "created", 3 "created"}
+ (target-status-by-order target-table))))
+ (testing "new events update existing keys in place and insert new ones"
+ (insert-order-status! [[1 "shipped" 4] [4 "created" 5]])
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {1 "shipped", 2 "created", 3 "created", 4 "created"}
+ (target-status-by-order target-table))
+ "order 1 updated (paid->shipped), order 4 inserted")))))))))
+
+(deftest merge-resolves-field-ids-test
+ (testing "after the first build the stored unique key gets its field-ids resolved"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-test
+ (with-transform-cleanup! [target-table (target-table-gen "merge_fieldids")]
+ (mt/with-temp [:model/Transform transform (merge-transform-payload target-table)]
+ (is (nil? (-> transform :target :target-incremental-strategy :unique-key first :field-id))
+ "no field-id before the first run — the target table doesn't exist yet")
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (let [reloaded (t2/select-one :model/Transform (:id transform))
+ unique-key (-> reloaded :target :target-incremental-strategy :unique-key)
+ target-tbl-id (:target_table_id reloaded)
+ expected (t2/select-one-pk :model/Field :name "order_id"
+ :table_id target-tbl-id :active true)]
+ (is (= ["order_id"] (mapv :name unique-key)))
+ (is (= expected (:field-id (first unique-key)))
+ "field-id resolved to the target's order_id field")))))))))
+
+(deftest merge-composite-key-test
+ (testing "a merge transform can upsert on a two-column key"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-composite-test
+ (with-transform-cleanup! [target-table {:type :table
+ :name "merge_composite"
+ :schema (t2/select-one-fn :schema :model/Table
+ (mt/id :order_region_status))
+ :database (mt/id)}]
+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
+ seq-field-id (t2/select-one-pk :model/Field :name "change_seq"
+ :table_id (mt/id :order_region_status))
+ source-query {:database (mt/id)
+ :type :native
+ :native {:query (format "SELECT %s, %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "region") (q "status") (q "t"))
+ :template-tags {"t" {:id "t"
+ :name "t"
+ :display-name "T"
+ :type "table"
+ :table-id (mt/id :order_region_status)
+ :required true}}}}
+ payload {:name "Composite Merge"
+ :source_database_id (mt/id)
+ :source {:type "query"
+ :query source-query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id seq-field-id}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "merge"
+ :unique-key [{:name "order_id"}
+ {:name "region"}]}})}
+ insert! (fn [rows]
+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table
+ (mt/id :order_region_status))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ values (str/join ", " (map (fn [[oid rg st sq]]
+ (format "(%d, '%s', '%s', %d)" oid rg st sq))
+ rows))
+ sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema tbl)
+ (str/join "," (map q ["order_id" "region" "status" "change_seq"]))
+ values)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+ read-target (fn []
+ (let [sql (format "SELECT %s, %s, %s FROM %s"
+ (q "order_id") (q "region") (q "status")
+ (sql.u/quote-name driver/*driver* :table
+ (:schema target-table) (:name target-table)))
+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})
+ :data :rows)]
+ (into {} (map (fn [[oid rg st]] [[(long oid) rg] st])) rows)))]
+ (mt/with-temp [:model/Transform transform payload]
+ (testing "first run builds the full target keyed on (order_id, region)"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (is (= {[1 "us"] "created", [1 "eu"] "created", [2 "us"] "created"}
+ (read-target))))
+ (testing "new events upsert one composite key and insert another"
+ (insert! [[1 "us" "shipped" 4] [3 "us" "created" 5]])
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {[1 "us"] "shipped", [1 "eu"] "created", [2 "us"] "created", [3 "us"] "created"}
+ (read-target))
+ "(1,us) updated, (3,us) inserted; (1,eu) and (2,us) untouched"))))))))))
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index df4491ff7d9..bcadba83497 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -128,6 +128,16 @@ export type TransformSource = QueryTransformSource | PythonTransformSource;
export type TransformTargetAppendStrategy = {
type: "append";
};
+
+export type MergeKeyColumn = {
+ name?: string;
+ "field-id"?: number | null;
+};
+
+export type TransformTargetMergeStrategy = {
+ type: "merge";
+ "unique-key": MergeKeyColumn[];
+};
export type DraftTransformSource =
| Transform["source"]
| PythonTransformSourceDraft;
@@ -136,7 +146,9 @@ export type DraftTransform = Partial<
Pick<Transform, "id" | "name" | "description" | "target">
> & { source: DraftTransformSource };
-export type TargetIncrementalStrategy = TransformTargetAppendStrategy;
+export type TargetIncrementalStrategy =
+ | TransformTargetAppendStrategy
+ | TransformTargetMergeStrategy;
export type TransformTargetType = "table" | "table-incremental";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index 010b13baa87..77a2058d135 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -7,10 +7,7 @@ import { FormSelect } from "metabase/forms";
import { PLUGIN_REMOTE_SYNC } from "metabase/plugins";
import { useSelector } from "metabase/redux";
import { getMetadata } from "metabase/selectors/metadata";
-import {
- SOURCE_STRATEGY_OPTIONS,
- TARGET_STRATEGY_OPTIONS,
-} from "metabase/transforms/constants";
+import { SOURCE_STRATEGY_OPTIONS } from "metabase/transforms/constants";
import { getLibQuery } from "metabase/transforms/utils";
import {
Anchor,
@@ -23,7 +20,7 @@ import {
Tooltip,
} from "metabase/ui";
import * as Lib from "metabase-lib";
-import type { TransformSource } from "metabase-types/api";
+import type { TableId, TransformSource } from "metabase-types/api";
import type { TransformType } from "metabase-types/api/transform";
import {
@@ -31,6 +28,7 @@ import {
NativeQueryTableTagFieldSelect,
PythonKeysetColumnSelect,
} from "./KeysetColumnSelect";
+import { UniqueKeyField } from "./UniqueKeyField";
import type { IncrementalSettingsFormValues } from "./form";
import { useHasCheckpointOptions } from "./useHasCheckpointOptions";
@@ -41,6 +39,8 @@ type IncrementalTransformSettingsProps = {
variant?: "embedded" | "standalone";
readOnly?: boolean;
extraActions?: React.ReactNode;
+ // When the target table already exists, its id powers a column picker for the unique key.
+ targetTableId?: TableId;
};
export const IncrementalTransformSettings = ({
@@ -50,6 +50,7 @@ export const IncrementalTransformSettings = ({
variant = "embedded",
readOnly,
extraActions,
+ targetTableId,
}: IncrementalTransformSettingsProps) => {
const metadata = useSelector(getMetadata);
const libQuery = getLibQuery(source, metadata);
@@ -165,7 +166,11 @@ export const IncrementalTransformSettings = ({
<Group p="lg">{extraActions}</Group>
</>
)}
- <TargetStrategyFields variant={variant} />
+ <TargetStrategyFields
+ variant={variant}
+ targetTableId={targetTableId}
+ readOnly={readOnly}
+ />
</>
)}
</TitleSection>
@@ -188,7 +193,11 @@ export const IncrementalTransformSettings = ({
query={libQuery}
transformType={transformType}
/>
- <TargetStrategyFields variant={variant} />
+ <TargetStrategyFields
+ variant={variant}
+ targetTableId={targetTableId}
+ readOnly={readOnly}
+ />
</>
)}
</Stack>
@@ -197,19 +206,18 @@ export const IncrementalTransformSettings = ({
function TargetStrategyFields({
variant,
+ targetTableId,
+ readOnly,
}: {
variant: "embedded" | "standalone";
+ targetTableId?: TableId;
+ readOnly?: boolean;
}) {
- const content = TARGET_STRATEGY_OPTIONS.length > 1 && (
+ // No explicit strategy picker: leaving the unique key empty appends; setting it upserts
+ // (merge/restate) matching rows in place.
+ const content = (
<Stack>
- <FormSelect
- name="targetStrategy"
- label={t`Target Strategy`}
- description={t`How to update the target table`}
- data={TARGET_STRATEGY_OPTIONS}
- />
- {/* Append strategy has no additional fields */}
- {/* Future strategies like "merge" could add fields here */}
+ <UniqueKeyField targetTableId={targetTableId} readOnly={readOnly} />
</Stack>
);
@@ -218,12 +226,10 @@ function TargetStrategyFields({
}
return (
- content && (
- <>
- <Divider />
- <Group p="lg">{content}</Group>
- </>
- )
+ <>
+ <Divider />
+ <Group p="lg">{content}</Group>
+ </>
);
}
@@ -260,7 +266,7 @@ function SourceStrategyFields({
name="checkpointFilterFieldId"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
- description={t`Pick the field that we should scan to determine which records are new or changed`}
+ description={t`Pick the input field we should scan to determine which records are new or changed`}
descriptionProps={{ lh: "1rem" }}
query={query}
source={source}
@@ -272,7 +278,7 @@ function SourceStrategyFields({
name="checkpointFilterFieldId"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
- description={t`Pick the field that we should scan to determine which records are new or changed`}
+ description={t`Pick the input field we should scan to determine which records are new or changed`}
descriptionProps={{ lh: "1rem" }}
query={query}
disabled={readOnly}
@@ -283,7 +289,7 @@ function SourceStrategyFields({
name="checkpointFilterFieldId"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
- description={t`Pick the field that we should scan to determine which records are new or changed`}
+ description={t`Pick the input field we should scan to determine which records are new or changed`}
descriptionProps={{ lh: "1rem" }}
sourceTables={source["source-tables"]}
disabled={readOnly}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx
index aeb92dcb302..7d63635eac8 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx
@@ -108,4 +108,14 @@ describe("IncrementalTransformSettings", () => {
expect(getSwitch()).toBeEnabled();
});
+
+ it("shows the merge key field when incremental is enabled", () => {
+ setup({ incremental: true });
+ expect(screen.getByText("Merge key")).toBeInTheDocument();
+ });
+
+ it("does not show the merge key field when incremental is disabled", () => {
+ setup({ incremental: false });
+ expect(screen.queryByText("Merge key")).not.toBeInTheDocument();
+ });
});
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsx
new file mode 100644
index 00000000000..33fd09532dc
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/UniqueKeyField.tsx
@@ -0,0 +1,88 @@
+import { useField } from "formik";
+import { t } from "ttag";
+
+import { skipToken, useGetTableQueryMetadataQuery } from "metabase/api";
+import { MultiSelect, TagsInput } from "metabase/ui";
+import type { TableId } from "metabase-types/api";
+
+// Form value is a comma-separated string of physical column names; this adapter presents it as a
+// column multi-select. Labels use the column display name (matching the checkpoint field select),
+// while the stored value stays the physical name the merge SQL needs.
+function UniqueKeyColumnSelect({
+ options,
+ readOnly,
+}: {
+ options: { value: string; label: string }[];
+ readOnly?: boolean;
+}) {
+ const [field, , helpers] = useField<string>("uniqueKey");
+ const value = field.value
+ ? field.value
+ .split(",")
+ .map((name) => name.trim())
+ .filter((name) => name.length > 0)
+ : [];
+
+ return (
+ <MultiSelect
+ label={t`Merge key`}
+ description={t`Optional. Output columns to upsert on.`}
+ placeholder={t`Pick columns`}
+ data={options}
+ value={value}
+ onChange={(names) => helpers.setValue(names.join(", "))}
+ disabled={readOnly}
+ searchable
+ clearable
+ />
+ );
+}
+
+// Used before the target table exists, so we can't offer a column picker. Free-text entry that turns
+// each comma- or Enter-separated name into a chip, like the column selector once the table exists.
+function MergeKeyTagsInput({ readOnly }: { readOnly?: boolean }) {
+ const [field, , helpers] = useField<string>("uniqueKey");
+ const value = field.value
+ ? field.value
+ .split(",")
+ .map((name) => name.trim())
+ .filter((name) => name.length > 0)
+ : [];
+
+ return (
+ <TagsInput
+ label={t`Merge key`}
+ description={t`Optional. Output columns to upsert on. Type a column name and press comma or enter.`}
+ placeholder={t`e.g. id`}
+ value={value}
+ onChange={(names) => helpers.setValue(names.join(", "))}
+ splitChars={[","]}
+ disabled={readOnly}
+ clearable
+ />
+ );
+}
+
+// The merge key input: a column picker once the target table exists, otherwise free-text chips.
+export function UniqueKeyField({
+ targetTableId,
+ readOnly,
+}: {
+ targetTableId?: TableId;
+ readOnly?: boolean;
+}) {
+ const { data: table } = useGetTableQueryMetadataQuery(
+ targetTableId != null ? { id: targetTableId } : skipToken,
+ );
+ const options =
+ table?.fields?.map((fieldObj) => ({
+ value: fieldObj.name,
+ label: fieldObj.display_name ?? fieldObj.name,
+ })) ?? [];
+
+ if (options.length > 0) {
+ return <UniqueKeyColumnSelect options={options} readOnly={readOnly} />;
+ }
+
+ return <MergeKeyTagsInput readOnly={readOnly} />;
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
index 2b7a9be0e55..0a9c68d969f 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
@@ -12,7 +12,9 @@ export type IncrementalSettingsFormValues = {
incremental: boolean;
sourceStrategy: "checkpoint";
checkpointFilterFieldId: string | null; // String because Mantine Select requires string values
- targetStrategy: "append";
+ // Comma-separated target column names. Empty → append. Non-empty → the target is upserted
+ // (merge/restate) on these columns instead of appended.
+ uniqueKey: string;
};
export const VALIDATION_SCHEMA = Yup.object({
@@ -27,7 +29,7 @@ export const VALIDATION_SCHEMA = Yup.object({
then: (schema) => schema.required(Errors.required),
otherwise: (schema) => schema.nullable().defined(),
}),
- targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
+ uniqueKey: Yup.string().default(""),
});
export const getInitialValues = (
@@ -36,10 +38,17 @@ export const getInitialValues = (
incremental: false,
sourceStrategy: "checkpoint",
checkpointFilterFieldId: null,
- targetStrategy: "append",
+ uniqueKey: "",
...defaults,
});
+const parseUniqueKey = (uniqueKey: string) =>
+ uniqueKey
+ .split(",")
+ .map((name) => name.trim())
+ .filter((name) => name.length > 0)
+ .map((name) => ({ name }));
+
export const getIncrementalSettingsFromTransform = (
transform: Transform,
): IncrementalSettingsFormValues => {
@@ -51,6 +60,11 @@ export const getIncrementalSettingsFromTransform = (
? (strategy["checkpoint-filter-field-id"] ?? null)
: null;
+ const targetStrategy =
+ transform.target.type === "table-incremental"
+ ? transform.target["target-incremental-strategy"]
+ : undefined;
+
return {
incremental: isIncremental,
sourceStrategy: "checkpoint",
@@ -58,7 +72,13 @@ export const getIncrementalSettingsFromTransform = (
isIncremental && checkpointFilterFieldId != null
? String(checkpointFilterFieldId)
: null,
- targetStrategy: "append",
+ uniqueKey:
+ targetStrategy?.type === "merge"
+ ? targetStrategy["unique-key"]
+ .map((column) => column.name)
+ .filter((name): name is string => name != null)
+ .join(", ")
+ : "",
};
};
@@ -114,11 +134,13 @@ export const buildIncrementalTarget = (
if (!values.incremental) {
return base;
}
+ const uniqueKey = parseUniqueKey(values.uniqueKey);
return {
...base,
type: "table-incremental",
- "target-incremental-strategy": {
- type: values.targetStrategy,
- },
+ "target-incremental-strategy":
+ uniqueKey.length > 0
+ ? { type: "merge", "unique-key": uniqueKey }
+ : { type: "append" },
};
};
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
index 27d91e8eccf..5a1939a746c 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
@@ -1,4 +1,11 @@
-import { type IncrementalSettingsFormValues, VALIDATION_SCHEMA } from "./form";
+import { createMockTransform } from "metabase-types/api/mocks";
+
+import {
+ type IncrementalSettingsFormValues,
+ VALIDATION_SCHEMA,
+ buildIncrementalTarget,
+ getIncrementalSettingsFromTransform,
+} from "./form";
const getValues = (
values: Partial<IncrementalSettingsFormValues> = {},
@@ -6,10 +13,12 @@ const getValues = (
incremental: true,
sourceStrategy: "checkpoint",
checkpointFilterFieldId: "123",
- targetStrategy: "append",
+ uniqueKey: "",
...values,
});
+const BASE_TARGET = { name: "orders_latest", schema: "public", database: 1 };
+
describe("IncrementalTransform form validation", () => {
it("requires checkpoint field for incremental transforms", async () => {
const values = getValues({ checkpointFilterFieldId: null });
@@ -34,3 +43,85 @@ describe("IncrementalTransform form validation", () => {
await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);
});
});
+
+describe("buildIncrementalTarget (merge)", () => {
+ it("produces an append target when no unique key is set", () => {
+ const target = buildIncrementalTarget(
+ BASE_TARGET,
+ getValues({ uniqueKey: "" }),
+ );
+ expect(target).toMatchObject({
+ type: "table-incremental",
+ "target-incremental-strategy": { type: "append" },
+ });
+ });
+
+ it("produces a merge target keyed on the unique key column", () => {
+ const target = buildIncrementalTarget(
+ BASE_TARGET,
+ getValues({ uniqueKey: "order_id" }),
+ );
+ expect(target).toMatchObject({
+ type: "table-incremental",
+ "target-incremental-strategy": {
+ type: "merge",
+ "unique-key": [{ name: "order_id" }],
+ },
+ });
+ });
+
+ it("splits a comma-separated unique key and trims whitespace", () => {
+ const target = buildIncrementalTarget(
+ BASE_TARGET,
+ getValues({ uniqueKey: " order_id , region " }),
+ );
+ expect(target).toMatchObject({
+ "target-incremental-strategy": {
+ type: "merge",
+ "unique-key": [{ name: "order_id" }, { name: "region" }],
+ },
+ });
+ });
+
+ it("produces a plain table target when incremental is off", () => {
+ const target = buildIncrementalTarget(
+ BASE_TARGET,
+ getValues({ incremental: false, uniqueKey: "order_id" }),
+ );
+ expect(target.type).toBe("table");
+ });
+});
+
+describe("getIncrementalSettingsFromTransform (merge)", () => {
+ it("reads the merge unique key into a comma-separated string", () => {
+ const transform = createMockTransform({
+ target: {
+ type: "table-incremental",
+ name: "t",
+ schema: "public",
+ database: 1,
+ "target-incremental-strategy": {
+ type: "merge",
+ "unique-key": [{ name: "order_id" }, { name: "region" }],
+ },
+ },
+ });
+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({
+ incremental: true,
+ uniqueKey: "order_id, region",
+ });
+ });
+
+ it("leaves the unique key empty for an append target", () => {
+ const transform = createMockTransform({
+ target: {
+ type: "table-incremental",
+ name: "t",
+ schema: "public",
+ database: 1,
+ "target-incremental-strategy": { type: "append" },
+ },
+ });
+ expect(getIncrementalSettingsFromTransform(transform).uniqueKey).toBe("");
+ });
+});
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts
index 0524111d9a9..31c4aa2a3fc 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/form.unit.spec.ts
@@ -9,7 +9,7 @@ const baseValues = {
incremental: false,
sourceStrategy: "checkpoint" as const,
checkpointFilterFieldId: null,
- targetStrategy: "append" as const,
+ uniqueKey: "",
};
describe("CreateTransformModal VALIDATION_SCHEMA (GDGT-2144)", () => {
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
index e8c1a02bc5d..bac4560ff0f 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
@@ -44,6 +44,7 @@ const IncrementalTransformSettingsWrapper = ({
onIncrementalChange={handleIncrementalChange}
variant="standalone"
readOnly={readOnly}
+ targetTableId={transform.table?.id}
extraActions={
!readOnly && transform.last_checkpoint_value != null ? (
<ResetCheckpointSection transform={transform} />
diff --git a/frontend/src/metabase/ui/components/inputs/TagsInput/index.ts b/frontend/src/metabase/ui/components/inputs/TagsInput/index.ts
new file mode 100644
index 00000000000..842b483ea5b
--- /dev/null
+++ b/frontend/src/metabase/ui/components/inputs/TagsInput/index.ts
@@ -0,0 +1,2 @@
+export { TagsInput } from "@mantine/core";
+export type { TagsInputProps } from "@mantine/core";
diff --git a/frontend/src/metabase/ui/components/inputs/index.ts b/frontend/src/metabase/ui/components/inputs/index.ts
index 96e99f43959..a63284fa0b4 100644
--- a/frontend/src/metabase/ui/components/inputs/index.ts
+++ b/frontend/src/metabase/ui/components/inputs/index.ts
@@ -21,6 +21,7 @@ export * from "./SegmentedControl";
export * from "./Select";
export * from "./Slider";
export * from "./Switch";
+export * from "./TagsInput";
export * from "./Textarea";
export * from "./TextareaBlurChange";
export * from "./TextInput";
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 35182cfe266..de73a343ef8 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -1114,6 +1114,15 @@
^String table-name (first (sql.qp/format-honeysql driver (keyword output-table)))]
[(format "INSERT INTO %s %s" table-name sql-query) sql-params]))
+(defmethod driver/run-transform! [:sqlserver :table-incremental]
+ [driver transform-details opts]
+ ;; SQL Server has no row-valued `IN (...)`, so the shared `:sql` merge can't express a composite-key
+ ;; delete the default way. Ask it to use a correlated `EXISTS` instead by threading `:delete-strategy`
+ ;; through `merge-opts`; only inject it when a merge is actually happening (leave append/create alone).
+ (let [opts (cond-> opts
+ (:merge opts) (assoc-in [:merge :delete-strategy] :exists))]
+ ((get-method driver/run-transform! [:sql :table-incremental]) driver transform-details opts)))
+
(defmethod driver/table-exists? :sqlserver
[driver database {:keys [schema name] :as _table}]
(sql-jdbc.execute/do-with-connection-with-options
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 3ef12bdc9c4..a17349b02e9 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -238,15 +238,63 @@
:else
(run-with-drop-create-fallback-strategy! driver database output-table transform-details conn-spec))))
+(defn- merge-delete-query
+ "DELETE the rows of `target` whose `unique-key` matches a row in `temp`.
+
+ `delete-strategy` selects how the match is expressed:
+ - `:in` — uncorrelated `WHERE (k…) IN (SELECT k… FROM temp)`
+ - `:exists` — correlated `WHERE EXISTS (SELECT 1 FROM temp WHERE temp.k = target.k …)"
+ [driver target temp unique-key delete-strategy]
+ (let [honeysql (case delete-strategy
+ :exists (let [temp-col (fn [c] (keyword (name temp) c))
+ target-col (fn [c] (keyword (name target) c))]
+ {:delete-from target
+ :where [:exists {:select [[[:inline 1]]]
+ :from [temp]
+ :where (into [:and]
+ (map (fn [c] [:= (temp-col c) (target-col c)]))
+ unique-key)}]})
+ :in (let [cols (mapv keyword unique-key)
+ lhs (if (= 1 (count cols)) (first cols) (into [:composite] cols))]
+ {:delete-from target
+ :where [:in lhs {:select cols, :from [temp]}]}))]
+ (sql.qp/format-honeysql driver honeysql)))
+
+(defn- merge-insert-query
+ "INSERT every row of `temp` into `target`."
+ [driver target temp]
+ (sql.qp/format-honeysql driver
+ {:insert-into [target {:select [:*] :from [temp]}]}))
+
+(defn compile-merge
+ "Returns `[sql params]` queries that upsert the rows of `select` (a compiled `{:query sql :params}`)
+ into `target`. `merge-spec` is `{:unique-key [col-names] :columns [all-col-names] :delete-strategy k}`."
+ [driver target select {:keys [unique-key delete-strategy] :or {delete-strategy :in}}]
+ (let [temp (driver.u/temp-table-name driver target)]
+ [(driver/compile-transform driver {:query select, :output-table temp})
+ (merge-delete-query driver target temp unique-key delete-strategy)
+ (merge-insert-query driver target temp)
+ (driver/compile-drop-table driver temp)]))
+
(mu/defmethod driver/run-transform! [:sql :table-incremental] :- ::driver/run-transform-result
- [driver {:keys [conn-spec database output-table] :as transform-details} _opts]
- (let [queries (if (driver/table-exists? driver database {:schema (namespace output-table)
- :name (name output-table)})
- (driver/compile-insert driver transform-details)
- (driver/compile-transform driver transform-details))]
- (log/tracef "Executing incremental transform queries: %s" (pr-str queries))
- ;; `execute-raw-queries!` already yields `{:rows-affected N}` maps; take the last as-is.
- (last (driver/execute-raw-queries! driver conn-spec [queries]))))
+ [driver {:keys [conn-spec database output-table query] :as transform-details} {merge-opts :merge}]
+ (let [table-exists? (driver/table-exists? driver database {:schema (namespace output-table)
+ :name (name output-table)})]
+ (cond
+ ;; First run — no target yet — create it.
+ (not table-exists?)
+ (last (driver/execute-raw-queries! driver conn-spec [(driver/compile-transform driver transform-details)]))
+
+ ;; Key-based upsert
+ merge-opts
+ (let [results (driver/execute-raw-queries!
+ driver conn-spec
+ (compile-merge driver output-table query merge-opts))]
+ {:rows-affected (reduce max 0 (keep :rows-affected results))})
+
+ ;; Append.
+ :else
+ (last (driver/execute-raw-queries! driver conn-spec [(driver/compile-insert driver transform-details)])))))
(defn qualified-name
"Return the name of the target table of a transform as a possibly qualified symbol."
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 64e6c172653..b1db52505e2 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -33,8 +33,7 @@
python-transform?
query-transform?
transform-source-database
- transform-source-type
- transform-type]
+ transform-source-type]
[metabase.transforms.util
add-source-readable
is-temp-transform-table?]
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index a7cd2dc383c..1496843e6c9 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -66,16 +66,15 @@
Resolves the field by ID and checks that its base-type is numeric or temporal.
Throws a 400 error if the field cannot be found or its type is not supported."
- [{:keys [source]}]
- (when-let [{:keys [checkpoint-filter-field-id] strategy-type :type}
- (:source-incremental-strategy source)]
- (when (= "checkpoint" strategy-type)
- (let [field (t2/select-one :model/Field checkpoint-filter-field-id)]
- (api/check-400 field (deferred-tru "Checkpoint field not found."))
- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))
- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
- (:name field)
- (pr-str (:base_type field))))))))
+ [{:keys [source] :as transform}]
+ (when (transforms-base.u/checkpoint-source? transform)
+ (let [checkpoint-filter-field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
+ field (t2/select-one :model/Field checkpoint-filter-field-id)]
+ (api/check-400 field (deferred-tru "Checkpoint field not found."))
+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))
+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
+ (:name field)
+ (pr-str (:base_type field)))))))
(defn validate-incremental-table-tag!
"Reject a table-incremental native-query transform whose source query has no table template tag for
@@ -84,8 +83,8 @@
Without that table variable the incremental filter has nowhere to be injected, so the transform is
in a broken state. This surfaces the error eagerly at create/update time instead of only when the
transform is run."
- [{:keys [target] :as transform}]
- (when (and (= :table-incremental (keyword (:type target)))
+ [transform]
+ (when (and (transforms-base.u/incremental-target? transform)
(transforms-base.u/native-query-transform? transform))
(api/check-400 (transforms-base.u/incremental-table-tag-name transform)
(deferred-tru "Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field."))))
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index e219359ddf9..0fd4827321b 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -146,6 +146,23 @@
:target_db_id (when valid-db-id? target-db-id)
:source_database_id (or source_database_id (transforms-base.i/source-db-id transform))))))
+(defn- resolve-merge-key-field-ids
+ "Fill `:field-id` on a merge target's unique-key columns from the synced target table's fields, once
+ the table exists and any are still by-name only."
+ [transform]
+ (let [target (or (:target transform) (:target (t2/original transform)))
+ table-id (or (:target_table_id transform) (:target_table_id (t2/original transform)))
+ unique-key (get-in target [:target-incremental-strategy :unique-key])]
+ (if (and (transforms-base.u/merge-target? {:target target})
+ table-id
+ (some #(nil? (:field-id %)) unique-key))
+ (let [name->id (t2/select-fn->fn :name :id [:model/Field :name :id] :table_id table-id :active true)]
+ (assoc transform :target
+ (assoc-in target [:target-incremental-strategy :unique-key]
+ (mapv (fn [e] (cond-> e (:name e) (assoc :field-id (name->id (:name e)))))
+ unique-key))))
+ transform)))
+
(t2/define-before-update :model/Transform
[{:keys [source source_database_id] :as transform}]
(when-let [new-collection (:collection_id (t2/changes transform))]
@@ -174,7 +191,10 @@
(let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]
(and old-field-id (not= old-field-id new-field-id)))
- (assoc :last_checkpoint_value nil))))
+ (assoc :last_checkpoint_value nil)
+
+ true
+ resolve-merge-key-field-ids)))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index 6c7089509e4..69749c90078 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -51,7 +51,7 @@
(tracing/with-span :tasks "task.transform.query"
{:transform/id id
:transform/target-type (name target-type)
- :transform/incremental (= :table-incremental target-type)
+ :transform/incremental (transforms-base.u/incremental-target? transform)
:transform/full-incremental-run (transforms-base.u/full-incremental-run? transform)
:db/id (:id db)
:db/engine (name driver)}
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index f38427e8e42..51a45bf63af 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -35,9 +35,22 @@
(mr/def ::append-config
[:map [:type [:= "append"]]])
+(mr/def ::merge-key-column
+ "One column of a merge unique key. Carries a resolved `:field-id` when the target column is known,
+ degrading to a `:name` ref when the target table doesn't exist yet (mirrors `::source-table-entry`)."
+ [:map
+ [:name {:optional true} ms/NonBlankString]
+ [:field-id {:optional true} [:maybe ::lib.schema.id/field]]])
+
+(mr/def ::merge-config
+ [:map
+ [:type [:= "merge"]]
+ [:unique-key [:sequential ::merge-key-column]]])
+
(mr/def ::target-incremental-strategy
[:multi {:dispatch :type}
- ["append" ::append-config]])
+ ["append" ::append-config]
+ ["merge" ::merge-config]])
(mr/def ::table-target
[:map
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
index a6dea66bfe6..23abb773d37 100644
--- a/src/metabase/transforms_base/query.clj
+++ b/src/metabase/transforms_base/query.clj
@@ -49,11 +49,16 @@
;;; ------------------------------------------------- Helpers -------------------------------------------------
-(defn- transform-opts [{:keys [transform-type]}]
+(defn- transform-opts [transform-type transform]
(case transform-type
:table {:overwrite? true}
- ;; once we have more than just append, dispatch on :target-incremental-strategy
- :table-incremental {}))
+ :table-incremental
+ (if (transforms-base.u/merge-target? transform)
+ (let [unique-key (transforms-base.u/merge-target-unique-key transform)
+ columns (transforms-base.u/target-column-names transform)]
+ (transforms-base.u/validate-merge-unique-key! unique-key columns)
+ {:merge {:unique-key unique-key, :columns columns}})
+ {})))
;;; ------------------------------------------------- Base Execution -------------------------------------------------
@@ -124,7 +129,7 @@
;; `metabase.driver.sql.query-processor/compile-transform :sql`.
:output-db (:db target)
:output-table (transforms-base.u/qualified-table-name driver target)}
- opts (transform-opts transform-details)
+ opts (transform-opts effective-transform-type transform)
features (transforms-base.u/required-database-features transform)]
(when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
(throw (ex-info "The database does not support the requested transform target type."
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 79c22cfa523..a68091a0f28 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -46,15 +46,15 @@
;;; ------------------------------------------------- Transform Type Predicates -------------------------------------------------
-(defn transform-type
- "Get the type of a transform"
- [transform]
- (-> transform :source :type keyword))
+(defn- type-is?
+ "True if `m`'s keywordized `:type` equals `t`."
+ [m t]
+ (= t (keyword (:type m))))
(defn query-transform?
"Check if this is a query transform: native query / mbql query."
[transform]
- (= :query (transform-type transform)))
+ (type-is? (:source transform) :query))
(defn native-query-transform?
"Check if this is a native query transform.
@@ -64,6 +64,45 @@
(let [query (-> transform :source :query)]
(lib/native-only-query? query))))
+(defn python-transform?
+ "Check if this is a Python transform."
+ [transform]
+ (type-is? (:source transform) :python))
+
+(defn incremental-target?
+ "True if `transform` writes to an incremental table."
+ [transform]
+ (type-is? (:target transform) :table-incremental))
+
+(defn merge-target?
+ "True if `transform`'s target uses the merge strategy."
+ [transform]
+ (type-is? (get-in transform [:target :target-incremental-strategy]) :merge))
+
+(defn checkpoint-source?
+ "True if `transform`'s source uses the checkpoint strategy."
+ [transform]
+ (type-is? (get-in transform [:source :source-incremental-strategy]) :checkpoint))
+
+(defn transform-source-database
+ "Get the source database from a transform"
+ [transform]
+ (cond
+ (query-transform? transform) (-> transform :source :query :database)
+ (python-transform? transform) (-> transform :source :source-database)))
+
+(defn full-incremental-run?
+ "True when an incremental transform should drop-and-recreate the target rather than append.
+ Fires whenever `last_checkpoint_value` is nil — covers the literal first run, and any run after the
+ `:model/Transform` before-update hook clears the watermark on a `checkpoint-filter-field-id`
+ change. Stays true across failed attempts since the predicate only inspects
+ `:last_checkpoint_value`."
+ [transform]
+ (and (incremental-target? transform)
+ (nil? (:last_checkpoint_value transform))))
+
+;;; ------------------------------------------------- Table Template Tags -------------------------------------------------
+
(defn table-template-tag-name
"Return the name (key) of the table template tag in `query` that refers to `table-id`, or nil.
@@ -90,18 +129,6 @@
table-id (when field-id (t2/select-one-fn :table_id :model/Field field-id))]
(table-template-tag-name (:query source) table-id)))
-(defn python-transform?
- "Check if this is a Python transform."
- [transform]
- (= :python (transform-type transform)))
-
-(defn transform-source-database
- "Get the source database from a transform"
- [transform]
- (case (transform-type transform)
- :query (-> transform :source :query :database)
- :python (-> transform :source :source-database)))
-
;;; ------------------------------------------------- Transform Normalization -------------------------------------------------
(defn normalize-transform
@@ -109,7 +136,7 @@
This should be called on transforms before processing them to ensure queries are in the expected format."
[transform]
(if (and (map? transform)
- (= (:type transform) "transform")
+ (type-is? transform :transform)
(get-in transform [:source :query]))
(update-in transform [:source :query] lib-be/normalize-query)
transform))
@@ -119,13 +146,13 @@
Throws if the source type cannot be detected.
Note: The transform should be normalized (via `normalize-transform`) before calling this function."
[source]
- (case (keyword (:type source))
- :python :python
- :query (if (lib/native-only-query? (:query source))
- :native
- :mbql)
- (throw (ex-info (str "Unknown transform source type: " (:type source))
- {:source source}))))
+ (cond
+ (type-is? source :python) :python
+ (type-is? source :query) (if (lib/native-only-query? (:query source))
+ :native
+ :mbql)
+ :else (throw (ex-info (str "Unknown transform source type: " (:type source))
+ {:source source}))))
;;; ------------------------------------------------- Feature Checks -------------------------------------------------
@@ -198,16 +225,6 @@
(or (isa? base-type :type/Temporal)
(isa? base-type :type/Number)))
-(defn full-incremental-run?
- "True when an incremental transform should drop-and-recreate the target rather than append.
- Fires whenever `last_checkpoint_value` is nil — covers the literal first run, and any run after the
- `:model/Transform` before-update hook clears the watermark on a `checkpoint-filter-field-id`
- change. Stays true across failed attempts since the predicate only inspects
- `:last_checkpoint_value`."
- [{:keys [target last_checkpoint_value]}]
- (and (= :table-incremental (keyword (:type target)))
- (nil? last_checkpoint_value)))
-
(defn- encode-checkpoint-value [v]
(if (number? v)
(str v)
@@ -310,17 +327,17 @@
:lo values in the source table must be > this :value.
:hi values in the source table must be <= this :value.
:rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."
- [{:keys [source target] :as transform}]
+ [{:keys [source] :as transform}]
(let [{:keys [source-incremental-strategy]} source
{:keys [checkpoint-filter-field-id]} source-incremental-strategy]
- (when (and (= "table-incremental" (:type target))
+ (when (and (incremental-target? transform)
(native-query-transform? transform)
(not (some (fn [[_k v]] (#{:table "table"} (:type v)))
(lib/template-tags (:query source)))))
(let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "
"Please add a table variable to the query and update the checkpoint field."))]
(throw (ex-info msg {:transform-message msg}))))
- (when (and (= "table-incremental" (:type target))
+ (when (and (incremental-target? transform)
(not checkpoint-filter-field-id))
(let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "
"Please select a checkpoint field in the transform settings."))]
@@ -614,6 +631,28 @@
[transform]
(t2/select-one :model/Table :transform_id (:id transform)))
+(defn merge-target-unique-key
+ "Physical column names of a transform's `merge` unique key, or nil when the target isn't a merge target."
+ [transform]
+ (when (merge-target? transform)
+ (not-empty (mapv :name (get-in transform [:target :target-incremental-strategy :unique-key])))))
+
+(defn target-column-names
+ "Physical column names of a transform's synced target table, ordered by position, or nil."
+ [transform]
+ (when-let [table (output-table transform)]
+ (not-empty (t2/select-fn-vec :name [:model/Field :name :position]
+ :table_id (:id table) :active true
+ {:order-by [[:position :asc]]}))))
+
+(defn validate-merge-unique-key!
+ "Throws if any of `unique-key` is not present in `columns`. Returns `unique-key`."
+ [unique-key columns]
+ (when-let [missing (not-empty (remove (set columns) unique-key))]
+ (let [msg (i18n/tru "Merge unique key references columns not present in the target: {0}"
+ (str/join ", " missing))]
+ (throw (ex-info msg {:transform-message msg}))))
+ unique-key)
;;; ------------------------------------------------- Source Table Schemas -------------------------------------------------
;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index 86be4dcf9c3..88ad39bcfa9 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -9,6 +9,48 @@
(set! *warn-on-reflection* true)
+;;; ------------------------------------------------- Merge target helpers -------------------------------------------------
+
+(deftest merge-target?-test
+ (testing "true only for a table-incremental target with a merge strategy"
+ (is (transforms-base.u/merge-target?
+ {:target {:type "table-incremental"
+ :target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}}))
+ (is (not (transforms-base.u/merge-target?
+ {:target {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}}})))
+ (is (not (transforms-base.u/merge-target? {:target {:type "table"}})))))
+
+(deftest merge-target-unique-key-test
+ (testing "returns the physical column names of the merge key"
+ (is (= ["id"]
+ (transforms-base.u/merge-target-unique-key
+ {:target {:target-incremental-strategy {:type "merge" :unique-key [{:name "id"}]}}})))
+ (is (= ["order_id" "region"]
+ (transforms-base.u/merge-target-unique-key
+ {:target {:target-incremental-strategy
+ {:type "merge" :unique-key [{:name "order_id"} {:name "region"}]}}}))))
+ (testing "nil when the target isn't a merge target"
+ (is (nil? (transforms-base.u/merge-target-unique-key
+ {:target {:target-incremental-strategy {:type "append"}}})))))
+
+(deftest validate-merge-unique-key!-test
+ (testing "returns the key when every column is present in the target columns"
+ (is (= ["id"] (transforms-base.u/validate-merge-unique-key! ["id"] ["id" "status"])))
+ (is (= ["a" "b"] (transforms-base.u/validate-merge-unique-key! ["a" "b"] ["a" "b" "c"]))))
+ (testing "throws when a key column is not among the target columns"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"not present in the target"
+ (transforms-base.u/validate-merge-unique-key! ["id" "ghost"] ["id" "status"])))
+ (testing "the thrown error carries a user-facing :transform-message"
+ (is (-> (try
+ (transforms-base.u/validate-merge-unique-key! ["ghost"] ["id"])
+ (catch clojure.lang.ExceptionInfo e
+ (ex-data e)))
+ :transform-message
+ some?)))))
+
(deftest throw-if-db-routing-enabled!-oss-test
(testing "on OSS (no :database-routing premium feature) the check is a no-op"
(mt/with-premium-features #{}