Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

support lookback windows for incremental transforms (#78212)

Nicola Mometto04407a3c3520
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
index 6a0a5cabf45..76cf0f27dcc 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [java-time.api :as t]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql.util :as sql.u]
@@ -10,6 +11,7 @@
[metabase.test :as mt]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
+ [metabase.util.date-2 :as u.date]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -37,6 +39,28 @@
[2 "created" #t "2026-01-31T21:00:04"]
[3 "created" #t "2026-01-31T21:00:04"]]]])
+(mt/defdataset merge-lookback-data
+ "An event-log dataset with a timestamp checkpoint column, for the lookback-window test."
+ [["order_status_lb"
+ [{:field-name "order_id" :base-type :type/Integer}
+ {:field-name "status" :base-type :type/Text}
+ {:field-name "changed_at" :base-type :type/DateTime}]
+ [[1 "created" #t "2026-01-30T10:00:00"]
+ [2 "created" #t "2026-01-31T21:00:04"]
+ [3 "created" #t "2026-01-31T21:00:04"]]]])
+
+;; A unix-timestamp (epoch seconds) checkpoint column: numeric in storage, temporal via its
+;; coercion strategy / effective type (set by the test after sync).
+(mt/defdataset merge-unix-ts-data
+ "An event-log dataset whose checkpoint column is a unix-timestamp integer."
+ [["order_status_unix"
+ [{:field-name "order_id" :base-type :type/Integer}
+ {:field-name "status" :base-type :type/Text}
+ {:field-name "changed_at" :base-type :type/BigInteger}]
+ [[1 "created" 1000000]
+ [2 "created" 2000000]
+ [3 "created" 2000000]]]])
+
;; Two key columns (order_id, region) for testing a composite merge key.
(mt/defdataset merge-composite-test
"An event log keyed on (order_id, region), for testing a two-column merge key."
@@ -218,6 +242,185 @@
(is (= expected (:field-id (first unique-key)))
"field-id resolved to the target's order_id field")))))))))
+(deftest merge-lookback-test
+ (testing "a lookback window re-reads rows behind the checkpoint so late-arriving data is upserted (GDGT-2868)"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-lookback-data
+ (with-transform-cleanup! [target-table {:type :table
+ :name "merge_lookback"
+ :schema (t2/select-one-fn :schema :model/Table
+ (mt/id :order_status_lb))
+ :database (mt/id)}]
+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
+ ts-field-id (t2/select-one-pk :model/Field :name "changed_at"
+ :table_id (mt/id :order_status_lb))
+ source-query {:database (mt/id)
+ :type :native
+ :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "status") (q "t"))
+ :template-tags {"t" {:id "t"
+ :name "t"
+ :display-name "T"
+ :type "table"
+ :table-id (mt/id :order_status_lb)
+ :required true}}}}
+ payload {:name "Lookback Merge"
+ :source_database_id (mt/id)
+ :source {:type "query"
+ :query source-query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id
+ :lookback {:value 1 :unit "day"}}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "merge"
+ :unique-key [{:name "order_id"}]}})}
+ insert! (fn [rows]
+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table
+ (mt/id :order_status_lb))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ values (str/join ", " (map (fn [[oid st ts]]
+ (format "(%d, '%s', '%s')" oid st ts))
+ rows))
+ sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema tbl)
+ (str/join "," (map q ["order_id" "status" "changed_at"]))
+ values)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+ read-target (fn []
+ (let [sql (format "SELECT %s, %s FROM %s"
+ (q "order_id") (q "status")
+ (sql.u/quote-name driver/*driver* :table
+ (:schema target-table) (:name target-table)))
+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})
+ :data :rows)]
+ {:by-order (into {} (map (fn [[oid st]] [(long oid) st])) rows)
+ :count (count rows)}))
+ delete-order! (fn [order-id]
+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table
+ (mt/id :order_status_lb))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ sql (format "DELETE FROM %s WHERE %s = %d"
+ (sql.u/quote-name driver/*driver* :table schema tbl)
+ (q "order_id") order-id)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+ watermark (fn [id] (t2/select-one-fn :last_checkpoint_value :model/Transform id))]
+ (mt/with-temp [:model/Transform transform payload]
+ (testing "first run builds the full target and records the watermark"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (is (= {:by-order {1 "created", 2 "created", 3 "created"} :count 3}
+ (read-target)))
+ (is (some? (watermark (:id transform)))))
+ (testing "a late-arriving row older than the watermark but inside the lookback window is picked up"
+ ;; watermark after run 1 is 2026-01-31T21:00:04; a 1-day lookback rescans from
+ ;; 2026-01-30T21:00:04. The late row (order 1, 01-31T00:00) is behind the watermark but
+ ;; inside the window; order 1's original row (01-30T10:00) is outside it.
+ (let [wm-before (watermark (:id transform))]
+ (insert! [[1 "shipped" "2026-01-31T00:00:00"] [4 "created" "2026-02-01T10:00:00"]])
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}
+ (read-target))
+ "late order 1 upserted, order 4 inserted, re-read rows not duplicated")
+ (testing "the run's recorded lo is the stored watermark pushed back by the lookback"
+ (let [run (t2/select-one :model/TransformRun :transform_id (:id transform)
+ {:order-by [[:id :desc]]})]
+ (is (= (u.date/format (t/minus (u.date/parse wm-before) (t/days 1)))
+ (:checkpoint_lo_value run)))))))
+ (testing "a run with no new rows leaves the target and the watermark unchanged"
+ (let [before (watermark (:id transform))]
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}
+ (read-target)))
+ (is (= before (watermark (:id transform)))
+ "the watermark must not slide backwards by the lookback window")))
+ (testing "deleting the watermark row from the source does not regress the watermark"
+ ;; With order 4 (the row that set the watermark, 02-01T10:00) deleted, max() over
+ ;; the lookback window is orders 2/3 at 01-31T21:00:04 — behind the stored
+ ;; watermark. The new-watermark clamp must keep the old value rather than let it
+ ;; slide back (see get-source-range-params' `hi`).
+ (let [before (watermark (:id transform))]
+ (delete-order! 4)
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= before (watermark (:id transform)))
+ "the watermark must not regress when the max row disappears from the source")
+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}
+ (read-target))
+ "merge does not propagate source deletes; re-read rows upsert idempotently")))))))))))
+
+(deftest merge-lookback-unix-timestamp-test
+ (testing "a lookback works on a unix-timestamp checkpoint column via its effective type (GDGT-2868)"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-unix-ts-data
+ (with-transform-cleanup! [target-table {:type :table
+ :name "merge_lookback_unix"
+ :schema (t2/select-one-fn :schema :model/Table
+ (mt/id :order_status_unix))
+ :database (mt/id)}]
+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
+ ts-field-id (mt/id :order_status_unix :changed_at)
+ _ (t2/update! :model/Field ts-field-id
+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime
+ :effective_type :type/Instant})
+ payload {:name "Unix Timestamp Lookback Merge"
+ :source_database_id (mt/id)
+ :source {:type "query"
+ :query {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :order_status_unix)}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id
+ :lookback {:value 1 :unit "day"}}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "merge"
+ :unique-key [{:name "order_id"}]}})}
+ insert! (fn [rows]
+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table
+ (mt/id :order_status_unix))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ values (str/join ", " (map (fn [[oid st ts]]
+ (format "(%d, '%s', %d)" oid st ts))
+ rows))
+ sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema tbl)
+ (str/join "," (map q ["order_id" "status" "changed_at"]))
+ values)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+ read-target (fn []
+ (let [sql (format "SELECT %s, %s FROM %s"
+ (q "order_id") (q "status")
+ (sql.u/quote-name driver/*driver* :table
+ (:schema target-table) (:name target-table)))
+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})
+ :data :rows)]
+ {:by-order (into {} (map (fn [[oid st]] [(long oid) st])) rows)
+ :count (count rows)}))]
+ (mt/with-temp [:model/Transform transform payload]
+ (testing "first run builds the full target and stores a temporal watermark"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (is (= {:by-order {1 "created", 2 "created", 3 "created"} :count 3}
+ (read-target)))
+ (let [wm (t2/select-one-fn :last_checkpoint_value :model/Transform (:id transform))]
+ (is (re-find #"T" (str wm))
+ "the watermark encodes as a temporal value, not the raw epoch number")))
+ (testing "a late-arriving epoch inside the lookback window is upserted on rerun"
+ ;; watermark = epoch 2000000; a 1-day lookback rescans from 1913600. Order 4's
+ ;; epoch 1950000 is behind the watermark but inside the window; order 1's
+ ;; original row (1000000) is outside it.
+ (insert! [[1 "shipped" 2100000] [4 "created" 1950000]])
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}
+ (read-target))
+ "late order 4 picked up, order 1 upserted, re-read rows not duplicated"))))))))))
+
(deftest merge-composite-key-test
(testing "a merge transform can upsert on a two-column key"
(mt/test-drivers (test-drivers)
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 6fb5dfab6dd..86eaf7d0756 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -3,7 +3,7 @@ import type { DatabaseId } from "./database";
import type { RowValue } from "./dataset";
import type { RequestableIndexes } from "./index-manager";
import type { PaginationRequest, PaginationResponse } from "./pagination";
-import type { DatasetQuery, JoinStrategy } from "./query";
+import type { DatasetQuery, DateTimeAbsoluteUnit, JoinStrategy } from "./query";
import type { ScheduleDisplayType } from "./settings";
import type { SortDirection } from "./sorting";
import type { ConcreteTableId, SchemaName, Table } from "./table";
@@ -97,9 +97,18 @@ export type PythonTransformTableEntry = {
export type PythonTransformTableAliases = PythonTransformTableEntry[];
+export type LookbackUnit = "millisecond" | "second" | DateTimeAbsoluteUnit;
+
+// Only supported for temporal checkpoint columns.
+export type TransformLookback = {
+ value: number;
+ unit: LookbackUnit;
+};
+
export type TransformSourceCheckpointStrategy = {
type: "checkpoint";
"checkpoint-filter-field-id": number;
+ lookback?: TransformLookback | null;
};
export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;
diff --git a/frontend/src/metabase/api/field.ts b/frontend/src/metabase/api/field.ts
index a859d7a5997..0e560e2c964 100644
--- a/frontend/src/metabase/api/field.ts
+++ b/frontend/src/metabase/api/field.ts
@@ -166,6 +166,7 @@ export const fieldApi = Api.injectEndpoints({
export const {
useGetFieldQuery,
+ useLazyGetFieldQuery,
useGetFieldTableIdsQuery,
useGetFieldValuesQuery,
useGetRemappedFieldValueQuery,
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index 77a2058d135..d98dfa698ae 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -28,6 +28,7 @@ import {
NativeQueryTableTagFieldSelect,
PythonKeysetColumnSelect,
} from "./KeysetColumnSelect";
+import { LookbackField } from "./LookbackField";
import { UniqueKeyField } from "./UniqueKeyField";
import type { IncrementalSettingsFormValues } from "./form";
import { useHasCheckpointOptions } from "./useHasCheckpointOptions";
@@ -295,6 +296,7 @@ function SourceStrategyFields({
disabled={readOnly}
/>
)}
+ <LookbackField readOnly={readOnly} />
</>
)}
</>
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
index b45618cb29b..ea13ac9c1bc 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
@@ -5,20 +5,23 @@ import {
type DataAttributes,
type InputDescriptionProps,
Loader,
- type SelectOption,
} from "metabase/ui";
import * as Lib from "metabase-lib";
import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";
+import {
+ type CheckpointFieldOption,
+ useClearUnsupportedLookback,
+} from "../useClearUnsupportedLookback";
/**
* Get filterable numeric/temporal field options from the source table of a query.
- * Returns SelectOption[] with field IDs as string values.
+ * Returns CheckpointFieldOption[] with field IDs as string values.
*/
export function getSourceFieldOptions(
query: Lib.Query,
opts?: { labelPrefix?: string; seenFieldIds?: Set<number> },
-): Array<SelectOption> {
+): Array<CheckpointFieldOption> {
// Stage 0 is the source stage (raw table data). Keyset pagination requires columns from the
// actual source table; later stages may have aggregations, joins, or expressions instead.
const stageIndex = 0;
@@ -36,10 +39,14 @@ export function getSourceFieldOptions(
const sourceColumns = Lib.getColumnsFromColumnGroup(sourceGroup);
const seenFieldIds = opts?.seenFieldIds ?? new Set<number>();
- const options: Array<SelectOption> = [];
+ const options: Array<CheckpointFieldOption> = [];
for (const column of sourceColumns) {
- if (!Lib.isNumeric(column) && !Lib.isTemporal(column)) {
+ // Numeric or date/datetime only — time-only columns are excluded since their
+ // watermarks wrap at midnight
+ const isSupported =
+ (Lib.isNumeric(column) || Lib.isTemporal(column)) && !Lib.isTime(column);
+ if (!isSupported) {
continue;
}
@@ -55,7 +62,11 @@ export function getSourceFieldOptions(
? `${opts.labelPrefix}: ${columnInfo.displayName}`
: columnInfo.longDisplayName;
- options.push({ value: String(fieldId), label });
+ options.push({
+ value: String(fieldId),
+ label,
+ supportsLookback: Lib.isDateOrDateTime(column),
+ });
}
return options;
@@ -82,7 +93,7 @@ export function KeysetColumnSelect({
disabled,
isLoading,
}: KeysetColumnSelectProps) {
- const columnOptions = useMemo((): Array<SelectOption> => {
+ const columnOptions = useMemo((): Array<CheckpointFieldOption> => {
if (!query) {
return [];
}
@@ -104,6 +115,8 @@ export function KeysetColumnSelect({
disabled: disabled || isLoading,
});
+ useClearUnsupportedLookback({ name, options: columnOptions });
+
return (
<FormSelect
name={name}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.ts
new file mode 100644
index 00000000000..2112d32280b
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.ts
@@ -0,0 +1,21 @@
+import * as Lib from "metabase-lib";
+import { SAMPLE_PROVIDER } from "metabase-lib/test-helpers";
+import { ORDERS, ORDERS_ID } from "metabase-types/api/mocks/presets";
+
+import { getSourceFieldOptions } from "./KeysetColumnSelect";
+
+const query = Lib.createTestQuery(SAMPLE_PROVIDER, {
+ stages: [{ source: { type: "table", id: ORDERS_ID } }],
+});
+
+describe("getSourceFieldOptions", () => {
+ it("marks only date/datetime columns as supporting a lookback", () => {
+ const options = getSourceFieldOptions(query);
+ const supportsLookbackByFieldId = new Map(
+ options.map(({ value, supportsLookback }) => [value, supportsLookback]),
+ );
+
+ expect(supportsLookbackByFieldId.get(String(ORDERS.CREATED_AT))).toBe(true);
+ expect(supportsLookbackByFieldId.get(String(ORDERS.TOTAL))).toBe(false);
+ });
+});
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx
index ec51c74de3f..ce215abacf4 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx
@@ -12,6 +12,7 @@ import {
import type * as Lib from "metabase-lib";
import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";
+import { useClearUnsupportedLookback } from "../useClearUnsupportedLookback";
import { useNativeCheckpointFieldOptions } from "../useNativeCheckpointFieldOptions";
type NativeQueryTableTagFieldSelectProps = {
@@ -52,6 +53,8 @@ export function NativeQueryTableTagFieldSelect({
disabled: disabled || isLoading,
});
+ useClearUnsupportedLookback({ name, options: fieldOptions });
+
if (noQueryMessage) {
return (
<Alert size="compact" icon={<Icon name="warning" />} color="warning">
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsx
new file mode 100644
index 00000000000..fd630cc393f
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsx
@@ -0,0 +1,102 @@
+import { useFormikContext } from "formik";
+import { useEffect } from "react";
+import { t } from "ttag";
+
+import { skipToken, useGetFieldQuery } from "metabase/api";
+import { FormField, FormNumberInput, FormSelect } from "metabase/forms";
+import { Group } from "metabase/ui";
+import { isDateWithoutTime } from "metabase-lib/v1/types/utils/isa";
+import type { LookbackUnit } from "metabase-types/api";
+
+import {
+ type IncrementalSettingsFormValues,
+ fieldSupportsLookback,
+} from "./form";
+
+const SUB_DAY_UNITS: ReadonlySet<LookbackUnit> = new Set([
+ "millisecond",
+ "second",
+ "minute",
+ "hour",
+]);
+
+const getUnitOptions = (dateOnly: boolean) => {
+ const options: { value: LookbackUnit; label: string }[] = [
+ { value: "minute", label: t`minutes` },
+ { value: "hour", label: t`hours` },
+ { value: "day", label: t`days` },
+ { value: "week", label: t`weeks` },
+ { value: "month", label: t`months` },
+ { value: "year", label: t`years` },
+ ];
+ return dateOnly
+ ? options.filter(({ value }) => !SUB_DAY_UNITS.has(value))
+ : options;
+};
+
+// The lookback window input: a number plus a unit. Only shown for temporal checkpoint columns —
+// the only kind that supports a lookback.
+export function LookbackField({ readOnly }: { readOnly?: boolean }) {
+ const { values, setFieldValue } =
+ useFormikContext<IncrementalSettingsFormValues>();
+ const fieldId = values.checkpointFilterFieldId;
+ const { data: field } = useGetFieldQuery(
+ fieldId != null ? { id: Number(fieldId) } : skipToken,
+ );
+
+ const isSupported = fieldSupportsLookback(field);
+ const isDateOnly = isDateWithoutTime(field);
+
+ // Keep a configured lookback consistent after a checkpoint-field change: clear it when the new
+ // column doesn't support one (hiding the input alone would still submit the stale value), and
+ // snap sub-day units back to days on date-only columns. Guarded on a set value — a write here
+ // counts as a form change and triggers an inline save.
+ useEffect(() => {
+ if (field == null || values.lookbackValue == null) {
+ return;
+ }
+ if (!isSupported) {
+ setFieldValue("lookbackValue", null);
+ } else if (isDateOnly && SUB_DAY_UNITS.has(values.lookbackUnit)) {
+ setFieldValue("lookbackUnit", "day");
+ }
+ }, [
+ field,
+ isSupported,
+ isDateOnly,
+ values.lookbackValue,
+ values.lookbackUnit,
+ setFieldValue,
+ ]);
+
+ if (fieldId == null || !isSupported) {
+ return null;
+ }
+
+ return (
+ <FormField
+ title={t`Lookback window`}
+ description={t`Optional. Re-process this much already-seen data on each run, to catch late-arriving rows.`}
+ maw="24rem"
+ >
+ <Group gap="sm" wrap="nowrap">
+ <FormNumberInput
+ name="lookbackValue"
+ nullable
+ min={1}
+ placeholder={t`e.g. 4`}
+ aria-label={t`Lookback amount`}
+ disabled={readOnly}
+ w="10rem"
+ />
+ <FormSelect
+ name="lookbackUnit"
+ aria-label={t`Lookback unit`}
+ data={getUnitOptions(isDateOnly)}
+ disabled={readOnly}
+ w="9rem"
+ />
+ </Group>
+ </FormField>
+ );
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
index 0a9c68d969f..8fd68574a4d 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
@@ -1,13 +1,26 @@
import * as Yup from "yup";
import * as Errors from "metabase/utils/errors";
+import {
+ type FieldTypeInfo,
+ isDate,
+ isTime,
+} from "metabase-lib/v1/types/utils/isa";
import type {
+ LookbackUnit,
Transform,
TransformSource,
TransformTarget,
UpdateTransformRequest,
} from "metabase-types/api";
+// Same rule as the BE's `date-or-datetime?`: temporal but not time-only (time watermarks wrap at
+// midnight). The predicates check effective_type first, so coerced columns (e.g. unix
+// timestamps) count as temporal.
+export const fieldSupportsLookback = (
+ field: FieldTypeInfo | null | undefined,
+): boolean => isDate(field) && !isTime(field);
+
export type IncrementalSettingsFormValues = {
incremental: boolean;
sourceStrategy: "checkpoint";
@@ -15,6 +28,10 @@ export type IncrementalSettingsFormValues = {
// Comma-separated target column names. Empty → append. Non-empty → the target is upserted
// (merge/restate) on these columns instead of appended.
uniqueKey: string;
+ // Lookback window: re-read this much data behind the checkpoint on every run. Null → none.
+ // Only supported for temporal checkpoint columns.
+ lookbackValue: number | null;
+ lookbackUnit: LookbackUnit;
};
export const VALIDATION_SCHEMA = Yup.object({
@@ -30,6 +47,8 @@ export const VALIDATION_SCHEMA = Yup.object({
otherwise: (schema) => schema.nullable().defined(),
}),
uniqueKey: Yup.string().default(""),
+ lookbackValue: Yup.number().nullable().positive().integer().default(null),
+ lookbackUnit: Yup.mixed<LookbackUnit>().default("day"),
});
export const getInitialValues = (
@@ -39,6 +58,8 @@ export const getInitialValues = (
sourceStrategy: "checkpoint",
checkpointFilterFieldId: null,
uniqueKey: "",
+ lookbackValue: null,
+ lookbackUnit: "day",
...defaults,
});
@@ -60,6 +81,8 @@ export const getIncrementalSettingsFromTransform = (
? (strategy["checkpoint-filter-field-id"] ?? null)
: null;
+ const lookback = strategy?.type === "checkpoint" ? strategy.lookback : null;
+
const targetStrategy =
transform.target.type === "table-incremental"
? transform.target["target-incremental-strategy"]
@@ -72,6 +95,8 @@ export const getIncrementalSettingsFromTransform = (
isIncremental && checkpointFilterFieldId != null
? String(checkpointFilterFieldId)
: null,
+ lookbackValue: lookback?.value ?? null,
+ lookbackUnit: lookback?.unit ?? "day",
uniqueKey:
targetStrategy?.type === "merge"
? targetStrategy["unique-key"]
@@ -116,6 +141,13 @@ export const buildIncrementalSource = (
"checkpoint-filter-field-id": Number(
formValues.checkpointFilterFieldId,
),
+ lookback:
+ formValues.lookbackValue !== null
+ ? {
+ value: formValues.lookbackValue,
+ unit: formValues.lookbackUnit,
+ }
+ : undefined,
}
: undefined,
};
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
index 5a1939a746c..d036d78af39 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
@@ -1,8 +1,12 @@
-import { createMockTransform } from "metabase-types/api/mocks";
+import {
+ createMockTransform,
+ createMockTransformSource,
+} from "metabase-types/api/mocks";
import {
type IncrementalSettingsFormValues,
VALIDATION_SCHEMA,
+ buildIncrementalSource,
buildIncrementalTarget,
getIncrementalSettingsFromTransform,
} from "./form";
@@ -14,6 +18,8 @@ const getValues = (
sourceStrategy: "checkpoint",
checkpointFilterFieldId: "123",
uniqueKey: "",
+ lookbackValue: null,
+ lookbackUnit: "day",
...values,
});
@@ -92,6 +98,63 @@ describe("buildIncrementalTarget (merge)", () => {
});
});
+describe("buildIncrementalSource (lookback)", () => {
+ it("omits lookback when no value is set", () => {
+ const source = buildIncrementalSource(
+ createMockTransformSource(),
+ getValues(),
+ );
+ expect(source["source-incremental-strategy"]?.lookback).toBeUndefined();
+ });
+
+ it("includes value and unit when a lookback is set", () => {
+ const source = buildIncrementalSource(
+ createMockTransformSource(),
+ getValues({ lookbackValue: 4, lookbackUnit: "week" }),
+ );
+ expect(source["source-incremental-strategy"]?.lookback).toEqual({
+ value: 4,
+ unit: "week",
+ });
+ });
+});
+
+describe("getIncrementalSettingsFromTransform (lookback)", () => {
+ it("reads the lookback into form values", () => {
+ const transform = createMockTransform({
+ source: {
+ ...createMockTransformSource(),
+ "source-incremental-strategy": {
+ type: "checkpoint",
+ "checkpoint-filter-field-id": 123,
+ lookback: { value: 4, unit: "day" },
+ },
+ },
+ target: {
+ type: "table-incremental",
+ name: "t",
+ schema: "public",
+ database: 1,
+ "target-incremental-strategy": { type: "append" },
+ },
+ });
+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({
+ incremental: true,
+ checkpointFilterFieldId: "123",
+ lookbackValue: 4,
+ lookbackUnit: "day",
+ });
+ });
+
+ it("defaults to no lookback when the strategy has none", () => {
+ const transform = createMockTransform();
+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({
+ lookbackValue: null,
+ lookbackUnit: "day",
+ });
+ });
+});
+
describe("getIncrementalSettingsFromTransform (merge)", () => {
it("reads the merge unique key into a comma-separated string", () => {
const transform = createMockTransform({
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.ts
new file mode 100644
index 00000000000..e062736505b
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.ts
@@ -0,0 +1,32 @@
+import { useField, useFormikContext } from "formik";
+import { useEffect } from "react";
+
+import type { SelectOption } from "metabase/ui";
+
+import type { IncrementalSettingsFormValues } from "./form";
+
+export type CheckpointFieldOption = SelectOption & {
+ supportsLookback: boolean;
+};
+
+// Clears a configured lookback when the selected checkpoint column doesn't support one
+// (only date/datetime columns do), so a column change never sends a stale lookback to the API.
+export function useClearUnsupportedLookback({
+ name,
+ options,
+}: {
+ name: string;
+ options: CheckpointFieldOption[];
+}) {
+ const [{ value: selectedFieldId }] = useField<string | null>(name);
+ const { values, setFieldValue } =
+ useFormikContext<IncrementalSettingsFormValues>();
+ const { lookbackValue } = values;
+
+ useEffect(() => {
+ const option = options.find(({ value }) => value === selectedFieldId);
+ if (option != null && !option.supportsLookback && lookbackValue != null) {
+ setFieldValue("lookbackValue", null);
+ }
+ }, [options, selectedFieldId, lookbackValue, setFieldValue]);
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsx
new file mode 100644
index 00000000000..7c4d0436184
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsx
@@ -0,0 +1,64 @@
+import { renderHook, waitFor } from "@testing-library/react";
+import { useFormikContext } from "formik";
+import type { ReactNode } from "react";
+
+import { FormProvider } from "metabase/forms";
+
+import { type IncrementalSettingsFormValues, getInitialValues } from "./form";
+import {
+ type CheckpointFieldOption,
+ useClearUnsupportedLookback,
+} from "./useClearUnsupportedLookback";
+
+const OPTIONS: CheckpointFieldOption[] = [
+ { value: "1", label: "Total", supportsLookback: false },
+ { value: "2", label: "Created At", supportsLookback: true },
+];
+
+function setup({
+ checkpointFilterFieldId,
+ options = OPTIONS,
+}: {
+ checkpointFilterFieldId: string | null;
+ options?: CheckpointFieldOption[];
+}) {
+ const initialValues = getInitialValues({
+ incremental: true,
+ checkpointFilterFieldId,
+ lookbackValue: 4,
+ lookbackUnit: "hour",
+ });
+
+ const wrapper = ({ children }: { children: ReactNode }) => (
+ <FormProvider initialValues={initialValues} onSubmit={jest.fn()}>
+ {children}
+ </FormProvider>
+ );
+
+ return renderHook(
+ () => {
+ useClearUnsupportedLookback({ name: "checkpointFilterFieldId", options });
+ return useFormikContext<IncrementalSettingsFormValues>();
+ },
+ { wrapper },
+ );
+}
+
+describe("useClearUnsupportedLookback", () => {
+ it("clears the lookback when the selected column doesn't support one", async () => {
+ const { result } = setup({ checkpointFilterFieldId: "1" });
+ await waitFor(() => {
+ expect(result.current.values.lookbackValue).toBeNull();
+ });
+ });
+
+ it("keeps the lookback when the selected column supports one", () => {
+ const { result } = setup({ checkpointFilterFieldId: "2" });
+ expect(result.current.values.lookbackValue).toBe(4);
+ });
+
+ it("keeps the lookback when the selected column isn't among the options", () => {
+ const { result } = setup({ checkpointFilterFieldId: "99" });
+ expect(result.current.values.lookbackValue).toBe(4);
+ });
+});
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts
index ade88e1237f..a78b35f8692 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts
@@ -2,11 +2,11 @@ import { useMemo } from "react";
import { useSelector } from "metabase/redux";
import { getMetadata } from "metabase/selectors/metadata";
-import type { SelectOption } from "metabase/ui";
import * as Lib from "metabase-lib";
import { isConcreteTableId } from "metabase-types/api/table";
import { getSourceFieldOptions } from "./KeysetColumnSelect/KeysetColumnSelect";
+import type { CheckpointFieldOption } from "./useClearUnsupportedLookback";
import { useTableQueryMetadataResults } from "./useTableQueryMetadataResults";
export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {
@@ -30,13 +30,13 @@ export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {
const { tables, isLoading, hasError } =
useTableQueryMetadataResults(tableIds);
- const fieldOptions = useMemo((): Array<SelectOption> => {
+ const fieldOptions = useMemo((): Array<CheckpointFieldOption> => {
if (tables.length === 0) {
return [];
}
try {
- const allOptions: Array<SelectOption> = [];
+ const allOptions: Array<CheckpointFieldOption> = [];
const seenFieldIds = new Set<number>();
const showTablePrefix = tables.length > 1;
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
index 38fdb3a59ac..74e243b8908 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
@@ -1,26 +1,57 @@
import { useMemo } from "react";
-import { useUpdateTransformMutation } from "metabase/api";
+import { useLazyGetFieldQuery, useUpdateTransformMutation } from "metabase/api";
import type { Transform } from "metabase-types/api";
import {
type IncrementalSettingsFormValues,
VALIDATION_SCHEMA,
convertTransformFormToUpdateRequest,
+ fieldSupportsLookback,
getIncrementalSettingsFromTransform,
} from "./form";
export function useUpdateIncrementalSettings(transform: Transform) {
const [updateTransform] = useUpdateTransformMutation();
+ const [fetchField] = useLazyGetFieldQuery();
const initialValues = useMemo(
() => getIncrementalSettingsFromTransform(transform),
[transform],
);
+ // The inline save can fire before the newly selected checkpoint field's metadata has loaded
+ // (and before LookbackField's effect clears a now-unsupported lookback), so re-check against
+ // the authoritative field data before building the request. Usually a cache hit.
+ const dropUnsupportedLookback = async (
+ values: IncrementalSettingsFormValues,
+ ): Promise<IncrementalSettingsFormValues> => {
+ if (
+ values.lookbackValue == null ||
+ values.checkpointFilterFieldId == null
+ ) {
+ return values;
+ }
+ try {
+ const field = await fetchField(
+ { id: Number(values.checkpointFilterFieldId) },
+ true,
+ ).unwrap();
+ return fieldSupportsLookback(field)
+ ? values
+ : { ...values, lookbackValue: null };
+ } catch {
+ // let the save proceed; the BE validates anyway
+ return values;
+ }
+ };
+
const updateIncrementalSettings = async (
values: IncrementalSettingsFormValues,
) => {
- const requestData = convertTransformFormToUpdateRequest(transform, values);
+ const requestData = convertTransformFormToUpdateRequest(
+ transform,
+ await dropUnsupportedLookback(values),
+ );
return await updateTransform(requestData).unwrap();
};
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index f6db0aedcb9..e9e6b4b119e 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -9,6 +9,7 @@
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
+ [metabase.lib.types.isa :as lib.types.isa]
[metabase.models.interface :as mi]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as transforms-base.ordering]
@@ -63,6 +64,11 @@
(api/check-400 (not (str/blank? (get-in transform [:target :schema])))
(deferred-tru "A target schema is required for this database.")))))
+(defn- field->column
+ "Adapt a t2 Field row to the kebab-case column shape `lib.types.isa` predicates expect."
+ [field]
+ {:base-type (:base_type field), :effective-type (:effective_type field)})
+
(defn validate-incremental-column-type!
"Validates that the checkpoint column for an incremental transform has a supported type.
@@ -73,10 +79,25 @@
(let [checkpoint-filter-field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
field (t2/select-one :model/Field checkpoint-filter-field-id)]
(api/check-400 field (deferred-tru "Checkpoint field not found."))
- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))
- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
- (:name field)
- (pr-str (:base_type field)))))))
+ (let [column (field->column field)]
+ (api/check-400 (transforms-base.u/supported-checkpoint-column? column)
+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
+ (:name field)
+ (pr-str (lib.types.isa/column-type column))))))))
+
+(defn validate-lookback!
+ "Validates a configured lookback window: only date and datetime checkpoint columns support one
+ (time-only columns wrap at midnight), and date-only columns need a day-or-coarser unit."
+ [{:keys [source] :as transform}]
+ (let [{:keys [lookback checkpoint-filter-field-id]} (:source-incremental-strategy source)]
+ (when-let [{:keys [unit]} (and (transforms-base.u/checkpoint-source? transform) lookback)]
+ (when-let [field (t2/select-one :model/Field checkpoint-filter-field-id)]
+ (let [column (field->column field)]
+ (api/check-400 (lib.types.isa/date-or-datetime? column)
+ (deferred-tru "A lookback window is only supported for date or datetime checkpoint columns."))
+ (when (lib.types.isa/date-without-time? column)
+ (api/check-400 (contains? #{"day" "week" "month" "quarter" "year"} unit)
+ (deferred-tru "A lookback window on a date checkpoint column must use days or a coarser unit."))))))))
(defn validate-incremental-table-tag!
"Reject a table-incremental native-query transform whose source query has no table template tag for
@@ -142,6 +163,7 @@
(validate-transform-query! body))
(validate-target-schema! body)
(validate-incremental-table-tag! body)
+ (validate-lookback! body)
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
(let [tag-ids (:tag_ids body)
@@ -180,7 +202,8 @@
(when (contains? body :source)
(validate-incremental-column-type! new))
(when (or (contains? body :source) (contains? body :target))
- (validate-incremental-table-tag! new))
+ (validate-incremental-table-tag! new)
+ (validate-lookback! new))
(when (transforms-base.u/query-transform? old)
(validate-transform-query! new)
(when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 36c36c99099..2eb7e88db24 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -111,7 +111,10 @@
mi/json-out-without-keywordization
(update-keys keyword)
(m/update-existing :query (fn [q] (if (orphan-query? q) q (lib-be/normalize-query q))))
- (m/update-existing :source-incremental-strategy #(update-keys % keyword))
+ (m/update-existing :source-incremental-strategy
+ (fn [strategy]
+ (-> (update-keys strategy keyword)
+ (m/update-existing :lookback #(some-> % (update-keys keyword))))))
(m/update-existing :source-tables (fn [st] (mapv #(update-keys % keyword) st)))
(m/update-existing :type keyword)))
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index 51a45bf63af..7b93e4a5346 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -4,13 +4,23 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.schema :as queries.schema]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.util.date-2 :as u.date]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]))
+(mr/def ::lookback
+ "A lookback window: each run re-reads source rows up to `value` `unit`s behind the checkpoint,
+ so late-arriving rows older than the watermark still get picked up. Only supported for
+ temporal checkpoint columns."
+ [:map
+ [:value pos-int?]
+ [:unit (into [:enum] (map name) (sort u.date/add-units))]])
+
(mr/def ::checkpoint-strategy
[:map
[:type [:= "checkpoint"]]
- [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]])
+ [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]
+ [:lookback {:optional true} [:maybe ::lookback]]])
(mr/def ::source-incremental-strategy
[:multi {:dispatch :type}
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 79baec35366..d2b78ca6213 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -16,6 +16,7 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.types.isa :as lib.types.isa]
[metabase.lib.util :as lib.util]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.core :as qp]
@@ -236,13 +237,15 @@
;;; ------------------------------------------------- Incremental/Checkpoint Helpers -------------------------------------------------
-(defn supported-incremental-filter-type?
- "Returns true if the given base-type is supported for incremental filtering.
+(defn supported-checkpoint-column?
+ "Returns true if `column` (or any map with `:base-type`/`:effective-type`) can be used for
+ incremental checkpoint filtering.
- We only support temporal (timestamp/tz) and numeric (int/float) types."
- [base-type]
- (or (isa? base-type :type/Temporal)
- (isa? base-type :type/Number)))
+ We support date/datetime and numeric (int/float) columns. Time-only columns are excluded:
+ their watermarks wrap at midnight, so `>` comparisons against them are meaningless."
+ [column]
+ (or (lib.types.isa/date-or-datetime? column)
+ (lib.types.isa/numeric? column)))
(defn- encode-checkpoint-value [v]
(if (number? v)
@@ -309,10 +312,66 @@
:else (throw (ex-info (str "Unsupported checkpoint type: " (pr-str base-type))
{:base-type base-type}))))
-(defn- tag-checkpoint-value
- "Wrap a raw checkpoint value from the QP into a map `{:value parsed}`."
- [base-type raw-value]
- {:value (parse-checkpoint-value base-type raw-value)})
+(declare ->instant)
+
+(defn- apply-lookback
+ "Push a checkpoint lower bound back by the lookback window (`value` `unit`s). Only supported
+ for date and datetime checkpoint columns: time-only columns wrap at midnight, so a window
+ behind the watermark is meaningless (and day-based units don't apply to them)."
+ [checkpoint column {:keys [value unit] :as lookback}]
+ (let [invalid! (fn [msg] (throw (ex-info msg {:transform-message msg, :lookback lookback})))]
+ (cond
+ (not (lib.types.isa/date-or-datetime? column))
+ (invalid! (i18n/tru "A lookback window is only supported for date or datetime checkpoint columns."))
+
+ (nil? unit)
+ (invalid! (i18n/tru "A lookback window requires a unit."))
+
+ (not (pos-int? value))
+ (invalid! (i18n/tru "A lookback window requires a positive integer value."))
+
+ :else
+ (u.date/add checkpoint (keyword unit) (- value)))))
+
+(defn- checkpoint-compare
+ "Compare two parsed checkpoint values. Temporal values compare as instants, since the stored
+ watermark and a fresh QP max can parse to incomparable temporal classes."
+ [base-type a b]
+ (if (isa? base-type :type/Temporal)
+ (compare (->instant a) (->instant b))
+ (compare a b)))
+
+(defn- validate-incremental-source!
+ "Throws a user-facing error when an incremental transform can't compute its source range: a
+ native query without a table variable to inject the filter into, or no checkpoint field
+ selected."
+ [{:keys [source] :as transform}]
+ (when (incremental-target? transform)
+ (when (and (native-query-transform? transform)
+ (not (some (fn [tag] (#{:table "table"} (:type tag)))
+ (lib/template-tags (:query source)))))
+ (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "
+ "Please add a table variable to the query and update the checkpoint field."))]
+ (throw (ex-info msg {:transform-message msg}))))
+ (when-not (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])
+ (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "
+ "Please select a checkpoint field in the transform settings."))]
+ (throw (ex-info msg {:transform-message msg}))))))
+
+(defn- checkpoint-column
+ "The checkpoint column fetched from `metadata-provider`, validated to exist, be active, and have
+ a type supported for incremental filtering."
+ [metadata-provider checkpoint-filter-field-id]
+ (let [column (lib.metadata/field metadata-provider checkpoint-filter-field-id)]
+ (when (or (nil? column) (not (:active column)))
+ (throw (ex-info "Checkpoint field does not exist or is not active"
+ {:checkpoint-filter-field-id checkpoint-filter-field-id})))
+ (when-not (supported-checkpoint-column? column)
+ (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type "
+ (pr-str (lib.types.isa/column-type column)) ". "
+ "Only numeric and temporal columns are supported for incremental filtering.")
+ {:column column})))
+ column))
(defn- inject-filters-into-table-tag
"Inject `:source-filters` into the table template tag matching the checkpoint field's table.
@@ -353,36 +412,20 @@
:hi values in the source table must be <= this :value.
:rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."
[{:keys [source] :as transform}]
- (let [{:keys [source-incremental-strategy]} source
- {:keys [checkpoint-filter-field-id]} source-incremental-strategy]
- (when (and (incremental-target? transform)
- (native-query-transform? transform)
- (not (some (fn [tag] (#{:table "table"} (:type tag)))
- (lib/template-tags (:query source)))))
- (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "
- "Please add a table variable to the query and update the checkpoint field."))]
- (throw (ex-info msg {:transform-message msg}))))
- (when (and (incremental-target? transform)
- (not checkpoint-filter-field-id))
- (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "
- "Please select a checkpoint field in the transform settings."))]
- (throw (ex-info msg {:transform-message msg}))))
+ (let [{:keys [checkpoint-filter-field-id lookback]} (:source-incremental-strategy source)]
+ (validate-incremental-source! transform)
(when checkpoint-filter-field-id
(let [{:keys [last_checkpoint_value]} (cond-> transform
(full-incremental-run? transform)
(assoc :last_checkpoint_value nil))
db-id (transforms-base.i/target-db-id transform)
metadata-provider (lib-be/application-database-metadata-provider db-id)
- column (lib.metadata/field metadata-provider checkpoint-filter-field-id)
- _ (when (or (nil? column) (not (:active column)))
- (throw (ex-info "Checkpoint field does not exist or is not active"
- {:checkpoint-filter-field-id checkpoint-filter-field-id})))
- _ (when-not (supported-incremental-filter-type? (:base-type column))
- (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type " (pr-str (:base-type column)) ". "
- "Only numeric and temporal columns are supported for incremental filtering.")
- {:column column})))
- base-type (:base-type column)
- lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))
+ column (checkpoint-column metadata-provider checkpoint-filter-field-id)
+ base-type (lib.types.isa/column-type column)
+ ;; `checkpoint-lo` is the stored watermark; `lo` is the scan bound, pushed back by any lookback.
+ checkpoint-lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))
+ lo (cond-> checkpoint-lo
+ (and checkpoint-lo lookback) (apply-lookback column lookback))
;; Combine max + count in one scan: avoids a second round-trip and pins both
;; numbers to the same point-in-time view of the source.
@@ -400,12 +443,23 @@
;; instead of NULL for `max()` over an empty relation when the column is non-nullable. Only
;; trust the max when the count from the same scan says there were rows, otherwise the
;; watermark would silently regress and the next run would reprocess already-seen rows.
- [(when-not (and cv (zero? cv)) mv) cv])]
+ [(when-not (and cv (zero? cv)) mv) cv])
+
+ ;; The new watermark, clamped to the stored one: with a lookback the scan starts behind
+ ;; it, so an empty scan or a max() over only late rows must not regress `hi` — it would
+ ;; slide further back on every run.
+ hi
+ (cond
+ (some? max-value) (let [parsed-max (parse-checkpoint-value base-type max-value)]
+ (if (and checkpoint-lo
+ (neg? (checkpoint-compare base-type parsed-max checkpoint-lo)))
+ checkpoint-lo
+ parsed-max))
+ checkpoint-lo checkpoint-lo)]
(cond-> {:column column
:checkpoint-filter-field-id checkpoint-filter-field-id
:lo (when lo {:value lo})
- :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)
- lo {:value lo})}
+ :hi (when (some? hi) {:value hi})}
(some? rows-available) (assoc :rows-available rows-available))))))
(defn preprocess-incremental-query
diff --git a/test/metabase/transforms/crud_test.clj b/test/metabase/transforms/crud_test.clj
new file mode 100644
index 00000000000..4e92d8b7cc6
--- /dev/null
+++ b/test/metabase/transforms/crud_test.clj
@@ -0,0 +1,52 @@
+(ns metabase.transforms.crud-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.test :as mt]
+ [metabase.transforms.crud :as transforms.crud]))
+
+(set! *warn-on-reflection* true)
+
+(defn- lookback-transform [field-id lookback]
+ {:source {:type "query"
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id field-id
+ :lookback lookback}}})
+
+(defn- validates?! [transform]
+ (transforms.crud/validate-lookback! transform)
+ true)
+
+(deftest validate-lookback!-test
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field {dt-field :id} {:table_id table-id :base_type :type/DateTime}
+ :model/Field {ts-field :id} {:table_id table-id :base_type :type/DateTimeWithTZ}
+ :model/Field {date-field :id} {:table_id table-id :base_type :type/Date}
+ :model/Field {time-field :id} {:table_id table-id :base_type :type/Time}
+ :model/Field {num-field :id} {:table_id table-id :base_type :type/BigInteger}
+ :model/Field {unix-field :id} {:table_id table-id
+ :base_type :type/BigInteger
+ :effective_type :type/Instant
+ :coercion_strategy :Coercion/UNIXSeconds->DateTime}]
+ (testing "datetime and timestamp checkpoint columns accept any unit"
+ (is (validates?! (lookback-transform dt-field {:value 4 :unit "hour"})))
+ (is (validates?! (lookback-transform ts-field {:value 4 :unit "minute"}))))
+ (testing "coerced unix-timestamp columns count as temporal via their effective type"
+ (is (validates?! (lookback-transform unix-field {:value 4 :unit "day"}))))
+ (testing "date-only checkpoint columns require day-or-coarser units"
+ (is (validates?! (lookback-transform date-field {:value 4 :unit "day"})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"days or a coarser unit"
+ (validates?! (lookback-transform date-field {:value 4 :unit "hour"})))))
+ (testing "time-only checkpoint columns are rejected"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"only supported for date or datetime"
+ (validates?! (lookback-transform time-field {:value 1 :unit "day"})))))
+ (testing "numeric checkpoint columns are rejected"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"only supported for date or datetime"
+ (validates?! (lookback-transform num-field {:value 4 :unit "day"})))))
+ (testing "no configured lookback is a no-op"
+ (is (validates?! (lookback-transform dt-field nil))))))
diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.clj
index b64834a4fdb..5486baf4175 100644
--- a/test/metabase/transforms/models_test.clj
+++ b/test/metabase/transforms/models_test.clj
@@ -357,6 +357,27 @@
(let [t (t2/select-one :model/Transform transform-id)]
(is (= "99" (:last_checkpoint_value t)))))))))
+(deftest lookback-round-trips-with-keyword-keys-test
+ (testing "the nested lookback map comes back from the app DB with keyword keys"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Transform {transform-id :id}
+ {:source {:type "query"
+ :query {:database db-id
+ :type "native"
+ :native {:query "SELECT 1"
+ :template-tags {}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id 100
+ :lookback {:value 4 :unit "day"}}}
+ :target {:type "table-incremental"
+ :schema "public"
+ :name "test_lookback"
+ :db_id db-id}}]
+ (is (= {:value 4 :unit "day"}
+ (get-in (t2/select-one :model/Transform transform-id)
+ [:source :source-incremental-strategy :lookback])))))))
+
(deftest schedules-for-transforms-test
(testing "returns the schedules of the active jobs that run a transform via shared tags"
(mt/with-temp [:model/TransformTag tag {}
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index 88ad39bcfa9..b8e22accdb5 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -5,7 +5,8 @@
[metabase.config.core :as config]
[metabase.sync.sync :as sync]
[metabase.test :as mt]
- [metabase.transforms-base.util :as transforms-base.u]))
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.util.date-2 :as u.date]))
(set! *warn-on-reflection* true)
@@ -100,6 +101,81 @@
(is (true? (transforms-base.u/full-incremental-run?
{:target {:type :table-incremental} :last_checkpoint_value nil})))))
+(deftest ^:parallel supported-checkpoint-column?-test
+ (testing "numeric and date/datetime columns are supported"
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Integer}))
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Float}))
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Date}))
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTime}))
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTimeWithTZ}))
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Instant})))
+ (testing "coerced columns count via their effective type"
+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/BigInteger
+ :effective-type :type/Instant})))
+ (testing "time-only and non-orderable columns are not supported"
+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Time})))
+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/TimeWithTZ})))
+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Text})))
+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Boolean})))))
+
+(deftest ^:parallel apply-lookback-test
+ (let [apply-lookback @#'transforms-base.u/apply-lookback]
+ (testing "temporal checkpoints are pushed back by value units"
+ (is (= (t/local-date-time 2026 1 27 21 0 4)
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4 :unit "day"})))
+ (is (= (t/local-date-time 2026 1 31 19 0 4)
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 2 :unit "hour"})))
+ (is (= (t/local-date 2026 1 24)
+ (apply-lookback (t/local-date 2026 1 31) {:base-type :type/Date} {:value 1 :unit "week"}))))
+ (testing "a coerced numeric column (e.g. a unix timestamp) is temporal via its effective type"
+ (is (= (u.date/parse "2026-01-30T21:00:04Z")
+ (apply-lookback (u.date/parse "2026-01-31T21:00:04Z")
+ {:base-type :type/BigInteger :effective-type :type/Instant}
+ {:value 1 :unit "day"}))))
+ (testing "a lookback on a non-temporal checkpoint throws a user-facing error"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"only supported for date or datetime"
+ (apply-lookback (biginteger 42) {:base-type :type/Integer} {:value 4 :unit "day"}))))
+ (testing "a lookback on a time-only checkpoint throws a user-facing error"
+ ;; time-only watermarks wrap at midnight, and day-based units would throw inside u.date/add
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"only supported for date or datetime"
+ (apply-lookback (t/local-time 21 0 4) {:base-type :type/Time} {:value 2 :unit "hour"})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"only supported for date or datetime"
+ (apply-lookback (t/offset-time 21 0 4) {:base-type :type/TimeWithTZ} {:value 1 :unit "day"}))))
+ (testing "a lookback without a unit throws a user-facing error"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"requires a unit"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4}))))
+ (testing "a lookback with a non-positive value throws instead of silently skipping data"
+ ;; a negative window would move `lo` forward past the watermark, permanently skipping rows
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"positive integer value"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 0 :unit "day"})))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"positive integer value"
+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value -4 :unit "day"}))))))
+
+(deftest ^:parallel checkpoint-compare-test
+ (let [checkpoint-compare @#'transforms-base.u/checkpoint-compare]
+ (testing "numeric values compare numerically across numeric classes"
+ (is (neg? (checkpoint-compare :type/Integer (biginteger 41) 42)))
+ (is (zero? (checkpoint-compare :type/Integer (biginteger 42) 42))))
+ (testing "temporal values compare as instants even across temporal classes"
+ (is (neg? (checkpoint-compare :type/DateTimeWithTZ
+ (t/instant "2026-01-30T10:00:00Z")
+ (t/offset-date-time "2026-01-31T10:00:00Z"))))
+ (is (pos? (checkpoint-compare :type/DateTime
+ (t/local-date-time 2026 2 1 0 0 0)
+ (t/local-date-time 2026 1 31 0 0 0)))))))
+
(deftest ^:parallel checkpoint-span-attrs-test
(testing "nil source-range-params yields an empty attrs map"
(is (= {} (transforms-base.u/checkpoint-span-attrs nil))))
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 126546a6dad..dd5901f8d0e 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -1855,6 +1855,121 @@
(is (string? response))
(is (re-find #"requires a table variable" response))))))))))))
+(deftest lookback-validated-on-create-test
+ (mt/with-premium-features #{:transforms-basic :hosting}
+ (testing "POST /api/transform lookback window validation (GDGT-2868)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)
+ query (native-query-with-table-tag
+ "SELECT id, name, price, created_at FROM {{source_table}}"
+ (mt/id :transforms_products))
+ ts-field-id (mt/id :transforms_products :created_at)
+ num-field-id (mt/id :transforms_products :price)
+ payload (fn [table-name checkpoint-field-id lookback target-strategy]
+ {:name "Lookback Transform"
+ :source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id checkpoint-field-id
+ :lookback lookback}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy target-strategy}})
+ merge-target {:type "merge" :unique-key [{:name "id"}]}]
+ (testing "a lookback window without a unit is rejected by the schema"
+ (mt/user-http-request :crowberto :post 400 "transform"
+ (payload "lookback_no_unit" ts-field-id
+ {:value 4}
+ merge-target)))
+ (testing "a lookback window on a numeric checkpoint column is rejected"
+ (let [response (mt/user-http-request :crowberto :post 400 "transform"
+ (payload "lookback_num" num-field-id
+ {:value 4 :unit "day"}
+ merge-target))]
+ (is (string? response))
+ (is (re-find #"only supported for date or datetime" response))))
+ (testing "an unknown lookback unit is rejected by the schema"
+ (mt/user-http-request :crowberto :post 400 "transform"
+ (payload "lookback_bad_unit" ts-field-id
+ {:value 4 :unit "fortnight"}
+ merge-target)))
+ (testing "a valid temporal lookback with a merge target is accepted"
+ (with-transform-cleanup! [table-name "lookback_ok"]
+ (let [created (mt/user-http-request :crowberto :post 200 "transform"
+ (payload table-name ts-field-id
+ {:value 4 :unit "day"}
+ merge-target))]
+ (is (= {:value 4 :unit "day"}
+ (get-in created [:source :source-incremental-strategy :lookback]))))))
+ (testing "a valid temporal lookback with an append target is accepted"
+ (with-transform-cleanup! [table-name "lookback_append_ok"]
+ (let [created (mt/user-http-request :crowberto :post 200 "transform"
+ (payload table-name ts-field-id
+ {:value 4 :unit "day"}
+ {:type "append"}))]
+ (is (= {:value 4 :unit "day"}
+ (get-in created [:source :source-incremental-strategy :lookback]))))))))))))
+
+(deftest lookback-validated-on-update-test
+ (mt/with-premium-features #{:transforms-basic :hosting}
+ (testing "PUT /api/transform lookback window validation (GDGT-2868)"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (let [schema (get-test-schema)
+ query (native-query-with-table-tag
+ "SELECT id, name, created_at FROM {{source_table}}"
+ (mt/id :transforms_products))
+ ts-field-id (mt/id :transforms_products :created_at)
+ num-field-id (mt/id :transforms_products :price)]
+ (with-transform-cleanup! [table-name "lookback_update"]
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Append Incremental"
+ :source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "append"}}}]
+ (testing "adding a lookback to an existing incremental transform is accepted"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (format "transform/%d" transform-id)
+ {:source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id
+ :lookback {:value 4 :unit "day"}}}})]
+ (is (= {:value 4 :unit "day"}
+ (get-in updated [:source :source-incremental-strategy :lookback])))))
+ (testing "switching the checkpoint to a numeric column while a lookback is set is rejected"
+ (let [response (mt/user-http-request :crowberto :put 400
+ (format "transform/%d" transform-id)
+ {:source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id num-field-id
+ :lookback {:value 4 :unit "day"}}}})]
+ (is (string? response))
+ (is (re-find #"only supported for date or datetime" response))))
+ (testing "adding a lookback together with a switch to the merge strategy is accepted"
+ (let [updated (mt/user-http-request :crowberto :put 200
+ (format "transform/%d" transform-id)
+ {:source {:type "query"
+ :query query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id
+ :lookback {:value 4 :unit "day"}}}
+ :target {:type "table-incremental"
+ :schema schema
+ :name table-name
+ :target-incremental-strategy {:type "merge"
+ :unique-key [{:name "id"}]}}})]
+ (is (= {:value 4 :unit "day"}
+ (get-in updated [:source :source-incremental-strategy :lookback])))))))))))))
+
(deftest search-filters-transform-source-types-test
(mt/with-premium-features #{:transforms-basic :hosting}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)