The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
support lookback windows for incremental transforms (#78212)
Nicola Mometto04407a3c3520
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.cljindex 6a0a5cabf45..76cf0f27dcc 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj@@ -3,6 +3,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [java-time.api :as t][metabase.driver :as driver][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql.util :as sql.u]@@ -10,6 +11,7 @@[metabase.test :as mt][metabase.transforms.execute :as transforms.execute][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]+ [metabase.util.date-2 :as u.date][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -37,6 +39,28 @@[2 "created" #t "2026-01-31T21:00:04"][3 "created" #t "2026-01-31T21:00:04"]]]])+(mt/defdataset merge-lookback-data+ "An event-log dataset with a timestamp checkpoint column, for the lookback-window test."+ [["order_status_lb"+ [{:field-name "order_id" :base-type :type/Integer}+ {:field-name "status" :base-type :type/Text}+ {:field-name "changed_at" :base-type :type/DateTime}]+ [[1 "created" #t "2026-01-30T10:00:00"]+ [2 "created" #t "2026-01-31T21:00:04"]+ [3 "created" #t "2026-01-31T21:00:04"]]]])++;; A unix-timestamp (epoch seconds) checkpoint column: numeric in storage, temporal via its+;; coercion strategy / effective type (set by the test after sync).+(mt/defdataset merge-unix-ts-data+ "An event-log dataset whose checkpoint column is a unix-timestamp integer."+ [["order_status_unix"+ [{:field-name "order_id" :base-type :type/Integer}+ {:field-name "status" :base-type :type/Text}+ {:field-name "changed_at" :base-type :type/BigInteger}]+ [[1 "created" 1000000]+ [2 "created" 2000000]+ [3 "created" 2000000]]]])+;; Two key columns (order_id, region) for testing a composite merge key.(mt/defdataset merge-composite-test"An event log keyed on (order_id, region), for testing a two-column merge key."@@ -218,6 +242,185 @@(is (= expected (:field-id (first unique-key)))"field-id resolved to the target's order_id field")))))))))+(deftest merge-lookback-test+ (testing "a lookback window re-reads rows behind the checkpoint so late-arriving data is upserted (GDGT-2868)"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-lookback-data+ (with-transform-cleanup! [target-table {:type :table+ :name "merge_lookback"+ :schema (t2/select-one-fn :schema :model/Table+ (mt/id :order_status_lb))+ :database (mt/id)}]+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))+ ts-field-id (t2/select-one-pk :model/Field :name "changed_at"+ :table_id (mt/id :order_status_lb))+ source-query {:database (mt/id)+ :type :native+ :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"+ (q "order_id") (q "status") (q "t"))+ :template-tags {"t" {:id "t"+ :name "t"+ :display-name "T"+ :type "table"+ :table-id (mt/id :order_status_lb)+ :required true}}}}+ payload {:name "Lookback Merge"+ :source_database_id (mt/id)+ :source {:type "query"+ :query source-query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id+ :lookback {:value 1 :unit "day"}}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "merge"+ :unique-key [{:name "order_id"}]}})}+ insert! (fn [rows]+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table+ (mt/id :order_status_lb))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ values (str/join ", " (map (fn [[oid st ts]]+ (format "(%d, '%s', '%s')" oid st ts))+ rows))+ sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema tbl)+ (str/join "," (map q ["order_id" "status" "changed_at"]))+ values)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))+ read-target (fn []+ (let [sql (format "SELECT %s, %s FROM %s"+ (q "order_id") (q "status")+ (sql.u/quote-name driver/*driver* :table+ (:schema target-table) (:name target-table)))+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})+ :data :rows)]+ {:by-order (into {} (map (fn [[oid st]] [(long oid) st])) rows)+ :count (count rows)}))+ delete-order! (fn [order-id]+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table+ (mt/id :order_status_lb))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ sql (format "DELETE FROM %s WHERE %s = %d"+ (sql.u/quote-name driver/*driver* :table schema tbl)+ (q "order_id") order-id)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))+ watermark (fn [id] (t2/select-one-fn :last_checkpoint_value :model/Transform id))]+ (mt/with-temp [:model/Transform transform payload]+ (testing "first run builds the full target and records the watermark"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (is (= {:by-order {1 "created", 2 "created", 3 "created"} :count 3}+ (read-target)))+ (is (some? (watermark (:id transform)))))+ (testing "a late-arriving row older than the watermark but inside the lookback window is picked up"+ ;; watermark after run 1 is 2026-01-31T21:00:04; a 1-day lookback rescans from+ ;; 2026-01-30T21:00:04. The late row (order 1, 01-31T00:00) is behind the watermark but+ ;; inside the window; order 1's original row (01-30T10:00) is outside it.+ (let [wm-before (watermark (:id transform))]+ (insert! [[1 "shipped" "2026-01-31T00:00:00"] [4 "created" "2026-02-01T10:00:00"]])+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}+ (read-target))+ "late order 1 upserted, order 4 inserted, re-read rows not duplicated")+ (testing "the run's recorded lo is the stored watermark pushed back by the lookback"+ (let [run (t2/select-one :model/TransformRun :transform_id (:id transform)+ {:order-by [[:id :desc]]})]+ (is (= (u.date/format (t/minus (u.date/parse wm-before) (t/days 1)))+ (:checkpoint_lo_value run)))))))+ (testing "a run with no new rows leaves the target and the watermark unchanged"+ (let [before (watermark (:id transform))]+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}+ (read-target)))+ (is (= before (watermark (:id transform)))+ "the watermark must not slide backwards by the lookback window")))+ (testing "deleting the watermark row from the source does not regress the watermark"+ ;; With order 4 (the row that set the watermark, 02-01T10:00) deleted, max() over+ ;; the lookback window is orders 2/3 at 01-31T21:00:04 — behind the stored+ ;; watermark. The new-watermark clamp must keep the old value rather than let it+ ;; slide back (see get-source-range-params' `hi`).+ (let [before (watermark (:id transform))]+ (delete-order! 4)+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= before (watermark (:id transform)))+ "the watermark must not regress when the max row disappears from the source")+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}+ (read-target))+ "merge does not propagate source deletes; re-read rows upsert idempotently")))))))))))++(deftest merge-lookback-unix-timestamp-test+ (testing "a lookback works on a unix-timestamp checkpoint column via its effective type (GDGT-2868)"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-unix-ts-data+ (with-transform-cleanup! [target-table {:type :table+ :name "merge_lookback_unix"+ :schema (t2/select-one-fn :schema :model/Table+ (mt/id :order_status_unix))+ :database (mt/id)}]+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))+ ts-field-id (mt/id :order_status_unix :changed_at)+ _ (t2/update! :model/Field ts-field-id+ {:coercion_strategy :Coercion/UNIXSeconds->DateTime+ :effective_type :type/Instant})+ payload {:name "Unix Timestamp Lookback Merge"+ :source_database_id (mt/id)+ :source {:type "query"+ :query {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :order_status_unix)}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id+ :lookback {:value 1 :unit "day"}}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "merge"+ :unique-key [{:name "order_id"}]}})}+ insert! (fn [rows]+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table+ (mt/id :order_status_unix))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ values (str/join ", " (map (fn [[oid st ts]]+ (format "(%d, '%s', %d)" oid st ts))+ rows))+ sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema tbl)+ (str/join "," (map q ["order_id" "status" "changed_at"]))+ values)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))+ read-target (fn []+ (let [sql (format "SELECT %s, %s FROM %s"+ (q "order_id") (q "status")+ (sql.u/quote-name driver/*driver* :table+ (:schema target-table) (:name target-table)))+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})+ :data :rows)]+ {:by-order (into {} (map (fn [[oid st]] [(long oid) st])) rows)+ :count (count rows)}))]+ (mt/with-temp [:model/Transform transform payload]+ (testing "first run builds the full target and stores a temporal watermark"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (is (= {:by-order {1 "created", 2 "created", 3 "created"} :count 3}+ (read-target)))+ (let [wm (t2/select-one-fn :last_checkpoint_value :model/Transform (:id transform))]+ (is (re-find #"T" (str wm))+ "the watermark encodes as a temporal value, not the raw epoch number")))+ (testing "a late-arriving epoch inside the lookback window is upserted on rerun"+ ;; watermark = epoch 2000000; a 1-day lookback rescans from 1913600. Order 4's+ ;; epoch 1950000 is behind the watermark but inside the window; order 1's+ ;; original row (1000000) is outside it.+ (insert! [[1 "shipped" 2100000] [4 "created" 1950000]])+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {:by-order {1 "shipped", 2 "created", 3 "created", 4 "created"} :count 4}+ (read-target))+ "late order 4 picked up, order 1 upserted, re-read rows not duplicated"))))))))))+(deftest merge-composite-key-test(testing "a merge transform can upsert on a two-column key"(mt/test-drivers (test-drivers)diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 6fb5dfab6dd..86eaf7d0756 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -3,7 +3,7 @@ import type { DatabaseId } from "./database";import type { RowValue } from "./dataset";import type { RequestableIndexes } from "./index-manager";import type { PaginationRequest, PaginationResponse } from "./pagination";-import type { DatasetQuery, JoinStrategy } from "./query";+import type { DatasetQuery, DateTimeAbsoluteUnit, JoinStrategy } from "./query";import type { ScheduleDisplayType } from "./settings";import type { SortDirection } from "./sorting";import type { ConcreteTableId, SchemaName, Table } from "./table";@@ -97,9 +97,18 @@ export type PythonTransformTableEntry = {export type PythonTransformTableAliases = PythonTransformTableEntry[];+export type LookbackUnit = "millisecond" | "second" | DateTimeAbsoluteUnit;++// Only supported for temporal checkpoint columns.+export type TransformLookback = {+ value: number;+ unit: LookbackUnit;+};+export type TransformSourceCheckpointStrategy = {type: "checkpoint";"checkpoint-filter-field-id": number;+ lookback?: TransformLookback | null;};export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;diff --git a/frontend/src/metabase/api/field.ts b/frontend/src/metabase/api/field.tsindex a859d7a5997..0e560e2c964 100644--- a/frontend/src/metabase/api/field.ts+++ b/frontend/src/metabase/api/field.ts@@ -166,6 +166,7 @@ export const fieldApi = Api.injectEndpoints({export const {useGetFieldQuery,+ useLazyGetFieldQuery,useGetFieldTableIdsQuery,useGetFieldValuesQuery,useGetRemappedFieldValueQuery,diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsxindex 77a2058d135..d98dfa698ae 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx@@ -28,6 +28,7 @@ import {NativeQueryTableTagFieldSelect,PythonKeysetColumnSelect,} from "./KeysetColumnSelect";+import { LookbackField } from "./LookbackField";import { UniqueKeyField } from "./UniqueKeyField";import type { IncrementalSettingsFormValues } from "./form";import { useHasCheckpointOptions } from "./useHasCheckpointOptions";@@ -295,6 +296,7 @@ function SourceStrategyFields({disabled={readOnly}/>)}+ <LookbackField readOnly={readOnly} /></>)}</>diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsxindex b45618cb29b..ea13ac9c1bc 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx@@ -5,20 +5,23 @@ import {type DataAttributes,type InputDescriptionProps,Loader,- type SelectOption,} from "metabase/ui";import * as Lib from "metabase-lib";import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";+import {+ type CheckpointFieldOption,+ useClearUnsupportedLookback,+} from "../useClearUnsupportedLookback";/*** Get filterable numeric/temporal field options from the source table of a query.- * Returns SelectOption[] with field IDs as string values.+ * Returns CheckpointFieldOption[] with field IDs as string values.*/export function getSourceFieldOptions(query: Lib.Query,opts?: { labelPrefix?: string; seenFieldIds?: Set<number> },-): Array<SelectOption> {+): Array<CheckpointFieldOption> {// Stage 0 is the source stage (raw table data). Keyset pagination requires columns from the// actual source table; later stages may have aggregations, joins, or expressions instead.const stageIndex = 0;@@ -36,10 +39,14 @@ export function getSourceFieldOptions(const sourceColumns = Lib.getColumnsFromColumnGroup(sourceGroup);const seenFieldIds = opts?.seenFieldIds ?? new Set<number>();- const options: Array<SelectOption> = [];+ const options: Array<CheckpointFieldOption> = [];for (const column of sourceColumns) {- if (!Lib.isNumeric(column) && !Lib.isTemporal(column)) {+ // Numeric or date/datetime only — time-only columns are excluded since their+ // watermarks wrap at midnight+ const isSupported =+ (Lib.isNumeric(column) || Lib.isTemporal(column)) && !Lib.isTime(column);+ if (!isSupported) {continue;}@@ -55,7 +62,11 @@ export function getSourceFieldOptions(? `${opts.labelPrefix}: ${columnInfo.displayName}`: columnInfo.longDisplayName;- options.push({ value: String(fieldId), label });+ options.push({+ value: String(fieldId),+ label,+ supportsLookback: Lib.isDateOrDateTime(column),+ });}return options;@@ -82,7 +93,7 @@ export function KeysetColumnSelect({disabled,isLoading,}: KeysetColumnSelectProps) {- const columnOptions = useMemo((): Array<SelectOption> => {+ const columnOptions = useMemo((): Array<CheckpointFieldOption> => {if (!query) {return [];}@@ -104,6 +115,8 @@ export function KeysetColumnSelect({disabled: disabled || isLoading,});+ useClearUnsupportedLookback({ name, options: columnOptions });+return (<FormSelectname={name}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.tsnew file mode 100644index 00000000000..2112d32280b--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.unit.spec.ts@@ -0,0 +1,21 @@+import * as Lib from "metabase-lib";+import { SAMPLE_PROVIDER } from "metabase-lib/test-helpers";+import { ORDERS, ORDERS_ID } from "metabase-types/api/mocks/presets";++import { getSourceFieldOptions } from "./KeysetColumnSelect";++const query = Lib.createTestQuery(SAMPLE_PROVIDER, {+ stages: [{ source: { type: "table", id: ORDERS_ID } }],+});++describe("getSourceFieldOptions", () => {+ it("marks only date/datetime columns as supporting a lookback", () => {+ const options = getSourceFieldOptions(query);+ const supportsLookbackByFieldId = new Map(+ options.map(({ value, supportsLookback }) => [value, supportsLookback]),+ );++ expect(supportsLookbackByFieldId.get(String(ORDERS.CREATED_AT))).toBe(true);+ expect(supportsLookbackByFieldId.get(String(ORDERS.TOTAL))).toBe(false);+ });+});diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsxindex ec51c74de3f..ce215abacf4 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx@@ -12,6 +12,7 @@ import {import type * as Lib from "metabase-lib";import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";+import { useClearUnsupportedLookback } from "../useClearUnsupportedLookback";import { useNativeCheckpointFieldOptions } from "../useNativeCheckpointFieldOptions";type NativeQueryTableTagFieldSelectProps = {@@ -52,6 +53,8 @@ export function NativeQueryTableTagFieldSelect({disabled: disabled || isLoading,});+ useClearUnsupportedLookback({ name, options: fieldOptions });+if (noQueryMessage) {return (<Alert size="compact" icon={<Icon name="warning" />} color="warning">diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsxnew file mode 100644index 00000000000..fd630cc393f--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/LookbackField.tsx@@ -0,0 +1,102 @@+import { useFormikContext } from "formik";+import { useEffect } from "react";+import { t } from "ttag";++import { skipToken, useGetFieldQuery } from "metabase/api";+import { FormField, FormNumberInput, FormSelect } from "metabase/forms";+import { Group } from "metabase/ui";+import { isDateWithoutTime } from "metabase-lib/v1/types/utils/isa";+import type { LookbackUnit } from "metabase-types/api";++import {+ type IncrementalSettingsFormValues,+ fieldSupportsLookback,+} from "./form";++const SUB_DAY_UNITS: ReadonlySet<LookbackUnit> = new Set([+ "millisecond",+ "second",+ "minute",+ "hour",+]);++const getUnitOptions = (dateOnly: boolean) => {+ const options: { value: LookbackUnit; label: string }[] = [+ { value: "minute", label: t`minutes` },+ { value: "hour", label: t`hours` },+ { value: "day", label: t`days` },+ { value: "week", label: t`weeks` },+ { value: "month", label: t`months` },+ { value: "year", label: t`years` },+ ];+ return dateOnly+ ? options.filter(({ value }) => !SUB_DAY_UNITS.has(value))+ : options;+};++// The lookback window input: a number plus a unit. Only shown for temporal checkpoint columns —+// the only kind that supports a lookback.+export function LookbackField({ readOnly }: { readOnly?: boolean }) {+ const { values, setFieldValue } =+ useFormikContext<IncrementalSettingsFormValues>();+ const fieldId = values.checkpointFilterFieldId;+ const { data: field } = useGetFieldQuery(+ fieldId != null ? { id: Number(fieldId) } : skipToken,+ );++ const isSupported = fieldSupportsLookback(field);+ const isDateOnly = isDateWithoutTime(field);++ // Keep a configured lookback consistent after a checkpoint-field change: clear it when the new+ // column doesn't support one (hiding the input alone would still submit the stale value), and+ // snap sub-day units back to days on date-only columns. Guarded on a set value — a write here+ // counts as a form change and triggers an inline save.+ useEffect(() => {+ if (field == null || values.lookbackValue == null) {+ return;+ }+ if (!isSupported) {+ setFieldValue("lookbackValue", null);+ } else if (isDateOnly && SUB_DAY_UNITS.has(values.lookbackUnit)) {+ setFieldValue("lookbackUnit", "day");+ }+ }, [+ field,+ isSupported,+ isDateOnly,+ values.lookbackValue,+ values.lookbackUnit,+ setFieldValue,+ ]);++ if (fieldId == null || !isSupported) {+ return null;+ }++ return (+ <FormField+ title={t`Lookback window`}+ description={t`Optional. Re-process this much already-seen data on each run, to catch late-arriving rows.`}+ maw="24rem"+ >+ <Group gap="sm" wrap="nowrap">+ <FormNumberInput+ name="lookbackValue"+ nullable+ min={1}+ placeholder={t`e.g. 4`}+ aria-label={t`Lookback amount`}+ disabled={readOnly}+ w="10rem"+ />+ <FormSelect+ name="lookbackUnit"+ aria-label={t`Lookback unit`}+ data={getUnitOptions(isDateOnly)}+ disabled={readOnly}+ w="9rem"+ />+ </Group>+ </FormField>+ );+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.tsindex 0a9c68d969f..8fd68574a4d 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts@@ -1,13 +1,26 @@import * as Yup from "yup";import * as Errors from "metabase/utils/errors";+import {+ type FieldTypeInfo,+ isDate,+ isTime,+} from "metabase-lib/v1/types/utils/isa";import type {+ LookbackUnit,Transform,TransformSource,TransformTarget,UpdateTransformRequest,} from "metabase-types/api";+// Same rule as the BE's `date-or-datetime?`: temporal but not time-only (time watermarks wrap at+// midnight). The predicates check effective_type first, so coerced columns (e.g. unix+// timestamps) count as temporal.+export const fieldSupportsLookback = (+ field: FieldTypeInfo | null | undefined,+): boolean => isDate(field) && !isTime(field);+export type IncrementalSettingsFormValues = {incremental: boolean;sourceStrategy: "checkpoint";@@ -15,6 +28,10 @@ export type IncrementalSettingsFormValues = {// Comma-separated target column names. Empty → append. Non-empty → the target is upserted// (merge/restate) on these columns instead of appended.uniqueKey: string;+ // Lookback window: re-read this much data behind the checkpoint on every run. Null → none.+ // Only supported for temporal checkpoint columns.+ lookbackValue: number | null;+ lookbackUnit: LookbackUnit;};export const VALIDATION_SCHEMA = Yup.object({@@ -30,6 +47,8 @@ export const VALIDATION_SCHEMA = Yup.object({otherwise: (schema) => schema.nullable().defined(),}),uniqueKey: Yup.string().default(""),+ lookbackValue: Yup.number().nullable().positive().integer().default(null),+ lookbackUnit: Yup.mixed<LookbackUnit>().default("day"),});export const getInitialValues = (@@ -39,6 +58,8 @@ export const getInitialValues = (sourceStrategy: "checkpoint",checkpointFilterFieldId: null,uniqueKey: "",+ lookbackValue: null,+ lookbackUnit: "day",...defaults,});@@ -60,6 +81,8 @@ export const getIncrementalSettingsFromTransform = (? (strategy["checkpoint-filter-field-id"] ?? null): null;+ const lookback = strategy?.type === "checkpoint" ? strategy.lookback : null;+const targetStrategy =transform.target.type === "table-incremental"? transform.target["target-incremental-strategy"]@@ -72,6 +95,8 @@ export const getIncrementalSettingsFromTransform = (isIncremental && checkpointFilterFieldId != null? String(checkpointFilterFieldId): null,+ lookbackValue: lookback?.value ?? null,+ lookbackUnit: lookback?.unit ?? "day",uniqueKey:targetStrategy?.type === "merge"? targetStrategy["unique-key"]@@ -116,6 +141,13 @@ export const buildIncrementalSource = ("checkpoint-filter-field-id": Number(formValues.checkpointFilterFieldId,),+ lookback:+ formValues.lookbackValue !== null+ ? {+ value: formValues.lookbackValue,+ unit: formValues.lookbackUnit,+ }+ : undefined,}: undefined,};diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.tsindex 5a1939a746c..d036d78af39 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts@@ -1,8 +1,12 @@-import { createMockTransform } from "metabase-types/api/mocks";+import {+ createMockTransform,+ createMockTransformSource,+} from "metabase-types/api/mocks";import {type IncrementalSettingsFormValues,VALIDATION_SCHEMA,+ buildIncrementalSource,buildIncrementalTarget,getIncrementalSettingsFromTransform,} from "./form";@@ -14,6 +18,8 @@ const getValues = (sourceStrategy: "checkpoint",checkpointFilterFieldId: "123",uniqueKey: "",+ lookbackValue: null,+ lookbackUnit: "day",...values,});@@ -92,6 +98,63 @@ describe("buildIncrementalTarget (merge)", () => {});});+describe("buildIncrementalSource (lookback)", () => {+ it("omits lookback when no value is set", () => {+ const source = buildIncrementalSource(+ createMockTransformSource(),+ getValues(),+ );+ expect(source["source-incremental-strategy"]?.lookback).toBeUndefined();+ });++ it("includes value and unit when a lookback is set", () => {+ const source = buildIncrementalSource(+ createMockTransformSource(),+ getValues({ lookbackValue: 4, lookbackUnit: "week" }),+ );+ expect(source["source-incremental-strategy"]?.lookback).toEqual({+ value: 4,+ unit: "week",+ });+ });+});++describe("getIncrementalSettingsFromTransform (lookback)", () => {+ it("reads the lookback into form values", () => {+ const transform = createMockTransform({+ source: {+ ...createMockTransformSource(),+ "source-incremental-strategy": {+ type: "checkpoint",+ "checkpoint-filter-field-id": 123,+ lookback: { value: 4, unit: "day" },+ },+ },+ target: {+ type: "table-incremental",+ name: "t",+ schema: "public",+ database: 1,+ "target-incremental-strategy": { type: "append" },+ },+ });+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({+ incremental: true,+ checkpointFilterFieldId: "123",+ lookbackValue: 4,+ lookbackUnit: "day",+ });+ });++ it("defaults to no lookback when the strategy has none", () => {+ const transform = createMockTransform();+ expect(getIncrementalSettingsFromTransform(transform)).toMatchObject({+ lookbackValue: null,+ lookbackUnit: "day",+ });+ });+});+describe("getIncrementalSettingsFromTransform (merge)", () => {it("reads the merge unique key into a comma-separated string", () => {const transform = createMockTransform({diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.tsnew file mode 100644index 00000000000..e062736505b--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.ts@@ -0,0 +1,32 @@+import { useField, useFormikContext } from "formik";+import { useEffect } from "react";++import type { SelectOption } from "metabase/ui";++import type { IncrementalSettingsFormValues } from "./form";++export type CheckpointFieldOption = SelectOption & {+ supportsLookback: boolean;+};++// Clears a configured lookback when the selected checkpoint column doesn't support one+// (only date/datetime columns do), so a column change never sends a stale lookback to the API.+export function useClearUnsupportedLookback({+ name,+ options,+}: {+ name: string;+ options: CheckpointFieldOption[];+}) {+ const [{ value: selectedFieldId }] = useField<string | null>(name);+ const { values, setFieldValue } =+ useFormikContext<IncrementalSettingsFormValues>();+ const { lookbackValue } = values;++ useEffect(() => {+ const option = options.find(({ value }) => value === selectedFieldId);+ if (option != null && !option.supportsLookback && lookbackValue != null) {+ setFieldValue("lookbackValue", null);+ }+ }, [options, selectedFieldId, lookbackValue, setFieldValue]);+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsxnew file mode 100644index 00000000000..7c4d0436184--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useClearUnsupportedLookback.unit.spec.tsx@@ -0,0 +1,64 @@+import { renderHook, waitFor } from "@testing-library/react";+import { useFormikContext } from "formik";+import type { ReactNode } from "react";++import { FormProvider } from "metabase/forms";++import { type IncrementalSettingsFormValues, getInitialValues } from "./form";+import {+ type CheckpointFieldOption,+ useClearUnsupportedLookback,+} from "./useClearUnsupportedLookback";++const OPTIONS: CheckpointFieldOption[] = [+ { value: "1", label: "Total", supportsLookback: false },+ { value: "2", label: "Created At", supportsLookback: true },+];++function setup({+ checkpointFilterFieldId,+ options = OPTIONS,+}: {+ checkpointFilterFieldId: string | null;+ options?: CheckpointFieldOption[];+}) {+ const initialValues = getInitialValues({+ incremental: true,+ checkpointFilterFieldId,+ lookbackValue: 4,+ lookbackUnit: "hour",+ });++ const wrapper = ({ children }: { children: ReactNode }) => (+ <FormProvider initialValues={initialValues} onSubmit={jest.fn()}>+ {children}+ </FormProvider>+ );++ return renderHook(+ () => {+ useClearUnsupportedLookback({ name: "checkpointFilterFieldId", options });+ return useFormikContext<IncrementalSettingsFormValues>();+ },+ { wrapper },+ );+}++describe("useClearUnsupportedLookback", () => {+ it("clears the lookback when the selected column doesn't support one", async () => {+ const { result } = setup({ checkpointFilterFieldId: "1" });+ await waitFor(() => {+ expect(result.current.values.lookbackValue).toBeNull();+ });+ });++ it("keeps the lookback when the selected column supports one", () => {+ const { result } = setup({ checkpointFilterFieldId: "2" });+ expect(result.current.values.lookbackValue).toBe(4);+ });++ it("keeps the lookback when the selected column isn't among the options", () => {+ const { result } = setup({ checkpointFilterFieldId: "99" });+ expect(result.current.values.lookbackValue).toBe(4);+ });+});diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.tsindex ade88e1237f..a78b35f8692 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts@@ -2,11 +2,11 @@ import { useMemo } from "react";import { useSelector } from "metabase/redux";import { getMetadata } from "metabase/selectors/metadata";-import type { SelectOption } from "metabase/ui";import * as Lib from "metabase-lib";import { isConcreteTableId } from "metabase-types/api/table";import { getSourceFieldOptions } from "./KeysetColumnSelect/KeysetColumnSelect";+import type { CheckpointFieldOption } from "./useClearUnsupportedLookback";import { useTableQueryMetadataResults } from "./useTableQueryMetadataResults";export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {@@ -30,13 +30,13 @@ export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {const { tables, isLoading, hasError } =useTableQueryMetadataResults(tableIds);- const fieldOptions = useMemo((): Array<SelectOption> => {+ const fieldOptions = useMemo((): Array<CheckpointFieldOption> => {if (tables.length === 0) {return [];}try {- const allOptions: Array<SelectOption> = [];+ const allOptions: Array<CheckpointFieldOption> = [];const seenFieldIds = new Set<number>();const showTablePrefix = tables.length > 1;diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.tsindex 38fdb3a59ac..74e243b8908 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts@@ -1,26 +1,57 @@import { useMemo } from "react";-import { useUpdateTransformMutation } from "metabase/api";+import { useLazyGetFieldQuery, useUpdateTransformMutation } from "metabase/api";import type { Transform } from "metabase-types/api";import {type IncrementalSettingsFormValues,VALIDATION_SCHEMA,convertTransformFormToUpdateRequest,+ fieldSupportsLookback,getIncrementalSettingsFromTransform,} from "./form";export function useUpdateIncrementalSettings(transform: Transform) {const [updateTransform] = useUpdateTransformMutation();+ const [fetchField] = useLazyGetFieldQuery();const initialValues = useMemo(() => getIncrementalSettingsFromTransform(transform),[transform],);+ // The inline save can fire before the newly selected checkpoint field's metadata has loaded+ // (and before LookbackField's effect clears a now-unsupported lookback), so re-check against+ // the authoritative field data before building the request. Usually a cache hit.+ const dropUnsupportedLookback = async (+ values: IncrementalSettingsFormValues,+ ): Promise<IncrementalSettingsFormValues> => {+ if (+ values.lookbackValue == null ||+ values.checkpointFilterFieldId == null+ ) {+ return values;+ }+ try {+ const field = await fetchField(+ { id: Number(values.checkpointFilterFieldId) },+ true,+ ).unwrap();+ return fieldSupportsLookback(field)+ ? values+ : { ...values, lookbackValue: null };+ } catch {+ // let the save proceed; the BE validates anyway+ return values;+ }+ };+const updateIncrementalSettings = async (values: IncrementalSettingsFormValues,) => {- const requestData = convertTransformFormToUpdateRequest(transform, values);+ const requestData = convertTransformFormToUpdateRequest(+ transform,+ await dropUnsupportedLookback(values),+ );return await updateTransform(requestData).unwrap();};diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex f6db0aedcb9..e9e6b4b119e 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -9,6 +9,7 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.events.core :as events]+ [metabase.lib.types.isa :as lib.types.isa][metabase.models.interface :as mi][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.ordering :as transforms-base.ordering]@@ -63,6 +64,11 @@(api/check-400 (not (str/blank? (get-in transform [:target :schema])))(deferred-tru "A target schema is required for this database.")))))+(defn- field->column+ "Adapt a t2 Field row to the kebab-case column shape `lib.types.isa` predicates expect."+ [field]+ {:base-type (:base_type field), :effective-type (:effective_type field)})+(defn validate-incremental-column-type!"Validates that the checkpoint column for an incremental transform has a supported type.@@ -73,10 +79,25 @@(let [checkpoint-filter-field-id (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])field (t2/select-one :model/Field checkpoint-filter-field-id)](api/check-400 field (deferred-tru "Checkpoint field not found."))- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."- (:name field)- (pr-str (:base_type field)))))))+ (let [column (field->column field)]+ (api/check-400 (transforms-base.u/supported-checkpoint-column? column)+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."+ (:name field)+ (pr-str (lib.types.isa/column-type column))))))))++(defn validate-lookback!+ "Validates a configured lookback window: only date and datetime checkpoint columns support one+ (time-only columns wrap at midnight), and date-only columns need a day-or-coarser unit."+ [{:keys [source] :as transform}]+ (let [{:keys [lookback checkpoint-filter-field-id]} (:source-incremental-strategy source)]+ (when-let [{:keys [unit]} (and (transforms-base.u/checkpoint-source? transform) lookback)]+ (when-let [field (t2/select-one :model/Field checkpoint-filter-field-id)]+ (let [column (field->column field)]+ (api/check-400 (lib.types.isa/date-or-datetime? column)+ (deferred-tru "A lookback window is only supported for date or datetime checkpoint columns."))+ (when (lib.types.isa/date-without-time? column)+ (api/check-400 (contains? #{"day" "week" "month" "quarter" "year"} unit)+ (deferred-tru "A lookback window on a date checkpoint column must use days or a coarser unit."))))))))(defn validate-incremental-table-tag!"Reject a table-incremental native-query transform whose source query has no table template tag for@@ -142,6 +163,7 @@(validate-transform-query! body))(validate-target-schema! body)(validate-incremental-table-tag! body)+ (validate-lookback! body)(let [creator-id (or creator-id api/*current-user-id*)transform (t2/with-transaction [_](let [tag-ids (:tag_ids body)@@ -180,7 +202,8 @@(when (contains? body :source)(validate-incremental-column-type! new))(when (or (contains? body :source) (contains? body :target))- (validate-incremental-table-tag! new))+ (validate-incremental-table-tag! new)+ (validate-lookback! new))(when (transforms-base.u/query-transform? old)(validate-transform-query! new)(when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 36c36c99099..2eb7e88db24 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -111,7 +111,10 @@mi/json-out-without-keywordization(update-keys keyword)(m/update-existing :query (fn [q] (if (orphan-query? q) q (lib-be/normalize-query q))))- (m/update-existing :source-incremental-strategy #(update-keys % keyword))+ (m/update-existing :source-incremental-strategy+ (fn [strategy]+ (-> (update-keys strategy keyword)+ (m/update-existing :lookback #(some-> % (update-keys keyword))))))(m/update-existing :source-tables (fn [st] (mapv #(update-keys % keyword) st)))(m/update-existing :type keyword)))diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex 51a45bf63af..7b93e4a5346 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -4,13 +4,23 @@[metabase.lib.schema.id :as lib.schema.id][metabase.queries.schema :as queries.schema][metabase.transforms-base.util :as transforms-base.u]+ [metabase.util.date-2 :as u.date][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]))+(mr/def ::lookback+ "A lookback window: each run re-reads source rows up to `value` `unit`s behind the checkpoint,+ so late-arriving rows older than the watermark still get picked up. Only supported for+ temporal checkpoint columns."+ [:map+ [:value pos-int?]+ [:unit (into [:enum] (map name) (sort u.date/add-units))]])+(mr/def ::checkpoint-strategy[:map[:type [:= "checkpoint"]]- [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]])+ [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]+ [:lookback {:optional true} [:maybe ::lookback]]])(mr/def ::source-incremental-strategy[:multi {:dispatch :type}diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 79baec35366..d2b78ca6213 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -16,6 +16,7 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util][metabase.query-processor.compile :as qp.compile][metabase.query-processor.core :as qp]@@ -236,13 +237,15 @@;;; ------------------------------------------------- Incremental/Checkpoint Helpers --------------------------------------------------(defn supported-incremental-filter-type?- "Returns true if the given base-type is supported for incremental filtering.+(defn supported-checkpoint-column?+ "Returns true if `column` (or any map with `:base-type`/`:effective-type`) can be used for+ incremental checkpoint filtering.- We only support temporal (timestamp/tz) and numeric (int/float) types."- [base-type]- (or (isa? base-type :type/Temporal)- (isa? base-type :type/Number)))+ We support date/datetime and numeric (int/float) columns. Time-only columns are excluded:+ their watermarks wrap at midnight, so `>` comparisons against them are meaningless."+ [column]+ (or (lib.types.isa/date-or-datetime? column)+ (lib.types.isa/numeric? column)))(defn- encode-checkpoint-value [v](if (number? v)@@ -309,10 +312,66 @@:else (throw (ex-info (str "Unsupported checkpoint type: " (pr-str base-type)){:base-type base-type}))))-(defn- tag-checkpoint-value- "Wrap a raw checkpoint value from the QP into a map `{:value parsed}`."- [base-type raw-value]- {:value (parse-checkpoint-value base-type raw-value)})+(declare ->instant)++(defn- apply-lookback+ "Push a checkpoint lower bound back by the lookback window (`value` `unit`s). Only supported+ for date and datetime checkpoint columns: time-only columns wrap at midnight, so a window+ behind the watermark is meaningless (and day-based units don't apply to them)."+ [checkpoint column {:keys [value unit] :as lookback}]+ (let [invalid! (fn [msg] (throw (ex-info msg {:transform-message msg, :lookback lookback})))]+ (cond+ (not (lib.types.isa/date-or-datetime? column))+ (invalid! (i18n/tru "A lookback window is only supported for date or datetime checkpoint columns."))++ (nil? unit)+ (invalid! (i18n/tru "A lookback window requires a unit."))++ (not (pos-int? value))+ (invalid! (i18n/tru "A lookback window requires a positive integer value."))++ :else+ (u.date/add checkpoint (keyword unit) (- value)))))++(defn- checkpoint-compare+ "Compare two parsed checkpoint values. Temporal values compare as instants, since the stored+ watermark and a fresh QP max can parse to incomparable temporal classes."+ [base-type a b]+ (if (isa? base-type :type/Temporal)+ (compare (->instant a) (->instant b))+ (compare a b)))++(defn- validate-incremental-source!+ "Throws a user-facing error when an incremental transform can't compute its source range: a+ native query without a table variable to inject the filter into, or no checkpoint field+ selected."+ [{:keys [source] :as transform}]+ (when (incremental-target? transform)+ (when (and (native-query-transform? transform)+ (not (some (fn [tag] (#{:table "table"} (:type tag)))+ (lib/template-tags (:query source)))))+ (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "+ "Please add a table variable to the query and update the checkpoint field."))]+ (throw (ex-info msg {:transform-message msg}))))+ (when-not (get-in source [:source-incremental-strategy :checkpoint-filter-field-id])+ (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "+ "Please select a checkpoint field in the transform settings."))]+ (throw (ex-info msg {:transform-message msg}))))))++(defn- checkpoint-column+ "The checkpoint column fetched from `metadata-provider`, validated to exist, be active, and have+ a type supported for incremental filtering."+ [metadata-provider checkpoint-filter-field-id]+ (let [column (lib.metadata/field metadata-provider checkpoint-filter-field-id)]+ (when (or (nil? column) (not (:active column)))+ (throw (ex-info "Checkpoint field does not exist or is not active"+ {:checkpoint-filter-field-id checkpoint-filter-field-id})))+ (when-not (supported-checkpoint-column? column)+ (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type "+ (pr-str (lib.types.isa/column-type column)) ". "+ "Only numeric and temporal columns are supported for incremental filtering.")+ {:column column})))+ column))(defn- inject-filters-into-table-tag"Inject `:source-filters` into the table template tag matching the checkpoint field's table.@@ -353,36 +412,20 @@:hi values in the source table must be <= this :value.:rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."[{:keys [source] :as transform}]- (let [{:keys [source-incremental-strategy]} source- {:keys [checkpoint-filter-field-id]} source-incremental-strategy]- (when (and (incremental-target? transform)- (native-query-transform? transform)- (not (some (fn [tag] (#{:table "table"} (:type tag)))- (lib/template-tags (:query source)))))- (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "- "Please add a table variable to the query and update the checkpoint field."))]- (throw (ex-info msg {:transform-message msg}))))- (when (and (incremental-target? transform)- (not checkpoint-filter-field-id))- (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "- "Please select a checkpoint field in the transform settings."))]- (throw (ex-info msg {:transform-message msg}))))+ (let [{:keys [checkpoint-filter-field-id lookback]} (:source-incremental-strategy source)]+ (validate-incremental-source! transform)(when checkpoint-filter-field-id(let [{:keys [last_checkpoint_value]} (cond-> transform(full-incremental-run? transform)(assoc :last_checkpoint_value nil))db-id (transforms-base.i/target-db-id transform)metadata-provider (lib-be/application-database-metadata-provider db-id)- column (lib.metadata/field metadata-provider checkpoint-filter-field-id)- _ (when (or (nil? column) (not (:active column)))- (throw (ex-info "Checkpoint field does not exist or is not active"- {:checkpoint-filter-field-id checkpoint-filter-field-id})))- _ (when-not (supported-incremental-filter-type? (:base-type column))- (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type " (pr-str (:base-type column)) ". "- "Only numeric and temporal columns are supported for incremental filtering.")- {:column column})))- base-type (:base-type column)- lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))+ column (checkpoint-column metadata-provider checkpoint-filter-field-id)+ base-type (lib.types.isa/column-type column)+ ;; `checkpoint-lo` is the stored watermark; `lo` is the scan bound, pushed back by any lookback.+ checkpoint-lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))+ lo (cond-> checkpoint-lo+ (and checkpoint-lo lookback) (apply-lookback column lookback));; Combine max + count in one scan: avoids a second round-trip and pins both;; numbers to the same point-in-time view of the source.@@ -400,12 +443,23 @@;; instead of NULL for `max()` over an empty relation when the column is non-nullable. Only;; trust the max when the count from the same scan says there were rows, otherwise the;; watermark would silently regress and the next run would reprocess already-seen rows.- [(when-not (and cv (zero? cv)) mv) cv])]+ [(when-not (and cv (zero? cv)) mv) cv])++ ;; The new watermark, clamped to the stored one: with a lookback the scan starts behind+ ;; it, so an empty scan or a max() over only late rows must not regress `hi` — it would+ ;; slide further back on every run.+ hi+ (cond+ (some? max-value) (let [parsed-max (parse-checkpoint-value base-type max-value)]+ (if (and checkpoint-lo+ (neg? (checkpoint-compare base-type parsed-max checkpoint-lo)))+ checkpoint-lo+ parsed-max))+ checkpoint-lo checkpoint-lo)](cond-> {:column column:checkpoint-filter-field-id checkpoint-filter-field-id:lo (when lo {:value lo})- :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)- lo {:value lo})}+ :hi (when (some? hi) {:value hi})}(some? rows-available) (assoc :rows-available rows-available))))))(defn preprocess-incremental-querydiff --git a/test/metabase/transforms/crud_test.clj b/test/metabase/transforms/crud_test.cljnew file mode 100644index 00000000000..4e92d8b7cc6--- /dev/null+++ b/test/metabase/transforms/crud_test.clj@@ -0,0 +1,52 @@+(ns metabase.transforms.crud-test+ (:require+ [clojure.test :refer :all]+ [metabase.test :as mt]+ [metabase.transforms.crud :as transforms.crud]))++(set! *warn-on-reflection* true)++(defn- lookback-transform [field-id lookback]+ {:source {:type "query"+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id field-id+ :lookback lookback}}})++(defn- validates?! [transform]+ (transforms.crud/validate-lookback! transform)+ true)++(deftest validate-lookback!-test+ (mt/with-temp [:model/Table {table-id :id} {}+ :model/Field {dt-field :id} {:table_id table-id :base_type :type/DateTime}+ :model/Field {ts-field :id} {:table_id table-id :base_type :type/DateTimeWithTZ}+ :model/Field {date-field :id} {:table_id table-id :base_type :type/Date}+ :model/Field {time-field :id} {:table_id table-id :base_type :type/Time}+ :model/Field {num-field :id} {:table_id table-id :base_type :type/BigInteger}+ :model/Field {unix-field :id} {:table_id table-id+ :base_type :type/BigInteger+ :effective_type :type/Instant+ :coercion_strategy :Coercion/UNIXSeconds->DateTime}]+ (testing "datetime and timestamp checkpoint columns accept any unit"+ (is (validates?! (lookback-transform dt-field {:value 4 :unit "hour"})))+ (is (validates?! (lookback-transform ts-field {:value 4 :unit "minute"}))))+ (testing "coerced unix-timestamp columns count as temporal via their effective type"+ (is (validates?! (lookback-transform unix-field {:value 4 :unit "day"}))))+ (testing "date-only checkpoint columns require day-or-coarser units"+ (is (validates?! (lookback-transform date-field {:value 4 :unit "day"})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"days or a coarser unit"+ (validates?! (lookback-transform date-field {:value 4 :unit "hour"})))))+ (testing "time-only checkpoint columns are rejected"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"only supported for date or datetime"+ (validates?! (lookback-transform time-field {:value 1 :unit "day"})))))+ (testing "numeric checkpoint columns are rejected"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"only supported for date or datetime"+ (validates?! (lookback-transform num-field {:value 4 :unit "day"})))))+ (testing "no configured lookback is a no-op"+ (is (validates?! (lookback-transform dt-field nil))))))diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.cljindex b64834a4fdb..5486baf4175 100644--- a/test/metabase/transforms/models_test.clj+++ b/test/metabase/transforms/models_test.clj@@ -357,6 +357,27 @@(let [t (t2/select-one :model/Transform transform-id)](is (= "99" (:last_checkpoint_value t)))))))))+(deftest lookback-round-trips-with-keyword-keys-test+ (testing "the nested lookback map comes back from the app DB with keyword keys"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Transform {transform-id :id}+ {:source {:type "query"+ :query {:database db-id+ :type "native"+ :native {:query "SELECT 1"+ :template-tags {}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id 100+ :lookback {:value 4 :unit "day"}}}+ :target {:type "table-incremental"+ :schema "public"+ :name "test_lookback"+ :db_id db-id}}]+ (is (= {:value 4 :unit "day"}+ (get-in (t2/select-one :model/Transform transform-id)+ [:source :source-incremental-strategy :lookback])))))))+(deftest schedules-for-transforms-test(testing "returns the schedules of the active jobs that run a transform via shared tags"(mt/with-temp [:model/TransformTag tag {}diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.cljindex 88ad39bcfa9..b8e22accdb5 100644--- a/test/metabase/transforms_base/util_test.clj+++ b/test/metabase/transforms_base/util_test.clj@@ -5,7 +5,8 @@[metabase.config.core :as config][metabase.sync.sync :as sync][metabase.test :as mt]- [metabase.transforms-base.util :as transforms-base.u]))+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.util.date-2 :as u.date]))(set! *warn-on-reflection* true)@@ -100,6 +101,81 @@(is (true? (transforms-base.u/full-incremental-run?{:target {:type :table-incremental} :last_checkpoint_value nil})))))+(deftest ^:parallel supported-checkpoint-column?-test+ (testing "numeric and date/datetime columns are supported"+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Integer}))+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Float}))+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Date}))+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTime}))+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/DateTimeWithTZ}))+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/Instant})))+ (testing "coerced columns count via their effective type"+ (is (transforms-base.u/supported-checkpoint-column? {:base-type :type/BigInteger+ :effective-type :type/Instant})))+ (testing "time-only and non-orderable columns are not supported"+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Time})))+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/TimeWithTZ})))+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Text})))+ (is (not (transforms-base.u/supported-checkpoint-column? {:base-type :type/Boolean})))))++(deftest ^:parallel apply-lookback-test+ (let [apply-lookback @#'transforms-base.u/apply-lookback]+ (testing "temporal checkpoints are pushed back by value units"+ (is (= (t/local-date-time 2026 1 27 21 0 4)+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4 :unit "day"})))+ (is (= (t/local-date-time 2026 1 31 19 0 4)+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 2 :unit "hour"})))+ (is (= (t/local-date 2026 1 24)+ (apply-lookback (t/local-date 2026 1 31) {:base-type :type/Date} {:value 1 :unit "week"}))))+ (testing "a coerced numeric column (e.g. a unix timestamp) is temporal via its effective type"+ (is (= (u.date/parse "2026-01-30T21:00:04Z")+ (apply-lookback (u.date/parse "2026-01-31T21:00:04Z")+ {:base-type :type/BigInteger :effective-type :type/Instant}+ {:value 1 :unit "day"}))))+ (testing "a lookback on a non-temporal checkpoint throws a user-facing error"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"only supported for date or datetime"+ (apply-lookback (biginteger 42) {:base-type :type/Integer} {:value 4 :unit "day"}))))+ (testing "a lookback on a time-only checkpoint throws a user-facing error"+ ;; time-only watermarks wrap at midnight, and day-based units would throw inside u.date/add+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"only supported for date or datetime"+ (apply-lookback (t/local-time 21 0 4) {:base-type :type/Time} {:value 2 :unit "hour"})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"only supported for date or datetime"+ (apply-lookback (t/offset-time 21 0 4) {:base-type :type/TimeWithTZ} {:value 1 :unit "day"}))))+ (testing "a lookback without a unit throws a user-facing error"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"requires a unit"+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 4}))))+ (testing "a lookback with a non-positive value throws instead of silently skipping data"+ ;; a negative window would move `lo` forward past the watermark, permanently skipping rows+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"positive integer value"+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value 0 :unit "day"})))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"positive integer value"+ (apply-lookback (t/local-date-time 2026 1 31 21 0 4) {:base-type :type/DateTime} {:value -4 :unit "day"}))))))++(deftest ^:parallel checkpoint-compare-test+ (let [checkpoint-compare @#'transforms-base.u/checkpoint-compare]+ (testing "numeric values compare numerically across numeric classes"+ (is (neg? (checkpoint-compare :type/Integer (biginteger 41) 42)))+ (is (zero? (checkpoint-compare :type/Integer (biginteger 42) 42))))+ (testing "temporal values compare as instants even across temporal classes"+ (is (neg? (checkpoint-compare :type/DateTimeWithTZ+ (t/instant "2026-01-30T10:00:00Z")+ (t/offset-date-time "2026-01-31T10:00:00Z"))))+ (is (pos? (checkpoint-compare :type/DateTime+ (t/local-date-time 2026 2 1 0 0 0)+ (t/local-date-time 2026 1 31 0 0 0)))))))+(deftest ^:parallel checkpoint-span-attrs-test(testing "nil source-range-params yields an empty attrs map"(is (= {} (transforms-base.u/checkpoint-span-attrs nil))))diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 126546a6dad..dd5901f8d0e 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -1855,6 +1855,121 @@(is (string? response))(is (re-find #"requires a table variable" response))))))))))))+(deftest lookback-validated-on-create-test+ (mt/with-premium-features #{:transforms-basic :hosting}+ (testing "POST /api/transform lookback window validation (GDGT-2868)"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)+ query (native-query-with-table-tag+ "SELECT id, name, price, created_at FROM {{source_table}}"+ (mt/id :transforms_products))+ ts-field-id (mt/id :transforms_products :created_at)+ num-field-id (mt/id :transforms_products :price)+ payload (fn [table-name checkpoint-field-id lookback target-strategy]+ {:name "Lookback Transform"+ :source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id checkpoint-field-id+ :lookback lookback}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy target-strategy}})+ merge-target {:type "merge" :unique-key [{:name "id"}]}]+ (testing "a lookback window without a unit is rejected by the schema"+ (mt/user-http-request :crowberto :post 400 "transform"+ (payload "lookback_no_unit" ts-field-id+ {:value 4}+ merge-target)))+ (testing "a lookback window on a numeric checkpoint column is rejected"+ (let [response (mt/user-http-request :crowberto :post 400 "transform"+ (payload "lookback_num" num-field-id+ {:value 4 :unit "day"}+ merge-target))]+ (is (string? response))+ (is (re-find #"only supported for date or datetime" response))))+ (testing "an unknown lookback unit is rejected by the schema"+ (mt/user-http-request :crowberto :post 400 "transform"+ (payload "lookback_bad_unit" ts-field-id+ {:value 4 :unit "fortnight"}+ merge-target)))+ (testing "a valid temporal lookback with a merge target is accepted"+ (with-transform-cleanup! [table-name "lookback_ok"]+ (let [created (mt/user-http-request :crowberto :post 200 "transform"+ (payload table-name ts-field-id+ {:value 4 :unit "day"}+ merge-target))]+ (is (= {:value 4 :unit "day"}+ (get-in created [:source :source-incremental-strategy :lookback]))))))+ (testing "a valid temporal lookback with an append target is accepted"+ (with-transform-cleanup! [table-name "lookback_append_ok"]+ (let [created (mt/user-http-request :crowberto :post 200 "transform"+ (payload table-name ts-field-id+ {:value 4 :unit "day"}+ {:type "append"}))]+ (is (= {:value 4 :unit "day"}+ (get-in created [:source :source-incremental-strategy :lookback]))))))))))))++(deftest lookback-validated-on-update-test+ (mt/with-premium-features #{:transforms-basic :hosting}+ (testing "PUT /api/transform lookback window validation (GDGT-2868)"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/dataset transforms-dataset/transforms-test+ (let [schema (get-test-schema)+ query (native-query-with-table-tag+ "SELECT id, name, created_at FROM {{source_table}}"+ (mt/id :transforms_products))+ ts-field-id (mt/id :transforms_products :created_at)+ num-field-id (mt/id :transforms_products :price)]+ (with-transform-cleanup! [table-name "lookback_update"]+ (mt/with-temp [:model/Transform {transform-id :id}+ {:name "Append Incremental"+ :source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "append"}}}]+ (testing "adding a lookback to an existing incremental transform is accepted"+ (let [updated (mt/user-http-request :crowberto :put 200+ (format "transform/%d" transform-id)+ {:source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id+ :lookback {:value 4 :unit "day"}}}})]+ (is (= {:value 4 :unit "day"}+ (get-in updated [:source :source-incremental-strategy :lookback])))))+ (testing "switching the checkpoint to a numeric column while a lookback is set is rejected"+ (let [response (mt/user-http-request :crowberto :put 400+ (format "transform/%d" transform-id)+ {:source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id num-field-id+ :lookback {:value 4 :unit "day"}}}})]+ (is (string? response))+ (is (re-find #"only supported for date or datetime" response))))+ (testing "adding a lookback together with a switch to the merge strategy is accepted"+ (let [updated (mt/user-http-request :crowberto :put 200+ (format "transform/%d" transform-id)+ {:source {:type "query"+ :query query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id+ :lookback {:value 4 :unit "day"}}}+ :target {:type "table-incremental"+ :schema schema+ :name table-name+ :target-incremental-strategy {:type "merge"+ :unique-key [{:name "id"}]}}})]+ (is (= {:value 4 :unit "day"}+ (get-in updated [:source :source-incremental-strategy :lookback])))))))))))))+(deftest search-filters-transform-source-types-test(mt/with-premium-features #{:transforms-basic :hosting}(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)