The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
incremental transforms with source filter (#70573)
Nicola Mometto525b09b87b35
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 5970b7e728e..2495f0f114d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1699,13 +1699,11 @@metabase.transforms-rest.api.transform-jobmetabase.transforms-rest.api.transform-tag}:uses #{api- drivereventsmodelsquery-processorrequestserver- sql-toolstransformstransforms-basetransforms-inspectordiff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.tsindex ec203205767..21a27362dae 100644--- a/e2e/support/helpers/e2e-transform-helpers.ts+++ b/e2e/support/helpers/e2e-transform-helpers.ts@@ -4,6 +4,7 @@ import type {CollectionId,PythonTransformTableAliases,SchemaName,+ TemplateTags,TransformId,TransformRun,TransformRunStatus,@@ -130,6 +131,10 @@ export function createSqlTransform({sourceCheckpointStrategy,name = "SQL transform",wrapId = true,+ templateTags,+ tableVariableTable,+ tableVariableSchema,+ databaseId = WRITABLE_DB_ID,}: {name?: string;sourceQuery: string;@@ -139,31 +144,67 @@ export function createSqlTransform({visitTransform?: boolean;sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;wrapId?: boolean;+ templateTags?: TemplateTags;+ /** When set, adds a template variable of type "table" named "table" for the given table */+ tableVariableTable?: string;+ tableVariableSchema?: string;+ databaseId?: number;}) {- return createTransform(- {- name,- source: {- type: "query",- query: {- database: WRITABLE_DB_ID,- type: "native",- native: {- query: sourceQuery,+ const create = (native: {+ query: string;+ "template-tags"?: TemplateTags;+ }) => {+ return createTransform(+ {+ name,+ source: {+ type: "query",+ query: {+ database: WRITABLE_DB_ID,+ type: "native",+ native,},+ "source-incremental-strategy": sourceCheckpointStrategy,},- "source-incremental-strategy": sourceCheckpointStrategy,- },- target: {- type: "table",- database: WRITABLE_DB_ID,- name: targetTable,- schema: targetSchema,+ target: {+ type: "table",+ database: WRITABLE_DB_ID,+ name: targetTable,+ schema: targetSchema,+ },+ tag_ids: tagIds,},- tag_ids: tagIds,- },- { wrapId, visitTransform },- );+ { wrapId, visitTransform },+ );+ };++ if (tableVariableTable) {+ return getTableId({+ databaseId,+ name: tableVariableTable,+ schema: tableVariableSchema,+ }).then((tableId) => {+ const resolvedTemplateTags: TemplateTags = {+ ...(templateTags ?? {}),+ table: {+ id: "table-tag-id",+ name: "table",+ "display-name": "Table",+ type: "table",+ "table-id": tableId,+ },+ };+ return create({+ query: sourceQuery,+ "template-tags": resolvedTemplateTags,+ });+ });+ }++ return create({+ query: sourceQuery,+ ...(templateTags && { "template-tags": templateTags }),+ });}export function createPythonTransform({diff --git a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.tsindex 0c6d92cc8ab..172a4ace9e2 100644--- a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts@@ -95,25 +95,30 @@ describe("issue GDGT-1774", () => {it("should display field options in the incremental update field picker (GDGT-1774)", () => {H.getTableId({ name: SOURCE_TABLE }).then((tableId) =>- H.createTransform({- name: "Incremental MBQL transform",- source: {- type: "query",- query: {- database: WRITABLE_DB_ID,+ H.getFieldId({ tableId, name: "score" }).then((fieldId) =>+ H.createTransform({+ name: "Incremental MBQL transform",+ source: {type: "query",- query: { "source-table": tableId },+ query: {+ database: WRITABLE_DB_ID,+ type: "query",+ query: { "source-table": tableId },+ },+ "source-incremental-strategy": {+ type: "checkpoint",+ "checkpoint-filter-field-id": fieldId,+ },},- "source-incremental-strategy": { type: "checkpoint" },- },- target: {- type: "table-incremental",- database: WRITABLE_DB_ID,- name: TARGET_TABLE,- schema: TARGET_SCHEMA,- "target-incremental-strategy": { type: "append" },- },- }),+ target: {+ type: "table-incremental",+ database: WRITABLE_DB_ID,+ name: TARGET_TABLE,+ schema: TARGET_SCHEMA,+ "target-incremental-strategy": { type: "append" },+ },+ }),+ ),).then((res) => H.DataStudio.Transforms.visitSettingsTab(res.body.id));diff --git a/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.tsnew file mode 100644index 00000000000..e66f9efd1f7--- /dev/null+++ b/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.ts@@ -0,0 +1,416 @@+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";++const { H } = cy;++const DB_NAME = "Writable Postgres12";+const SOURCE_TABLE = "Animals";+const TARGET_TABLE = "transform_table";+const TARGET_SCHEMA = "Schema A";++describe("scenarios > admin > transforms incremental", () => {+ beforeEach(() => {+ H.restore("postgres-writable");+ H.resetTestTable({ type: "postgres", table: "many_schemas" });+ H.resetSnowplow();+ cy.signInAsAdmin();+ H.activateToken("bleeding-edge");+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });++ cy.intercept("PUT", "/api/field/*").as("updateField");+ cy.intercept("POST", "/api/transform").as("createTransform");+ cy.intercept("PUT", "/api/transform/*").as("updateTransform");+ cy.intercept("DELETE", "/api/transform/*").as("deleteTransform");+ cy.intercept("DELETE", "/api/transform/*/table").as("deleteTransformTable");+ cy.intercept("POST", "/api/transform-tag").as("createTag");+ cy.intercept("PUT", "/api/transform-tag/*").as("updateTag");+ cy.intercept("DELETE", "/api/transform-tag/*").as("deleteTag");+ cy.intercept("POST", "/api/ee/dependencies/check-transform").as(+ "checkTransformDependencies",+ );+ cy.intercept("POST", "/api/transform/*/reset-checkpoint").as(+ "resetCheckpoint",+ );+ });++ afterEach(() => {+ H.expectNoBadSnowplowEvents();+ });++ describe("creation", () => {+ it("should be able to create and run an mbql incremental transform", () => {+ cy.log("create a new transform");+ visitTransformListPage();+ cy.button("Create a transform").click();+ H.popover().findByText("Query builder").click();++ H.expectUnstructuredSnowplowEvent({+ event: "transform_create",+ event_detail: "query",+ });++ H.miniPicker().within(() => {+ cy.findByText(DB_NAME).click();+ cy.findByText(TARGET_SCHEMA).click();+ cy.findByText(SOURCE_TABLE).click();+ });+ getQueryEditor().button("Save").click();++ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type("MBQL");++ cy.log("should auto-populate table name based on transform name...");+ cy.findByLabelText("Table name").should("have.value", "mbql");+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);++ cy.log("...unless user has manually modified the table name");+ cy.findByLabelText("Name").type(" transform");+ cy.findByLabelText("Table name").should("have.value", TARGET_TABLE);+ cy.findByRole("switch", {+ name: /Only process new and changed data/i,+ }).click({ force: true });++ cy.button("Save").click();+ cy.wait("@createTransform").then(({ response }) => {+ const transformId = response?.body?.id;+ if (transformId != null) {+ cy.wrap(transformId).as("transformId");+ }+ });+ });++ cy.log("run the transform and make sure its table can be queried");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ H.expectUnstructuredSnowplowEvent({+ event: "transform_trigger_manual_run",+ });++ cy.findByRole("link", { name: "See all runs" }).click();++ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findByRole("row", { name: /MBQL transform/i }).click();+ });++ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/30/).should("be.visible");+ });+ });++ cy.log(+ "add one element to the source table and run incremental transform again",+ );+ H.queryWritableDB(+ 'INSERT INTO "Schema A"."Animals" (name, score) VALUES (\'NewRow\', 31)',+ );++ cy.go("back");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();++ cy.log(+ "verify the new element was picked up in the incremental transfer",+ );+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /MBQL transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });++ cy.log("go to Transform Settings and reset checkpoint");+ cy.go("back");+ H.DataStudio.Transforms.settingsTab().click();+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {+ cy.findByText(/31/).should("exist");+ });+ cy.button("Reset checkpoint").click();+ H.modal().within(() => {+ cy.button("Reset").click();+ });+ cy.wait("@resetCheckpoint");++ cy.log(+ "go to Runs tab, run transform again and check new run has checkpoint to 31",+ );+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /MBQL transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });+ });++ it(+ "should be able to create and run a Python incremental transform",+ { tags: ["@python"] },+ () => {+ H.setPythonRunnerSettings();+ cy.log("create a new transform");+ visitTransformListPage();+ cy.button("Create a transform").click();+ H.popover().findByText("Python script").click();++ H.expectUnstructuredSnowplowEvent({+ event: "transform_create",+ event_detail: "python",+ });++ cy.findByTestId("python-transform-top-bar")+ .findByText("Writable Postgres12")+ .click();+ H.popover().findByText(DB_NAME).click();++ getPythonDataPicker().findByText("Select a table…").click();+ H.entityPickerModal().findByText(SOURCE_TABLE).click();++ H.PythonEditor.clear().paste(+ `import pandas as pd++def transform(animals):+ return pd.DataFrame([{"name": "test", "score": 0}])`,+ );++ getQueryEditor().button("Save").click();++ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type("Python transform");+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);+ cy.findByRole("switch", {+ name: /Only process new and changed data/i,+ }).click({ force: true });+ cy.button("Save").click();+ cy.wait("@createTransform").then(({ response }) => {+ const transformId = response?.body?.id;+ if (transformId != null) {+ cy.wrap(transformId).as("transformId");+ }+ });+ });++ cy.log("run the transform and make sure its table can be queried");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ H.expectUnstructuredSnowplowEvent({+ event: "transform_trigger_manual_run",+ });++ cy.findByRole("link", { name: "See all runs" }).click();++ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findByRole("row", { name: /Python transform/i }).click();+ });++ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/30/).should("be.visible");+ });+ });++ cy.log(+ "add one element to the source table and run incremental transform again",+ );+ H.queryWritableDB(+ 'INSERT INTO "Schema A"."Animals" (name, score) VALUES (\'NewRow\', 31)',+ );++ cy.go("back");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();++ cy.log(+ "verify the new element was picked up in the incremental transfer",+ );+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /Python transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });++ cy.log("go to Transform Settings and reset checkpoint");+ cy.go("back");+ H.DataStudio.Transforms.settingsTab().click();+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {+ cy.findByText(/31/).should("exist");+ });+ cy.button("Reset checkpoint").click();+ H.modal().within(() => {+ cy.button("Reset").click();+ });+ cy.wait("@resetCheckpoint");++ cy.log(+ "go to Runs tab, run transform again and check new run has checkpoint to 31",+ );+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /Python transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });+ },+ );++ it("should be able to create and run a native SQL incremental transform", () => {+ const SCHEMA_B = "Schema B";+ cy.log("create a new transform");+ visitTransformListPage();+ cy.button("Create a transform").click();+ H.popover().findByText("SQL query").click();++ H.expectUnstructuredSnowplowEvent({+ event: "transform_create",+ event_detail: "native",+ });++ H.popover().findByText(DB_NAME).click();+ H.NativeEditor.type("SELECT * FROM {{t}}", {+ allowFastSet: true,+ }).blur();++ cy.log("configure table variable for incremental support");+ cy.findByTestId("native-query-top-bar")+ .findByLabelText("Variables")+ .click();+ editorSidebar().findByLabelText("Variable type").click();+ H.popover().findByText("Table").click();+ H.popover().findByText(SCHEMA_B).click();+ H.popover().findByText(SOURCE_TABLE).click();++ getQueryEditor().button("Save").click();++ H.modal().within(() => {+ cy.findByLabelText("Name").clear().type("SQL transform");+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);+ cy.findByRole("switch", {+ name: /Only process new and changed data/i,+ }).click({ force: true });+ cy.button("Save").click();+ cy.wait("@createTransform");+ });++ cy.log("run the transform and make sure its table can be queried");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ H.expectUnstructuredSnowplowEvent({+ event: "transform_trigger_manual_run",+ });++ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findByRole("row", { name: /SQL transform/i }).click();+ });++ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/30/).should("be.visible");+ });+ });++ cy.log(+ "add one element to the source table and run incremental transform again",+ );+ H.queryWritableDB(+ `INSERT INTO "${SCHEMA_B}"."Animals" (name, score) VALUES (\'NewRow\', 31)`,+ );++ cy.go("back");+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();++ cy.log(+ "verify the new element was picked up in the incremental transfer",+ );+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /SQL transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });++ cy.log("go to Transform Settings and reset checkpoint");+ cy.go("back");+ H.DataStudio.Transforms.settingsTab().click();+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {+ cy.findByText(/31/).should("exist");+ });+ cy.button("Reset checkpoint").click();+ H.modal().within(() => {+ cy.button("Reset").click();+ });+ cy.wait("@resetCheckpoint");++ cy.log(+ "go to Runs tab, run transform again and check new run has checkpoint to 31",+ );+ H.DataStudio.Transforms.runTab().click();+ runTransformAndWaitForSuccess();+ cy.findByRole("link", { name: "See all runs" }).click();+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {+ cy.findAllByRole("row", { name: /SQL transform/i })+ .first()+ .click();+ });+ cy.findByRole("region", { name: "Info" }).within(() => {+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {+ cy.findByText(/31/).should("be.visible");+ });+ });+ });+ });+});++function visitTransformListPage() {+ return cy.visit("/data-studio/transforms");+}++function runTransformAndWaitForSuccess() {+ getRunButton().click();+ getRunButton().should("have.text", "Ran successfully");+}++function getRunButton(options: { timeout?: number } = {}) {+ return cy.findAllByTestId("run-button").eq(0, options);+}++function getQueryEditor() {+ return cy.findByTestId("transform-query-editor");+}++function editorSidebar() {+ return cy.findByTestId("editor-sidebar");+}++function getPythonDataPicker() {+ return cy.findByTestId("python-data-picker");+}diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex aa0d8edb714..453c2cd0b0e 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -925,7 +925,17 @@ LIMIT});it("should handle sequential changes correctly when first update is in progress", () => {- createMbqlTransform({ visitTransform: true });+ // Use composite_pk_table which has at least two numeric fields (id1, score)+ // so we can reliably select a second checkpoint field+ H.resetTestTable({ type: "postgres", table: "composite_pk_table" });+ H.resyncDatabase({+ dbId: WRITABLE_DB_ID,+ tableName: "composite_pk_table",+ });+ createMbqlTransform({+ sourceTable: "composite_pk_table",+ visitTransform: true,+ });H.DataStudio.Transforms.settingsTab().click();cy.log("Verify initial state");@@ -958,12 +968,11 @@ LIMITcy.wait(400);// Make a second change while first is still in progress- // Select any available checkpoint field+ // Select the second checkpoint field (composite_pk_table has id1 and score)getFieldPicker().scrollIntoView().should("be.visible");getFieldPicker().click();- // Click the first available option in the popover- H.popover().findAllByRole("option").first().click();+ H.popover().findAllByRole("option").eq(1).click();cy.log("Wait for both requests to complete");cy.wait("@updateTransformDelayed");@@ -983,7 +992,15 @@ LIMIT});it("should update source strategy and checkpoint field inline", () => {- createMbqlTransform({ visitTransform: true });+ H.resetTestTable({ type: "postgres", table: "composite_pk_table" });+ H.resyncDatabase({+ dbId: WRITABLE_DB_ID,+ tableName: "composite_pk_table",+ });+ createMbqlTransform({+ sourceTable: "composite_pk_table",+ visitTransform: true,+ });H.DataStudio.Transforms.settingsTab().click();cy.log("Enable incremental transformation");@@ -999,7 +1016,7 @@ LIMITcy.log("Select a checkpoint field");getFieldPicker().click();// Click the first available option in the popover- H.popover().findAllByRole("option").first().click();+ H.popover().findAllByRole("option").eq(1).click();cy.wait("@updateTransform");H.undoToast().should("contain.text",@@ -1658,98 +1675,6 @@ LIMITH.DataStudio.Transforms.pythonResults().should("not.exist");},);-- describe("query complexity warning", () => {- it("should show complexity warning modal when saving a complex SQL query", () => {- cy.log("create a simple SQL transform");- createSqlTransform({- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,- visitTransform: true,- sourceCheckpointStrategy: { type: "checkpoint" },- });-- cy.log("visit edit mode and change to a complex query with LIMIT");- H.DataStudio.Transforms.clickEditDefinition();- cy.url().should("include", "/edit");-- H.NativeEditor.type(" LIMIT 10");- getQueryEditor().button("Save").click();-- handleQueryComplexityWarningModal("cancel");- cy.log("verify modal is closed and still in edit mode");- H.modal().should("not.exist");- cy.url().should("include", "/edit");- cy.get("@updateTransform.all").should("have.length", 0);-- cy.log("Save anyway");- getQueryEditor().button("Save").click();- handleQueryComplexityWarningModal("save");-- cy.wait("@updateTransform");- cy.url().should("not.include", "/edit");- });-- it("should confirm incremental settings change if query is complex", () => {- createSqlTransform({- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,- sourceCheckpointStrategy: { type: "checkpoint" },- visitTransform: true,- });- H.DataStudio.Transforms.settingsTab().click();-- cy.log("Toggle incremental on");- isIncrementalSwitchDisabled();- getIncrementalSwitch().click();-- handleQueryComplexityWarningModal("cancel");-- cy.log("Verify that the switch is still off");- isIncrementalSwitchDisabled();-- cy.log("Toggle incremental on");- getIncrementalSwitch().click();- handleQueryComplexityWarningModal("save");-- cy.wait("@updateTransform");- isIncrementalSwitchEnabled();- H.undoToast().should(- "contain.text",- "Incremental transformation settings updated",- );- });-- it("should show complexity warning with danger button in create transform modal when enabling incremental with complex query", () => {- cy.log("create a new SQL transform with a complex query");- visitTransformListPage();- cy.button("Create a transform").click();- H.popover().findByText("SQL query").click();- H.popover().findByText(DB_NAME).click();-- H.NativeEditor.type(- `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,- );-- getQueryEditor().button("Save").click();-- H.modal().within(() => {- cy.findByLabelText("Name").clear().type("Complex SQL transform");- cy.findByLabelText("Table name").clear().type(TARGET_TABLE);-- cy.log("Enable incremental transformation");- getIncrementalSwitch().click();-- cy.log("Verify complexity warning appears inline");- cy.findByTestId("query-complexity-warning")- .scrollIntoView()- .should("be.visible");-- cy.log("Verify the submit button is styled as danger (red)");- cy.findByRole("button", { name: "Save anyway" })- .scrollIntoView()- .should("have.css", "background-color", "rgb(209, 44, 41)");- });- });- });});describe("runs", () => {@@ -2686,7 +2611,9 @@ LIMITbeforeEach(() => {cy.log("create a transform");createSqlTransform({- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,+ sourceQuery: "SELECT * FROM {{ table }}",+ tableVariableTable: SOURCE_TABLE,+ tableVariableSchema: TARGET_SCHEMA,});cy.log("set up remote sync");H.setupGitSync();@@ -3788,18 +3715,6 @@ function isIncrementalSwitchDisabled() {return getIncrementalSwitch().findByRole("switch").should("not.be.checked");}-function handleQueryComplexityWarningModal(action: "cancel" | "save") {- cy.log(`Verify complexity warning modal appears and ${action} it`);- return H.modal().within(() => {- cy.findByTestId("query-complexity-warning").should("be.visible");- if (action === "save") {- cy.button("Save anyway").click();- } else {- cy.button("Cancel").click();- }- });-}-function getJobTransformTable() {return cy.findByLabelText("Job transforms");}@@ -3891,6 +3806,8 @@ function createSqlTransform(opts: {tagIds?: TransformTagId[];visitTransform?: boolean;sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;+ tableVariableTable?: string;+ tableVariableSchema?: string;}) {return H.createSqlTransform({targetTable: TARGET_TABLE,diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex 9ec54e49cd0..1f501795297 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -165,18 +165,23 @@(fn [_ _ transform _ _] (-> transform :target :type keyword)))(defmethod transfer-file-to-db :table-incremental- [driver {db-id :id}+ [driver {db-id :id :as db}{:keys [target] :as transform}metadata temp-file]- (let [table-name (transforms-base.u/qualified-table-name driver target)- table-exists? (transforms-base.u/target-table-exists? transform)- data-source {:type :jsonl-file- :file temp-file}]- (if (not table-exists?)- (do- (log/info "New table")- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))- (insert-data! driver db-id (table-schema table-name metadata) data-source))))+ ;; First incremental run: no checkpoint exists yet, behave like non-incremental+ ;; to drop and recreate the table rather than appending to existing data.+ (if (nil? (:last_checkpoint_value transform))+ ((get-method transfer-file-to-db :table) driver db transform metadata temp-file)+ ;; Normal incremental: append if table exists, create if it doesn't+ (let [table-name (transforms-base.u/qualified-table-name driver target)+ table-exists? (transforms-base.u/target-table-exists? transform)+ data-source {:type :jsonl-file+ :file temp-file}]+ (if (not table-exists?)+ (do+ (log/info "New table")+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))+ (insert-data! driver db-id (table-schema table-name metadata) data-source)))))(defmethod transfer-file-to-db :table[driver {db-id :id :as db}@@ -216,18 +221,19 @@Options:- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"- [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn]}]+ [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn source-range-params]}];; Resolve name-based source table refs to table IDs (throws if any not found)(let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))](with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)](let [driver (:engine db)server-url (transforms-python.settings/python-runner-url)- _ (python-runner/copy-tables-to-s3! {:run-id run-id- :shared-storage @shared-storage-ref- :source (assoc source :source-tables resolved-source-tables)- :cancel-chan cancel-chan- :limit (:limit source)- :transform-id (:id transform)})+ _ (python-runner/copy-tables-to-s3! {:run-id run-id+ :shared-storage @shared-storage-ref+ :source (assoc source :source-tables resolved-source-tables)+ :cancel-chan cancel-chan+ :limit (:limit source)+ :transform-id (:id transform)+ :source-range-params source-range-params})_ (start-cancellation-process! server-url run-id cancel-chan){:keys [status body] :as response}(python-runner/execute-python-code-http-call!@@ -305,7 +311,7 @@:logs <string>:error <exception if failed>}"[transform :- ::transforms-base.schema/transform- {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan]} :- [:maybe ::transforms-base.schema/execute-base-options]]+ {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan source-range-params]} :- [:maybe ::transforms-base.schema/execute-base-options]](assert (transforms-base.u/python-transform? transform) "Transform must be a python transform")(let [message-log (or message-log (empty-message-log))](try@@ -336,7 +342,8 @@(log/info "Executing Python transform" transform-id "with target" (pr-str target))(let [result (run-python-transform-impl! transform db effective-run-id cancel-chan message-log- {:with-stage-timing-fn with-stage-timing-fn})]+ {:with-stage-timing-fn with-stage-timing-fn+ :source-range-params source-range-params})](log! message-log (i18n/tru "Python execution finished successfully in {0}"(u.format/format-milliseconds (u/since-ms start-ms))))@@ -346,7 +353,8 @@{:status :succeeded:result result- :logs (message-log->string message-log)}))+ :logs (message-log->string message-log)+ :source-range-params source-range-params}))(catch Exception e(let [data (ex-data e)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 1a88ad71a40..f4f680b03d4 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -110,12 +110,13 @@(driver.conn/with-write-connection(let [conn-spec (driver/connection-spec driver db)transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}- run-fn (fn [cancel-chan]+ run-fn (fn [cancel-chan source-range-params](let [result (transforms-base.i/execute-base!transform{:cancelled? #(boolean (a/poll! cancel-chan)):cancel-chan cancel-chan:run-id run-id+ :source-range-params source-range-params:message-log message-log:with-stage-timing-fn (fn [rid stage thunk](transforms.instrumentation/with-stage-timing [rid stage]@@ -126,10 +127,11 @@result))ex-message-fn #(exceptional-run-message message-log %)result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]- (transforms.u/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]+ (transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))];; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)(transforms-base.u/complete-execution! transform{:run-id run-id+ :source-range-params (:source-range-params result):with-stage-timing-fn (fn [rid stage thunk](transforms.instrumentation/with-stage-timing [rid stage](thunk)))})diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 49691b82aa6..aea8c43f44b 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -270,14 +270,13 @@(defn- build-table-query"Build a mbql query for table, might add a proper filter for incremental transforms."- [table-id source-incremental-strategy transform-id limit]+ [table-id source-incremental-strategy source-range-params limit](let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)metadata-provider (lib-be/application-database-metadata-provider db-id)- table-metadata (lib.metadata/table metadata-provider table-id)- transform (t2/select-one :model/Transform transform-id)]+ table-metadata (lib.metadata/table metadata-provider table-id)](cond-> (-> (lib/query metadata-provider table-metadata)lib/disable-default-limit)- source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-incremental-strategy (transforms-base.u/next-checkpoint transform))+ source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-range-params)limit (lib/limit limit))));; TODO break this up such that s3 can be swapped out for other transfer mechanisms.@@ -289,7 +288,7 @@sourcecancel-chanlimit- transform-id]}]+ source-range-params]}](when (and (:source-incremental-strategy source)(> (count (:source-tables source)) 1))(throw (ex-info "Incremental transforms for python only supports one source table" {})))@@ -307,7 +306,7 @@fields-meta (fields-metadata driver table-id)manifest (generate-manifest table-id fields-meta)](transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]- (let [query (build-table-query table-id (:source-incremental-strategy source) transform-id limit)]+ (let [query (build-table-query table-id (:source-incremental-strategy source) source-range-params limit)](write-query-data-to-file!{:query query:fields-meta fields-metadiff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex c527f347b13..ab502aea2a7 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -1,7 +1,6 @@(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms.incremental-test"Tests for incremental transforms functionality."(:require- [clojure.set :as set][clojure.string :as str][clojure.test :refer :all][honey.sql :as sql]@@ -26,25 +25,22 @@(def checkpoint-configs{:integer {:field-name "id":template-tag-type :number- :lib-column-key "column-unique-key-v1$id"- :expected-initial-checkpoint 10- :expected-second-checkpoint 16}+ :expected-initial-checkpoint 16+ :expected-second-checkpoint 17}:float {:field-name "price":template-tag-type :number- :lib-column-key "column-unique-key-v1$price"- :expected-initial-checkpoint 49.99- :expected-second-checkpoint 199.99}+ :expected-initial-checkpoint 199.99+ :expected-second-checkpoint 379.99}:temporal {:field-name "created_at":template-tag-type :text- :lib-column-key "column-unique-key-v1$created_at"- :expected-initial-checkpoint "2024-01-10"- :expected-second-checkpoint "2024-01-16"}})+ :expected-initial-checkpoint #t "2024-01-16T10:00:00Z"+ :expected-second-checkpoint #t "2024-01-20T10:00:00Z"}})(defn- valid-checkpoint-transform-combo?- "Check if a checkpoint type and transform type combination is valid. "- [checkpoint-type transform-type]- (not (and (= checkpoint-type :temporal)- (= transform-type :native))))+ "Check if a checkpoint type and transform type combination is valid.+ With table tag expansion, all combinations are now supported."+ [_checkpoint-type _transform-type]+ true)(defn- build-table-query-with-order-by"Wraps the original build-table-query to add ORDER BY for deterministic test results."@@ -77,49 +73,33 @@(transforms.execute/execute! transform run-opts))(transforms.execute/execute! transform run-opts)))-(defn- make-incremental-source-query- "Create a native query with optional checkpoint template tag."- [schema checkpoint-config]- (let [{:keys [field-name template-tag-type]} checkpoint-config+(defn- make-incremental-source-query-with-table-tag+ "Create a native query with a table tag for testing subquery expansion."+ [checkpoint-config]+ (let [{:keys [field-name]} checkpoint-config+ table-id (mt/id :transforms_products)timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))- query (format "SELECT *, %s AS %s FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"+ query (format "SELECT *, %s AS %s FROM {{source_table}} AS %s ORDER BY %s"timestamp-sql(sql.u/quote-name driver/*driver* :field "load_timestamp")- (if schema- (sql.u/quote-name driver/*driver* :table schema "transforms_products")- "transforms_products")- (sql.u/quote-name driver/*driver* :field field-name)+ (sql.u/quote-name driver/*driver* :field "source_table")(sql.u/quote-name driver/*driver* :field field-name))]{:database (mt/id):type :native:native {:query query- :template-tags {"checkpoint" {:id "checkpoint"- :name "checkpoint"- :display-name "Checkpoint"- :type template-tag-type- :required false}}}}))--(defn- make-incremental-source-query-without-template-tag- "Create a native query without template tags for testing automatic checkpoint insertion. "- [schema]- (let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))- query (format "SELECT *, %s AS %s FROM %s"- timestamp-sql- (sql.u/quote-name driver/*driver* :field "load_timestamp")- (if schema- (sql.u/quote-name driver/*driver* :table schema "transforms_products")- "transforms_products"))]- {:database (mt/id)- :type :native- :native {:query query}}))+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}))(defn- make-incremental-mbql-query"Create an MBQL query for incremental transforms. "[checkpoint-config](let [{:keys [field-name]} checkpoint-config](mt/mbql-query transforms_products {:expressions {"load_timestamp" [:now]}- :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]- :limit 10})))+ :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]})))(def incremental-python-body(str "import pandas as pd\n"@@ -132,32 +112,28 @@(defn- make-incremental-transform-payload"Create a transform payload for incremental transform testing.- transform-type can be :native, :native-auto-wrap, :mbql, or :python+ transform-type can be :native, :mbql, or :pythoncheckpoint-config should be from checkpoint-configs"[transform-name target-table transform-type checkpoint-config]- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))- {:keys [field-name lib-column-key]} checkpoint-config]+ (let [{:keys [field-name]} checkpoint-config+ checkpoint-filter-field (fn [] (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products)))]{:name transform-name:source_database_id (mt/id):source (case transform-type+ ;; Native queries now use table tags with checkpoint-filter-field-id:native {:type "query"- :query (make-incremental-source-query schema checkpoint-config)+ :query (make-incremental-source-query-with-table-tag checkpoint-config):source-incremental-strategy {:type "checkpoint"- :checkpoint-filter field-name}}- :native-auto-wrap {:type "query"- :query (make-incremental-source-query-without-template-tag schema)- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter field-name}}+ :checkpoint-filter-field-id (checkpoint-filter-field)}}:mbql {:type "query":query (make-incremental-mbql-query checkpoint-config):source-incremental-strategy {:type "checkpoint"- :checkpoint-filter-unique-key lib-column-key}}+ :checkpoint-filter-field-id (checkpoint-filter-field)}}:python {:type "python":source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]- :limit 10:body incremental-python-body:source-incremental-strategy {:type "checkpoint"- :checkpoint-filter-unique-key lib-column-key}})+ :checkpoint-filter-field-id (checkpoint-filter-field)}}):target (merge target-table {:type "table-incremental":target-incremental-strategy {:type "append"}})}))@@ -182,9 +158,14 @@result (qp/process-query native-query)](some-> result :data :rows first first bigint)))-(defn get-checkpoint-value [transform]- (#'transforms-base.u/next-checkpoint-value- (transforms-base.u/next-checkpoint transform)))+(defn- get-checkpoint-value [transform-id]+ (let [transform (t2/select-one :model/Transform transform-id)+ {:keys [last_checkpoint_value]} transform]+ (when last_checkpoint_value+ (let [source-incremental-strategy (get-in transform [:source :source-incremental-strategy])+ field-id (:checkpoint-filter-field-id source-incremental-strategy)+ field (t2/select-one :model/Field field-id)]+ (#'transforms-base.u/parse-checkpoint-value (:base_type field) last_checkpoint_value)))))(defn- compare-checkpoint-values"Compare two checkpoint values with type-appropriate logic. "@@ -193,8 +174,7 @@:integer (= (bigint expected) (bigint actual)):float (and (number? actual)(< (Math/abs (double (- expected actual))) 0.01))- :temporal (and (string? actual)- (str/starts-with? actual expected))))+ :temporal (= expected actual)))(defn- insert-test-products!"Insert new products into the transforms_products table."@@ -235,7 +215,7 @@(disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))(defn- target-table-gen [prefix]- {:type "table"+ {:type :table:name prefix:schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products)):database (mt/id)})@@ -251,6 +231,7 @@(with-transform-cleanup! [target-table (target-table-gen "incremental_test")](let [checkpoint-config (get checkpoint-configs checkpoint-type){:keys [field-name]} checkpoint-config+ expected-field-id (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products))transform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)](testing "Transform is created successfully"(mt/with-temp [:model/Transform transform transform-payload]@@ -258,10 +239,10 @@(is (= "Test Incremental Transform" (:name transform)))(is (= "table-incremental" (-> transform :target :type)))(is (= "checkpoint" (-> transform :source :source-incremental-strategy :type)))- (is (= field-name (-> transform :source :source-incremental-strategy :checkpoint-filter)))+ (is (= expected-field-id (-> transform :source :source-incremental-strategy :checkpoint-filter-field-id)))(testing "No checkpoint exists initially"- (is (nil? (get-checkpoint-value transform))))+ (is (nil? (get-checkpoint-value (:id transform)))))(testing "Can retrieve transform via API"(let [retrieved (mt/user-http-request :crowberto :get 200 (format "transform/%d" (:id transform)))]@@ -293,23 +274,44 @@(transforms.tu/wait-for-table (:name target-table) 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 10 row-count) "First run should process the first 10 products")+ (is (= 16 row-count) "First run should process all 16 products")(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")(testing "Checkpoint is created after first run"- (let [checkpoint (get-checkpoint-value transform)]+ (let [checkpoint (get-checkpoint-value (:id transform))](is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)- (format "Checkpoint should be MAX(%s) from first 10 rows" (:field-name checkpoint-config)))))))+ (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config)))))))- (testing "Second run should process the remaining rows"- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [row-count (get-table-row-count target-table)- distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 16 row-count) "Second run should add remaining 6 rows")- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")- (let [checkpoint (get-checkpoint-value transform)]- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)- (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config))))))))))))))))+ (testing "Second run with no new data adds nothing"+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows, no new data")+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))++ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)+ (not= driver/*driver* :clickhouse)+ (not= driver/*driver* :snowflake))+ (testing "After inserting new data, incremental run appends only new rows"+ (with-insert-test-products!+ [{:name "Incremental Twice Product"+ :category "Gadget"+ :price 379.99+ :created-at "2024-01-20T10:00:00"}]+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")+ (if (= checkpoint-type :integer)+ ;; Integer checkpoint uses auto-increment ID, just verify it advanced+ (is (> checkpoint expected-initial-checkpoint)+ "Integer checkpoint should advance past initial value")+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from all 17 rows" (:field-name checkpoint-config)))))))))))))))))))(deftest switch-incremental-to-non-incremental-test(testing "Switching an incremental transform to non-incremental overwrites data"@@ -325,47 +327,48 @@{:keys [expected-initial-checkpoint]} checkpoint-configinitial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)](mt/with-temp [:model/Transform transform initial-payload]- (testing "Initial incremental run processes first batch"- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (transforms.tu/wait-for-table (:name target-table) 10000)- (let [row-count (get-table-row-count target-table)- distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 10 row-count) "Initial run should process first 10 products")- (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")- (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))-- (testing "Second incremental run processes remaining data"- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [row-count (get-table-row-count target-table)- distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 16 row-count) "Should have 16 rows after second incremental run")- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")))-- (testing "Switch to non-incremental via PUT API"- (let [non-incremental-payload (-> initial-payload- (update :source dissoc :source-incremental-strategy)- (update :target dissoc :source-incremental-strategy)- (update :target assoc :type "table"))- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))- non-incremental-payload)]- (is (= "table" (-> updated :target :type)))- (is (nil? (-> updated :source :source-incremental-strategy)))))-- (testing "Non-incremental run overwrites all data"- (let [transform (t2/select-one :model/Transform (:id transform))]+ (let [get-transform #(t2/select-one :model/Transform (:id transform))]+ (testing "Initial incremental run processes all data"(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [row-count (get-table-row-count target-table)+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "Initial run should process all 16 products")+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))++ (testing "Second incremental run with no new data"+ (execute-transform-with-ordering! (get-transform) transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 10 row-count) "Should overwrite to 10 rows")- (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))-- (testing "Running again still overwrites"+ (is (= 16 row-count) "Should still have 16 rows, no new data")+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))++ (testing "Switch to non-incremental via PUT API"+ (let [non-incremental-payload (-> initial-payload+ (update :source dissoc :source-incremental-strategy)+ (update :target dissoc :source-incremental-strategy)+ (update :target assoc :type "table"))+ updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))+ non-incremental-payload)]+ (is (= "table" (-> updated :target :type)))+ (is (nil? (-> updated :source :source-incremental-strategy)))))++ (testing "Non-incremental run overwrites all data"+ (let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [row-count (get-table-row-count target-table)+ (let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 10 row-count) "Should still have 10 rows after another run")- (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))))))))))))))+ (is (= 16 row-count) "Should overwrite to 16 rows")+ (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))++ (testing "Running again still overwrites"+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows after another run")+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))))))))))))))(deftest switch-non-incremental-to-incremental-test(testing "Switching a non-incremental transform to incremental computes checkpoint from existing data"@@ -378,7 +381,7 @@(mt/dataset transforms-dataset/transforms-test(with-transform-cleanup! [target-table (target-table-gen "switch_non_incr_to_incr")](let [checkpoint-config (get checkpoint-configs checkpoint-type)- {:keys [expected-second-checkpoint]} checkpoint-config+ {:keys [expected-initial-checkpoint]} checkpoint-configincremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)initial-payload (-> incremental-payload(update :source dissoc :source-incremental-strategy)@@ -390,11 +393,10 @@(transforms.tu/wait-for-table (:name target-table) 10000)(let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)]- (is (= 10 row-count) "Initial run should process 10 products")+ (is (= 16 row-count) "Initial run should process all 16 products")(is (= 1 distinct-timestamps) "All rows should have same timestamp from non-incremental run")-(testing "No checkpoint exists"- (let [checkpoint (get-checkpoint-value transform)]+ (let [checkpoint (get-checkpoint-value (:id transform))](is (nil? checkpoint) "No checkpoint for non-incremental transform")))))(testing "Switch to incremental via PUT API"@@ -403,15 +405,23 @@(is (= "table-incremental" (-> updated :target :type)))(is (= "checkpoint" (-> updated :source :source-incremental-strategy :type)))))- (testing "First incremental run after switch processes no new data"+ (testing "First incremental run after switch recreates table with checkpoint"(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [row-count (get-table-row-count target-table)+ (let [row-count (get-table-row-count target-table)distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 16 row-count) "Should process remaining 6 entries")- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "Should have all 16 rows")+ (is (= 1 distinct-timestamps) "Should have 1 distinct timestamp")+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))++ (testing "Second incremental run with no new data"+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows, no new data")+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))(when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment(not= driver/*driver* :clickhouse)@@ -426,80 +436,34 @@(let [transform (t2/select-one :model/Transform (:id transform))](execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})(let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value transform)]+ checkpoint (get-checkpoint-value (:id transform))](is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)");; For integer checkpoints, we can verify exact value >= 17;; For float/temporal, just verify checkpoint exists(is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))-(deftest native-query-without-template-tag-test- (testing "Native query without template tags uses automatic checkpoint wrapping"- (doseq [checkpoint-type [:integer :float :temporal]]- (testing (format "with %s checkpoint" (name checkpoint-type))- (mt/test-drivers (test-drivers)- (mt/with-premium-features #{:transforms-basic}- (mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table (target-table-gen "native_no_template")]- (let [checkpoint-config (get checkpoint-configs checkpoint-type)- {:keys [expected-second-checkpoint field-name]} checkpoint-config- schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))- transform-payload {:name "Native Without Template Tag"- :source {:type "query"- :query (make-incremental-source-query-without-template-tag schema)- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter field-name}}- :target (merge target-table- {:type "table-incremental"- :target-incremental-strategy {:type "append"}})}]- (mt/with-temp [:model/Transform transform transform-payload]- (testing "First run processes all existing data"- (transforms.execute/execute! transform {:run-method :manual})- (transforms.tu/wait-for-table (:name target-table) 10000)- (let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 16 row-count) "First run should process all 16 products")- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)- (format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))-- (testing "Second run without new data adds nothing"- (transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)]- (is (= 16 row-count) "Should still have 16 rows, no new data")))-- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment- (not= driver/*driver* :clickhouse)- ;; this *should* work see #68965 for context, will plan follow-up task- (not= driver/*driver* :snowflake))- (testing "After inserting new data, incremental run appends only new rows"- (with-insert-test-products!- [{:name "New Product 1"- :category "Electronics"- :price 299.99- :created-at "2024-01-21T10:00:00"}- {:name "New Product 2"- :category "Books"- :price 319.99- :created-at "2024-01-21T11:00:00"}]-- (transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")- (is (some? checkpoint) "Checkpoint should be updated")))))))))))))))-(deftest unsupported-checkpoint-column-type-test(testing "Transform fails at runtime with unsupported checkpoint column type"- (mt/test-drivers #{:h2 :postgres}+ (mt/test-drivers #{:postgres}(mt/with-premium-features #{:transforms-basic}(mt/dataset transforms-dataset/transforms-test(with-transform-cleanup! [target-table "unsupported_type_test"]- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))- ;; Create transform with text column (unsupported type) as checkpoint+ (let [name-field-id (mt/id :transforms_products :name)+ table-id (mt/id :transforms_products)+ schema (t2/select-one-fn :schema :model/Table table-id)transform-payload {:name "Invalid Checkpoint Type Transform":source {:type "query"- :query (make-incremental-source-query-without-template-tag schema)+ :query {:database (mt/id)+ :type :native+ :native {:query "SELECT * FROM {{source_table}} AS s"+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}:source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "name"}}+ :checkpoint-filter-field-id name-field-id}}:target {:type "table-incremental":schema schema:name target-table@@ -510,266 +474,219 @@(is (string? response))(is (re-find #"unsupported type" response)))))))))))-(deftest ^:postgres-only native-query-with-temporal-checkpoint-test- (testing "Native query with temporal checkpoint"- ;; we test only in postgres because it's easy to cast to ::timestamp- (mt/test-drivers [:postgres]- (mt/with-premium-features #{:transforms-basic}- (mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table (target-table-gen "native_temporal_cast")]- (let [checkpoint-config (get checkpoint-configs :temporal)- {:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-config- schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))- timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))- query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}::timestamp]] ORDER BY %s LIMIT 10"- timestamp-sql- (if schema- (sql.u/quote-name driver/*driver* :table schema "transforms_products")- "transforms_products")- field-name- field-name)- transform-payload {:name "Native With Temporal Cast"- :source {:type "query"- :query {:database (mt/id)- :type :native- :native {:query query- :template-tags {"checkpoint" {:id "checkpoint"- :name "checkpoint"- :display-name "Checkpoint"- :type :text- :required false}}}}- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter field-name}}- :target (merge target-table- {:type "table-incremental"- :target-incremental-strategy {:type "append"}})}]- (mt/with-temp [:model/Transform transform transform-payload]- (testing "First run processes first batch"- (transforms.execute/execute! transform {:run-method :manual})- (transforms.tu/wait-for-table (:name target-table) 10000)- (let [row-count (get-table-row-count target-table)- distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 10 row-count) "First run should process the first 10 products")- (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")- (is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)- (format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))-- (testing "Second run processes remaining data"- (transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)- distinct-timestamps (get-distinct-timestamp-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 16 row-count) "Second run should add remaining 6 rows")- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")- (is (compare-checkpoint-values :temporal expected-second-checkpoint checkpoint)- (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))-- (testing "Third run without new data adds nothing"- (transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)]- (is (= 16 row-count) "Should still have 16 rows, no new data")))-- (testing "After inserting new data, incremental run appends only new rows"- (with-insert-test-products!- [{:name "New Temporal Product"- :category "Electronics"- :price 299.99- :created-at "2024-01-21T10:00:00"}]-- (transforms.execute/execute! transform {:run-method :manual})- (let [row-count (get-table-row-count target-table)- checkpoint (get-checkpoint-value transform)]- (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")- (is (some? checkpoint) "Checkpoint should be updated"))))))))))))-(defn- pg-table-rows [db-spec table-name](next.jdbc/execute! db-spec [(format "SELECT * FROM %s" table-name)] {:builder-fn jdbc.rs/as-unqualified-lower-maps}))(deftest empty-table-test- (mt/test-drivers #{:postgres} ; no db specifics+ (mt/test-drivers #{:postgres}(mt/with-premium-features #{:transforms-basic}- (with-transform-cleanup! [target-table "empty_table_target"]- (let [db-id (mt/id)- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)- source {:type "query"- :query {:database db-id- :type :native- :native {:query "SELECT * FROM (VALUES (42)) x(id) WHERE 1 = 2"}}- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}- target {:type "table-incremental"- :schema "public"- :name target-table- :database db-id- :target-incremental-strategy {:type "append"}}]- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]- (transforms.execute/execute! transform {:run-method :manual})- (testing "still creates target table"- (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))- (testing "sync has picked up table"- (is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))- (testing "checkpoint is recognized"- (is (some? (transforms-base.u/next-checkpoint transform))))))))))+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "empty_table_target"]+ (let [db-id (mt/id)+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)+ table-id (mt/id :transforms_products)+ checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)+ ;; Query that returns no rows but has the checkpoint column+ source {:type "query"+ :query {:database db-id+ :type :native+ :native {:query "SELECT * FROM {{source_table}} AS s WHERE 1 = 2"+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id checkpoint-field-id}}+ target {:type "table-incremental"+ :schema "public"+ :name target-table+ :database db-id+ :target-incremental-strategy {:type "append"}}]+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]+ (transforms.execute/execute! transform {:run-method :manual})+ (testing "still creates target table"+ (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))+ (testing "sync has picked up table"+ (is (=? {:name target-table} (t2/select-one :model/Table :name target-table))))+ (testing "checkpoint is set from source table MAX even though query returned no rows"+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (is (some? (:last_checkpoint_value transform))+ "Checkpoint is computed from source table, not query output"))))))))))(deftest checkpoint-field-does-not-exist-test- (mt/test-drivers #{:postgres} ; no db specifics- (mt/with-premium-features #{:transforms-basic}- (with-transform-cleanup! [target-table "missing_field_target"]- (let [db-id (mt/id)- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)- source {:type "query"- :query {:database db-id- :type :native- :native {:query "SELECT * FROM (VALUES (42)) x(id)"}}- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "no_such_column"}}- target {:type "table-incremental"- :schema "public"- :name target-table- :database db-id- :target-incremental-strategy {:type "append"}}]- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]- (transforms.execute/execute! transform {:run-method :manual})- (testing "still creates target table"- (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))- (testing "sync has picked up table"- (is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))- (testing "target table has expected data"- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))- (testing "checkpoint is not recognized, so transform acts as if no checkpoint"- (is (nil? (transforms-base.u/next-checkpoint transform))))- ;; Maybe this is unrealistic - you cannot select a column that does not exist- ;; But the source tables schema can change, e.g. rename: you change event_time to event_ts or something- ;; so one would have to be careful to disable or delete transforms ahead of a schema change like this- ;; For now asserting behaviour is-what-it-is, but this should provoke a hmmm... maybe it is better to fail early?- (testing "running a second time will duplicate the existing data"- (transforms.execute/execute! transform {:run-method :manual})- (is (= [{:id 42} {:id 42}] (pg-table-rows db-spec target-table))))))))))+ (testing "Transform with non-existent checkpoint-filter-field-id fails gracefully"+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "missing_field_target"]+ (let [db-id (mt/id)+ table-id (mt/id :transforms_products)+ invalid-field-id 999999+ source {:type "query"+ :query {:database db-id+ :type :native+ :native {:query "SELECT id FROM {{source_table}}"+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id invalid-field-id}}+ target {:type "table-incremental"+ :schema "public"+ :name target-table+ :database db-id+ :target-incremental-strategy {:type "append"}}]+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]+ (testing "Transform execution fails with invalid checkpoint field ID"+ (is (thrown-with-msg?+ Exception+ #"Checkpoint field does not exist"+ (transforms.execute/execute! transform {:run-method :manual}))))))))))))++(deftest checkpoint-field-is-not-active-test+ (testing "Transform with inactive checkpoint field fails gracefully"+ (mt/test-drivers #{:postgres}+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "inactive_field_target"]+ (let [db-id (mt/id)+ table-id (mt/id :transforms_products)+ field-id (mt/id :transforms_products :id)+ source {:type "query"+ :query {:database db-id+ :type :native+ :native {:query "SELECT id FROM {{source_table}} as s"+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id field-id}}+ target {:type "table-incremental"+ :schema "public"+ :name target-table+ :database db-id+ :target-incremental-strategy {:type "append"}}]+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]+ ;; Mark the field as inactive+ (t2/update! :model/Field field-id {:active false})+ (try+ (testing "Transform execution fails with inactive checkpoint field"+ (is (thrown-with-msg?+ Exception+ #"Checkpoint field does not exist or is not active"+ (transforms.execute/execute! transform {:run-method :manual}))))+ (finally+ ;; Restore field to active state for other tests+ (t2/update! :model/Field field-id {:active true})))))))))))(deftest changing-query-keeps-checkpoint-test- (mt/test-drivers #{:postgres} ; no db specifics+ (mt/test-drivers #{:postgres}(mt/with-premium-features #{:transforms-basic}- (with-transform-cleanup! [target-table "change_table_target"]- (let [db-id (mt/id)- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)- source {:type "query"- :query {:database db-id- :type :native- :native {:query "SELECT * FROM (VALUES (42)) x(id)"}}- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}- target {:type "table-incremental"- :schema "public"- :name target-table- :database db-id- :target-incremental-strategy {:type "append"}}]- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]- (testing "initial run"- (transforms.execute/execute! transform {:run-method :manual})- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))- (testing "initial increment"- (transforms.execute/execute! transform {:run-method :manual})- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))- (t2/update! :model/Transform- (:id transform)- {:source {:type "query"- :query {:database db-id- :type :native- :native {:query "SELECT * FROM (VALUES (42), (43)) x(id)"}}- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}})- (testing "second increment"- (transforms.execute/execute! (t2/select-one :model/Transform (:id transform)) {:run-method :manual})- (is (= [{:id 42} {:id 43}] (pg-table-rows db-spec target-table))))))))))--(deftest filter-column-indexed-test- (testing "Filter column is indexed"- (doseq [checkpoint-type [:integer]- transform-type [:native :mbql :python]- :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]- (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))- (mt/with-premium-features #{:transforms-basic :transforms-python}+ (mt/dataset transforms-dataset/transforms-test+ (with-transform-cleanup! [target-table "change_table_target"]+ (let [db-id (mt/id)+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)+ table-id (mt/id :transforms_products)+ checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)+ make-source (fn [query-sql]+ {:type "query"+ :query {:database db-id+ :type :native+ :native {:query query-sql+ :template-tags {"source_table" {:id "source_table"+ :name "source_table"+ :display-name "Source Table"+ :type "table"+ :table-id table-id+ :required true}}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id checkpoint-field-id}})+ source (make-source "SELECT id FROM {{source_table}} AS s")+ target {:type "table-incremental"+ :schema "public"+ :name target-table+ :database db-id+ :target-incremental-strategy {:type "append"}}]+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]+ (testing "initial run processes all rows"+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= 16 (count (pg-table-rows db-spec target-table)))))+ (testing "second run with no new data"+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= 16 (count (pg-table-rows db-spec target-table)))))+ (t2/update! :model/Transform+ (:id transform)+ {:source (make-source "SELECT id FROM {{source_table}} AS s ORDER BY id DESC")})+ (testing "run after query change with no new data keeps same row count"+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform)) {:run-method :manual})+ (is (= 16 (count (pg-table-rows db-spec target-table)))+ "Checkpoint preserved across query change, no new rows added"))+ (testing "after inserting new data, incremental run appends"+ (with-insert-test-products!+ [{:name "New Product" :category "Test" :price 99.99 :created-at "2024-01-21T10:00:00"}]+ (let [transform (t2/select-one :model/Transform (:id transform))]+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= 17 (count (pg-table-rows db-spec target-table)))+ "Should append 1 new row after query change")))))))))))++(deftest native-query-with-table-tag-test+ (testing "Native query with table tag uses subquery expansion for incremental filtering"+ (doseq [checkpoint-type [:integer :float]]+ (testing (format "with %s checkpoint" (name checkpoint-type))+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}(mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table (target-table-gen "incremental_index")]+ (with-transform-cleanup! [target-table (target-table-gen "native_table_tag")](let [checkpoint-config (get checkpoint-configs checkpoint-type)- transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]+ {:keys [field-name expected-initial-checkpoint]} checkpoint-config+ checkpoint-filter-field-id (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products))+ transform-payload {:name "Native With Table Tag"+ :source {:type "query"+ :query (make-incremental-source-query-with-table-tag checkpoint-config)+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id checkpoint-filter-field-id}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "append"}})}](mt/with-temp [:model/Transform transform transform-payload]- (testing "First run creates index on checkpoint column"- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)- field-name (:field-name checkpoint-config)]- (testing "Index was created"- (is (= 1 (count indexes)))- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes))))))- (testing "Data was processed correctly"- (is (= 10 (get-table-row-count target-table))))- (testing "Second run succeeds with existing index"- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)- field-name (:field-name checkpoint-config)]- (testing "Index still exists"- (is (= 1 (count indexes)))- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))- (testing "Data was processed correctly"- (is (= 16 (get-table-row-count target-table)))))))))))))))--(deftest index-cleanup-on-switch-to-non-incremental-test- (testing "Switching to non-incremental removes metabase-owned indexes"- (doseq [checkpoint-type [:integer]- transform-type [:native :mbql :python]- :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]- (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))- (mt/with-premium-features #{:transforms-basic :transforms-python}- (mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table (target-table-gen "index_cleanup_non_incr")]- (let [checkpoint-config (get checkpoint-configs checkpoint-type)- {:keys [field-name]} checkpoint-config- incremental-payload (make-incremental-transform-payload "Index Cleanup Transform" target-table transform-type checkpoint-config)]- (mt/with-temp [:model/Transform transform incremental-payload]- (testing "First incremental run creates index"- (execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))- (testing "Switch to non-incremental via API"- (let [non-incremental-payload (-> incremental-payload- (update :source dissoc :source-incremental-strategy))- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))- non-incremental-payload)]- (is (nil? (:source-incremental-strategy (:source updated))))))- (testing "Non-incremental run removes automatic indexes indexes"+ (testing "First run processes all data"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (let [row-count (get-table-row-count target-table)+ distinct-timestamps (get-distinct-timestamp-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 16 row-count) "First run should process all 16 products")+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)+ (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))++ (testing "Second run without new data adds nothing"(let [transform (t2/select-one :model/Transform (:id transform))]- (execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)- mb-indexes (filter #(str/starts-with? (:index-name %) "mb_transform_idx_") indexes)]- (testing "Automatic indexes are dropped"- (is (empty? mb-indexes))))))))))))))))--(deftest index-cleanup-on-checkpoint-column-change-test- (testing "Changing checkpoint column removes old index and creates new one"- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))- (mt/with-premium-features #{:transforms-basic}- (mt/dataset transforms-dataset/transforms-test- (with-transform-cleanup! [target-table (target-table-gen "icchange")]- (let [integer-config (get checkpoint-configs :integer)- float-config (get checkpoint-configs :float)- initial-field (:field-name integer-config)- new-field (:field-name float-config)- transform-type :native- initial-payload (make-incremental-transform-payload "Column Change Transform" target-table transform-type integer-config)]- (mt/with-temp [:model/Transform transform initial-payload]- (testing "First run with integer checkpoint creates index on id column"- (execute-transform-with-ordering! transform transform-type initial-field {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]- (is (= 1 (count indexes)))- (is (=? {:value initial-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))- (testing "Switch checkpoint column via API"- (let [new-payload (make-incremental-transform-payload "Column Change Transform" target-table transform-type float-config)- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))- new-payload)]- (is (= new-field (-> updated :source :source-incremental-strategy :checkpoint-filter)))))- (testing "Run with new checkpoint column updates indexes"- (let [transform (t2/select-one :model/Transform (:id transform))]- (execute-transform-with-ordering! transform transform-type new-field {:run-method :manual})- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]- (testing "Old index removed, new checkpoint column is now indexed"- (is (= 1 (count indexes)))- (is (=? {:value new-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))))))))))))+ (transforms.execute/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)]+ (is (= 16 row-count) "Should still have 16 rows, no new data"))))++ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)+ (not= driver/*driver* :clickhouse)+ (not= driver/*driver* :snowflake))+ (testing "After inserting new data, incremental run appends only new rows"+ (with-insert-test-products!+ [{:name "New Table Tag Product"+ :category "Electronics"+ :price 299.99+ :created-at "2024-01-21T10:00:00"}]++ (let [transform (t2/select-one :model/Transform (:id transform))]+ (transforms.execute/execute! transform {:run-method :manual})+ (let [row-count (get-table-row-count target-table)+ checkpoint (get-checkpoint-value (:id transform))]+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))diff --git a/frontend/src/metabase-types/analytics/event.ts b/frontend/src/metabase-types/analytics/event.tsindex e60429e6d53..d7fd2cbf4d8 100644--- a/frontend/src/metabase-types/analytics/event.ts+++ b/frontend/src/metabase-types/analytics/event.ts@@ -298,6 +298,7 @@ export type TransformCreateEvent = ValidateEvent<{export type TransformCreatedEvent = ValidateEvent<{event: "transform_created";target_id: number;+ event_detail: "incremental" | undefined;}>;export type TransformRunTagsUpdated = ValidateEvent<{diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.tsindex 11390fde65b..80df541c1f0 100644--- a/frontend/src/metabase-types/api/transform.ts+++ b/frontend/src/metabase-types/api/transform.ts@@ -23,12 +23,14 @@ export type TransformOwner = Pick<"id" | "email" | "first_name" | "last_name">;+export type TransformType = "native" | "python" | "mbql";+export type Transform = {id: TransformId;name: string;description: string | null;source: TransformSource;- source_type: "native" | "python" | "mbql";+ source_type: TransformType;target: TransformTarget;collection_id: CollectionId | null;created_at: string;@@ -46,6 +48,8 @@ export type Transform = {owner_email?: string | null;owner?: TransformOwner | null;+ last_checkpoint_value?: string | null;+// hydrated fieldscollection?: Collection | null;tag_ids?: TransformTagId[];@@ -61,10 +65,7 @@ export type PythonTransformTableAliases = Record<string, ConcreteTableId>;export type TransformSourceCheckpointStrategy = {type: "checkpoint";- // For native queries- "checkpoint-filter"?: string;- // For MBQL and Python queries- "checkpoint-filter-unique-key"?: string;+ "checkpoint-filter-field-id": number;};export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;@@ -132,6 +133,10 @@ export type TransformRun = {message: string | null;run_method: TransformRunMethod;+ checkpoint_filter_field_id?: number | null;+ checkpoint_lo_value?: string | null;+ checkpoint_hi_value?: string | null;+// hydrated fieldstransform?: Transform;};@@ -293,21 +298,6 @@ export type UpdatePythonLibraryRequest = {source: string;};-export type ExtractColumnsFromQueryRequest = {- query: DatasetQuery;-};--export type ExtractColumnsFromQueryResponse = {- columns: string[];-};--export type CheckQueryComplexityRequest = string;--export type QueryComplexity = {- is_simple: boolean;- reason: string;-};-export type InspectorFieldStats = {distinct_count?: number;nil_percent?: number;diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.tsindex d7e3f760390..f3813f04520 100644--- a/frontend/src/metabase/api/transform.ts+++ b/frontend/src/metabase/api/transform.ts@@ -1,17 +1,13 @@import { isResourceNotFoundError } from "metabase/lib/errors";import type {- CheckQueryComplexityRequest,CreateTransformRequest,Dataset,- ExtractColumnsFromQueryRequest,- ExtractColumnsFromQueryResponse,GetInspectorLensRequest,InspectorDiscoveryResponse,InspectorLens,ListTransformRunsRequest,ListTransformRunsResponse,ListTransformsRequest,- QueryComplexity,RunInspectorQueryRequest,RunTransformResponse,Transform,@@ -202,25 +198,13 @@ export const transformApi = Api.injectEndpoints({invalidatesTags: (_, error) =>invalidateTags(error, [listTag("transform"), listTag("table")]),}),- extractColumnsFromQuery: builder.mutation<- ExtractColumnsFromQueryResponse,- ExtractColumnsFromQueryRequest- >({- query: (body) => ({- method: "POST",- url: "/api/transform/extract-columns",- body,- }),- }),- checkQueryComplexity: builder.query<- QueryComplexity,- CheckQueryComplexityRequest- >({- query: (queryString) => ({+ resetCheckpoint: builder.mutation<void, TransformId>({+ query: (id) => ({method: "POST",- url: "/api/transform/is-simple-query",- body: { query: queryString },+ url: `/api/transform/${id}/reset-checkpoint`,}),+ invalidatesTags: (_, error, id) =>+ invalidateTags(error, [idTag("transform", id)]),}),getInspectorDiscovery: builder.query<InspectorDiscoveryResponse,@@ -264,7 +248,6 @@ export const {useUpdateTransformMutation,useDeleteTransformMutation,useDeleteTransformTargetMutation,- useExtractColumnsFromQueryMutation,- useLazyCheckQueryComplexityQuery,+ useResetCheckpointMutation,useRunInspectorQueryQuery,} = transformApi;diff --git a/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx b/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsxindex 1fbc7cce6b9..5814864a204 100644--- a/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx+++ b/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx@@ -10,6 +10,8 @@ type FormInlineUpdaterProps<T, TSuccess> = {onSuccess?: (value: TSuccess) => void;onError?: (error: unknown) => void;debounceMs?: number;+ /** If provided, called before each update. Return false to skip the update. */+ onBeforeUpdate?: (values: T) => Promise<boolean> | boolean;};const DEFAULT_INLINE_UPDATE_DEBOUNCE_MS = 300;@@ -23,6 +25,7 @@ const DEFAULT_INLINE_UPDATE_DEBOUNCE_MS = 300;* @param onSuccess - Optional callback called with the update result on success* @param onError - Optional callback called with the error on failure* @param debounceMs - Delay in ms before calling update (default: 300)+ * @param onBeforeUpdate - Optional callback called before each update. Return false to skip the update.** @example* ```tsx@@ -39,6 +42,7 @@ export const FormInlineUpdater = <T, TSuccess>({onSuccess,onError,debounceMs = DEFAULT_INLINE_UPDATE_DEBOUNCE_MS,+ onBeforeUpdate,}: FormInlineUpdaterProps<T, TSuccess>) => {const { initialValues } = useFormikContext<T>();const updateInProgress = useRef(false);@@ -85,6 +89,13 @@ export const FormInlineUpdater = <T, TSuccess>({return;}+ if (onBeforeUpdate) {+ const shouldProceed = await onBeforeUpdate(values);+ if (!shouldProceed) {+ return;+ }+ }+await processUpdate(values);}, debounceMs);diff --git a/frontend/src/metabase/transforms/analytics.ts b/frontend/src/metabase/transforms/analytics.tsindex c9435bf6851..86843c604c6 100644--- a/frontend/src/metabase/transforms/analytics.ts+++ b/frontend/src/metabase/transforms/analytics.ts@@ -43,12 +43,15 @@ export function trackTransformCreate({export function trackTransformCreated({transformId,+ isIncremental,}: {transformId: TransformId;+ isIncremental: boolean;}) {trackSimpleEvent({event: "transform_created",target_id: transformId,+ event_detail: isIncremental ? "incremental" : undefined,});}diff --git a/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsx b/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsxnew file mode 100644index 00000000000..1225926a064--- /dev/null+++ b/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsx@@ -0,0 +1,24 @@+import { DateTime } from "metabase/common/components/DateTime";+import { Tooltip } from "metabase/ui";+import { isDate } from "metabase-lib/v1/types/utils/isa";+import type { Field } from "metabase-types/api";++type CheckpointValueProps = {+ value: string;+ checkpointField?: Field;+};++export function CheckpointValue({+ value,+ checkpointField,+}: CheckpointValueProps) {+ if (isDate(checkpointField)) {+ return (+ <Tooltip label={value}>+ <DateTime value={value} unit="minute" />+ </Tooltip>+ );+ }++ return <>{value}</>;+}diff --git a/frontend/src/metabase/transforms/components/CheckpointValue/index.ts b/frontend/src/metabase/transforms/components/CheckpointValue/index.tsnew file mode 100644index 00000000000..8c09a0a4a4e--- /dev/null+++ b/frontend/src/metabase/transforms/components/CheckpointValue/index.ts@@ -0,0 +1 @@+export { CheckpointValue } from "./CheckpointValue";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsxindex 6450508e0d1..fcc150acd41 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx@@ -1,5 +1,4 @@import { useFormikContext } from "formik";-import { match } from "ts-pattern";import { t } from "ttag";import { useDocsUrl } from "metabase/common/hooks";@@ -12,7 +11,7 @@ import {SOURCE_STRATEGY_OPTIONS,TARGET_STRATEGY_OPTIONS,} from "metabase/transforms/constants";-import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";+import { getLibQuery } from "metabase/transforms/utils";import {Anchor,Box,@@ -23,15 +22,17 @@ import {Text,Tooltip,} from "metabase/ui";-import type * as Lib from "metabase-lib";+import * as Lib from "metabase-lib";import type { TransformSource } from "metabase-types/api";+import type { TransformType } from "metabase-types/api/transform";import {MBQLKeysetColumnSelect,+ NativeQueryTableTagFieldSelect,PythonKeysetColumnSelect,} from "./KeysetColumnSelect";-import { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";import type { IncrementalSettingsFormValues } from "./form";+import { useHasCheckpointOptions } from "./useHasCheckpointOptions";type IncrementalTransformSettingsProps = {source: TransformSource;@@ -39,6 +40,7 @@ type IncrementalTransformSettingsProps = {onIncrementalChange: (value: boolean) => void;variant?: "embedded" | "standalone";readOnly?: boolean;+ extraActions?: React.ReactNode;};export const IncrementalTransformSettings = ({@@ -47,6 +49,7 @@ export const IncrementalTransformSettings = ({onIncrementalChange,variant = "embedded",readOnly,+ extraActions,}: IncrementalTransformSettingsProps) => {const metadata = useSelector(getMetadata);const libQuery = getLibQuery(source, metadata);@@ -54,42 +57,55 @@ export const IncrementalTransformSettings = ({PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,);- // Check if this is a Python transform with exactly one source table- // Incremental transforms are only supported for single-table Python transforms- const isPythonTransform = source.type === "python";+ const { hasCheckpointOptions, transformType } =+ useHasCheckpointOptions(source);+const isMultiTablePythonTransform =- isPythonTransform && Object.keys(source["source-tables"]).length > 1;+ getIsPythonTransformWithMultipleTables(source);+ const isNativeWithoutTableTags = getIsNativeWithoutTableVariables(+ libQuery,+ transformType,+ );- const transformType = match({- isMbqlQuery: isMbqlQuery(source, metadata),- isPythonTransform,- })- .with({ isMbqlQuery: true }, () => "query" as const)- .with({ isPythonTransform: true }, () => "python" as const)- .otherwise(() => "native" as const);const { url: incrementalTransformsDocsUrl, showMetabaseLinks } = useDocsUrl(- isPythonTransform+ transformType === "python"? "data-studio/transforms/python-transforms#incremental-python-transforms": "data-studio/transforms/query-transforms#incremental-query-transforms",);const renderIncrementalSwitch = () => {+ const getLabel = () => {+ if (isMultiTablePythonTransform) {+ return t`Incremental transforms are only supported for single data source transforms.`;+ }+ if (isNativeWithoutTableTags) {+ return t`Incremental transforms for native queries require at least one table variable.`;+ }+ if (!hasCheckpointOptions) {+ return t`Incremental transforms require at least one numeric or temporal source field.`;+ }+ return t`Only process new and changed data`;+ };++ const transformHasIssues =+ isNativeWithoutTableTags ||+ !hasCheckpointOptions ||+ isMultiTablePythonTransform;+const switchContent = (<Switchdisabled={- readOnly || isMultiTablePythonTransform || isRemoteSyncReadOnly+ readOnly ||+ isRemoteSyncReadOnly ||+ (!incremental && transformHasIssues)}checked={incremental}size="sm"- label={- isMultiTablePythonTransform- ? t`Incremental transforms are only supported for single data source transforms.`- : t`Only process new and changed data`- }+ label={getLabel()}wrapperProps={{"data-testid": "incremental-switch",}}- onChange={(e) => onIncrementalChange(e.target.checked)}+ onChange={(event) => onIncrementalChange(event.target.checked)}/>);@@ -109,18 +125,21 @@ export const IncrementalTransformSettings = ({const label = t`Incremental transformation`;const renderDescription = () => {- const description = t`If you don’t need to reprocess all the data, incremental transforms can be faster. To use this, your transform definition should have a stable schema. `;+ const description = t`If you don’t need to reprocess all the data, incremental transforms can be faster. To use this, your transform definition should have a stable schema.`;return (<>{description}{showMetabaseLinks && (- <Anchor- href={incrementalTransformsDocsUrl}- target="_blank"- td="underline"- c="inherit"- size="sm"- >{t`Learn more.`}</Anchor>+ <>+ {" "}+ <Anchor+ href={incrementalTransformsDocsUrl}+ target="_blank"+ td="underline"+ c="inherit"+ size="sm"+ >{t`Learn more.`}</Anchor>+ </>)}{" "}</>);@@ -136,10 +155,16 @@ export const IncrementalTransformSettings = ({<SourceStrategyFieldssource={source}query={libQuery}- type={transformType}+ transformType={transformType}readOnly={readOnly}/></Group>+ {extraActions && (+ <>+ <Divider />+ <Group p="lg">{extraActions}</Group>+ </>+ )}<TargetStrategyFields variant={variant} /></>)}@@ -161,7 +186,7 @@ export const IncrementalTransformSettings = ({<SourceStrategyFieldssource={source}query={libQuery}- type={transformType}+ transformType={transformType}/><TargetStrategyFields variant={variant} /></>@@ -205,17 +230,18 @@ function TargetStrategyFields({type SourceStrategyFieldsProps = {source: TransformSource;query: Lib.Query | null;- type: "query" | "native" | "python";+ transformType: TransformType;readOnly?: boolean;};function SourceStrategyFields({source,query,- type,+ transformType,readOnly,}: SourceStrategyFieldsProps) {const { values } = useFormikContext<IncrementalSettingsFormValues>();+return (<>{SOURCE_STRATEGY_OPTIONS.length > 1 && (@@ -229,9 +255,9 @@ function SourceStrategyFields({)}{values.sourceStrategy === "checkpoint" && (<>- {type === "query" && query && (+ {transformType === "mbql" && query && (<MBQLKeysetColumnSelect- name="checkpointFilterUniqueKey"+ name="checkpointFilterFieldId"label={t`Field to check for new values`}placeholder={t`Pick a field`}description={t`Pick the field that we should scan to determine which records are new or changed`}@@ -241,20 +267,20 @@ function SourceStrategyFields({disabled={readOnly}/>)}- {type === "native" && query && (- <NativeQueryColumnSelect- name="checkpointFilter"- label={t`Column to check for new values`}- placeholder={t`Pick a column`}- description={t`Pick the column that we should scan to determine which records are new or changed`}+ {transformType === "native" && query && (+ <NativeQueryTableTagFieldSelect+ name="checkpointFilterFieldId"+ label={t`Field to check for new values`}+ placeholder={t`Pick a field`}+ description={t`Pick the field that we should scan to determine which records are new or changed`}descriptionProps={{ lh: "1rem" }}query={query}disabled={readOnly}/>)}- {type === "python" && "source-tables" in source && (+ {transformType === "python" && "source-tables" in source && (<PythonKeysetColumnSelect- name="checkpointFilterUniqueKey"+ name="checkpointFilterFieldId"label={t`Field to check for new values`}placeholder={t`Pick a field`}description={t`Pick the field that we should scan to determine which records are new or changed`}@@ -268,3 +294,30 @@ function SourceStrategyFields({</>);}++// Check if this is a Python transform with exactly one source table+// Incremental transforms are only supported for single-table Python transforms+function getIsPythonTransformWithMultipleTables(source: TransformSource) {+ const isPythonTransform = source.type === "python";+ const isMultiTablePythonTransform =+ isPythonTransform && Object.keys(source["source-tables"]).length > 1;++ return isMultiTablePythonTransform;+}++function getIsNativeWithoutTableVariables(+ query: Lib.Query | null,+ transformType: TransformType,+) {+ return transformType === "native" && !queryHasTableVariables(query);+}++function queryHasTableVariables(query: Lib.Query | null) {+ const hasTableTemplateTags = query+ ? Object.values(Lib.templateTags(query)).some(+ (tag) => tag.type === "table" && tag["table-id"] != null,+ )+ : false;++ return hasTableTemplateTags;+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsxnew file mode 100644index 00000000000..aeb92dcb302--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx@@ -0,0 +1,111 @@+import { screen } from "@testing-library/react";++import { renderWithProviders } from "__support__/ui";+import { FormProvider } from "metabase/forms";+import type { TransformSource } from "metabase-types/api";+import { createMockTransformSource } from "metabase-types/api/mocks";++import { IncrementalTransformSettings } from "./IncrementalTransformSettings";+import { getInitialValues } from "./form";++jest.mock("./useHasCheckpointOptions", () => ({+ useHasCheckpointOptions: jest.fn(),+}));++jest.mock("metabase/transforms/utils", () => ({+ getLibQuery: () => null,+ isMbqlQuery: () => false,+}));++const { useHasCheckpointOptions } = jest.requireMock(+ "./useHasCheckpointOptions",+);++type SetupOpts = {+ incremental?: boolean;+ source?: TransformSource;+ onIncrementalChange?: () => void;+ hookOverrides?: Partial<ReturnType<typeof useHasCheckpointOptions>>;+};++function setup({+ incremental = false,+ source = createMockTransformSource(),+ onIncrementalChange = jest.fn(),+ hookOverrides = {},+}: SetupOpts = {}) {+ const hookDefaults = {+ hasCheckpointOptions: true,+ hasNativeCheckpointOptions: true,+ transformType: "query" as const,+ ...hookOverrides,+ };++ useHasCheckpointOptions.mockReturnValue(hookDefaults);++ const initialValues = getInitialValues({ incremental });++ renderWithProviders(+ <FormProvider initialValues={initialValues} onSubmit={jest.fn()}>+ <IncrementalTransformSettings+ source={source}+ incremental={incremental}+ onIncrementalChange={onIncrementalChange}+ />+ </FormProvider>,+ );+}++function getSwitch() {+ return screen.getByRole("switch");+}++describe("IncrementalTransformSettings", () => {+ it("disables switch when hasCheckpointOptions is false and incremental is off", () => {+ setup({+ incremental: false,+ hookOverrides: {+ transformType: "mbql",+ hasCheckpointOptions: false,+ },+ });++ expect(getSwitch()).toBeDisabled();+ });++ it("enables switch when hasCheckpointOptions is false but incremental is on", () => {+ setup({+ incremental: true,+ hookOverrides: {+ transformType: "native",+ hasCheckpointOptions: false,+ },+ });++ expect(getSwitch()).toBeEnabled();+ });++ it("disables switch when hasNativeCheckpointOptions is false and incremental is off", () => {+ setup({+ incremental: false,+ hookOverrides: {+ transformType: "native",+ hasNativeCheckpointOptions: false,+ },+ });++ expect(getSwitch()).toBeDisabled();+ });++ it("enables switch when hasNativeCheckpointOptions is false but incremental is on", () => {+ setup({+ incremental: true,+ hookOverrides: {+ transformType: "native",+ hasNativeCheckpointOptions: false,+ },+ });++ expect(getSwitch()).toBeEnabled();+ });+});diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsxindex 5be50529d22..b45618cb29b 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx@@ -9,6 +9,58 @@ import {} from "metabase/ui";import * as Lib from "metabase-lib";+import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";++/**+ * Get filterable numeric/temporal field options from the source table of a query.+ * Returns SelectOption[] with field IDs as string values.+ */+export function getSourceFieldOptions(+ query: Lib.Query,+ opts?: { labelPrefix?: string; seenFieldIds?: Set<number> },+): Array<SelectOption> {+ // Stage 0 is the source stage (raw table data). Keyset pagination requires columns from the+ // actual source table; later stages may have aggregations, joins, or expressions instead.+ const stageIndex = 0;+ const filterableColumns = Lib.filterableColumns(query, stageIndex);++ const groups = Lib.groupColumns(filterableColumns);+ const sourceGroup = groups.find((group) => {+ const groupInfo = Lib.displayInfo(query, stageIndex, group);+ return groupInfo.isSourceTable;+ });++ if (!sourceGroup) {+ return [];+ }++ const sourceColumns = Lib.getColumnsFromColumnGroup(sourceGroup);+ const seenFieldIds = opts?.seenFieldIds ?? new Set<number>();+ const options: Array<SelectOption> = [];++ for (const column of sourceColumns) {+ if (!Lib.isNumeric(column) && !Lib.isTemporal(column)) {+ continue;+ }++ const { fieldId } = Lib.fieldValuesSearchInfo(query, column);+ if (fieldId == null || seenFieldIds.has(fieldId)) {+ continue;+ }++ seenFieldIds.add(fieldId);++ const columnInfo = Lib.displayInfo(query, stageIndex, column);+ const label = opts?.labelPrefix+ ? `${opts.labelPrefix}: ${columnInfo.displayName}`+ : columnInfo.longDisplayName;++ options.push({ value: String(fieldId), label });+ }++ return options;+}+type KeysetColumnSelectProps = {name: string;label: string;@@ -36,45 +88,8 @@ export function KeysetColumnSelect({}try {- // Use -1 to get the last stage- const stageIndex = -1;-- const returnedColumns = Lib.returnedColumns(query, stageIndex);- const filterableColumns = Lib.filterableColumns(query, stageIndex);-- const filterableIdentifiers = new Set(- filterableColumns.map((col) => {- return Lib.columnKey(col);- }),- );-- const numericFilterableColumns = returnedColumns.filter((column) => {- return (- filterableIdentifiers.has(Lib.columnKey(column)) &&- (Lib.isNumeric(column) || Lib.isTemporal(column))- );- });-- // Convert to select options with unique keys- const seenKeys = new Set<string>();- const uniqueColumns: Array<SelectOption> = [];-- numericFilterableColumns.forEach((column) => {- const columnInfo = Lib.displayInfo(query, stageIndex, column);- const uniqueKey = Lib.columnUniqueKey(column);-- if (!seenKeys.has(uniqueKey)) {- seenKeys.add(uniqueKey);-- const label = columnInfo.longDisplayName;-- uniqueColumns.push({ value: uniqueKey, label });- }- });-- return uniqueColumns;+ return getSourceFieldOptions(query);} catch (error) {- // If we can't extract columns (e.g., invalid query), return empty arrayconsole.error("KeysetColumnSelect: Error extracting columns from query:",error,@@ -83,6 +98,12 @@ export function KeysetColumnSelect({}}, [query]);+ useAutoSelectFirstOption({+ name,+ options: columnOptions,+ disabled: disabled || isLoading,+ });+return (<FormSelectname={name}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsxnew file mode 100644index 00000000000..46260aeda46--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx@@ -0,0 +1,76 @@+import { useMemo } from "react";+import { t } from "ttag";++import { FormSelect } from "metabase/forms";+import {+ Alert,+ type DataAttributes,+ Icon,+ type InputDescriptionProps,+ Loader,+} from "metabase/ui";+import type * as Lib from "metabase-lib";++import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";+import { useNativeCheckpointFieldOptions } from "../useNativeCheckpointFieldOptions";++type NativeQueryTableTagFieldSelectProps = {+ name: string;+ label: string;+ placeholder: string;+ description: React.ReactNode;+ descriptionProps?: InputDescriptionProps & DataAttributes;+ query: Lib.Query;+ disabled?: boolean;+};++export function NativeQueryTableTagFieldSelect({+ name,+ label,+ placeholder,+ description,+ descriptionProps,+ query,+ disabled,+}: NativeQueryTableTagFieldSelectProps) {+ const { fieldOptions, isLoading, tableIds, hasError } =+ useNativeCheckpointFieldOptions(query);++ const noQueryMessage = useMemo(() => {+ if (tableIds.length === 0) {+ return t`Native queries must use at least one table variable to enable incremental transforms`;+ }+ if (hasError) {+ return t`Unable to load table metadata. You may not have permission to access the table.`;+ }+ return null;+ }, [tableIds, hasError]);++ useAutoSelectFirstOption({+ name,+ options: fieldOptions,+ disabled: disabled || isLoading,+ });++ if (noQueryMessage) {+ return (+ <Alert icon={<Icon name="warning" />} color="warning">+ {noQueryMessage}+ </Alert>+ );+ }++ return (+ <FormSelect+ name={name}+ label={label}+ placeholder={placeholder}+ description={description}+ descriptionProps={descriptionProps}+ data={fieldOptions}+ searchable+ disabled={disabled || fieldOptions.length === 0}+ rightSection={isLoading ? <Loader size="xs" /> : undefined}+ />+ );+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.tsindex e9c499b9ff8..0eaf25380fc 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts@@ -1,3 +1,5 @@export { KeysetColumnSelect } from "./KeysetColumnSelect";+export { getSourceFieldOptions } from "./KeysetColumnSelect";export { MBQLKeysetColumnSelect } from "./MBQLKeysetColumnSelect";+export { NativeQueryTableTagFieldSelect } from "./NativeQueryTableTagFieldSelect";export { PythonKeysetColumnSelect } from "./PythonKeysetColumnSelect";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsxdeleted file mode 100644index d4cd9772ad4..00000000000--- a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx+++ /dev/null@@ -1,100 +0,0 @@-import { useEffect, useState } from "react";-import { useLatest } from "react-use";--import { useExtractColumnsFromQueryMutation } from "metabase/api";-import { FormSelect, FormTextInput } from "metabase/forms";-import {- type DataAttributes,- type InputDescriptionProps,- Loader,-} from "metabase/ui";-import * as Lib from "metabase-lib";--type NativeQueryColumnSelectProps = {- name: string;- label: string;- description: React.ReactNode;- descriptionProps?: InputDescriptionProps & DataAttributes;- placeholder: string;- query: Lib.Query;- disabled?: boolean;-};--export function NativeQueryColumnSelect({- name,- label,- description,- descriptionProps,- placeholder,- query,- disabled,-}: NativeQueryColumnSelectProps) {- const [columns, setColumns] = useState<string[] | null>(null);- const [extractColumns, { isLoading }] = useExtractColumnsFromQueryMutation();-- const queryRef = useLatest(query);-- useEffect(() => {- let isCancelled = false;- if (- columns &&- Lib.areLegacyQueriesEqual(- Lib.toLegacyQuery(queryRef.current),- Lib.toLegacyQuery(query),- )- ) {- return;- }-- const extract = async () => {- try {- const result = await extractColumns({- query: Lib.toJsQuery(query),- }).unwrap();-- if (!isCancelled) {- // Only use columns if we got at least one- setColumns(result.columns.length > 0 ? result.columns : null);- }- } catch (error) {- // On error, fall back to text input- if (!isCancelled) {- setColumns(null);- }- }- };-- extract();-- return () => {- isCancelled = true;- };- }, [extractColumns, query, queryRef, columns]);-- // If we successfully extracted columns, show a selector- if (columns && columns.length > 0) {- return (- <FormSelect- name={name}- rightSection={isLoading ? <Loader size="sm" /> : null}- disabled={disabled || isLoading}- label={label}- description={description}- placeholder={placeholder}- data={columns.map((col) => ({ value: col, label: col }))}- />- );- }-- // Otherwise, fall back to text input- return (- <FormTextInput- name={name}- label={label}- placeholder={placeholder}- description={description}- descriptionProps={descriptionProps}- disabled={disabled}- />- );-}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.tsdeleted file mode 100644index b95d8e9d607..00000000000--- a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts+++ /dev/null@@ -1 +0,0 @@-export { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.tsindex 0d63f80046a..a6e2c2a31cb 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts@@ -1,5 +1,6 @@import * as Yup from "yup";+import * as Errors from "metabase/lib/errors";import type {Transform,TransformSource,@@ -10,18 +11,22 @@ import type {export type IncrementalSettingsFormValues = {incremental: boolean;sourceStrategy: "checkpoint";- checkpointFilter: string | null;- checkpointFilterUniqueKey: string | null;+ checkpointFilterFieldId: string | null; // String because Mantine Select requires string valuestargetStrategy: "append";};export const VALIDATION_SCHEMA = Yup.object({incremental: Yup.boolean().required(),sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),- // For native queries, use checkpointFilter (plain string)- checkpointFilter: Yup.string().nullable().defined(),- // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)- checkpointFilterUniqueKey: Yup.string().nullable().defined(),+ checkpointFilterFieldId: Yup.string()+ .nullable()+ .defined()+ .when(["incremental", "sourceStrategy"], {+ is: (incremental: boolean, sourceStrategy: "checkpoint") =>+ incremental && sourceStrategy === "checkpoint",+ then: (schema) => schema.required(Errors.required),+ otherwise: (schema) => schema.nullable().defined(),+ }),targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),});@@ -30,8 +35,7 @@ export const getInitialValues = (): IncrementalSettingsFormValues => ({incremental: false,sourceStrategy: "checkpoint",- checkpointFilter: null,- checkpointFilterUniqueKey: null,+ checkpointFilterFieldId: null,targetStrategy: "append",...defaults,});@@ -42,22 +46,18 @@ export const getIncrementalSettingsFromTransform = (const isIncremental = transform.target.type === "table-incremental";const strategy = transform.source["source-incremental-strategy"];- // Read both fields from the strategy, whichever is present- const checkpointFilter =+ const checkpointFilterFieldId =strategy?.type === "checkpoint"- ? (strategy["checkpoint-filter"] ?? null)- : null;-- const checkpointFilterUniqueKey =- strategy?.type === "checkpoint"- ? (strategy["checkpoint-filter-unique-key"] ?? null)+ ? (strategy["checkpoint-filter-field-id"] ?? null): null;return {incremental: isIncremental,sourceStrategy: "checkpoint",- checkpointFilter: isIncremental ? checkpointFilter : null,- checkpointFilterUniqueKey: isIncremental ? checkpointFilterUniqueKey : null,+ checkpointFilterFieldId:+ isIncremental && checkpointFilterFieldId != null+ ? String(checkpointFilterFieldId)+ : null,targetStrategy: "append",};};@@ -87,19 +87,17 @@ export const buildIncrementalSource = (};}- // Build strategy fields based on which checkpoint field is present- const strategyFields = formValues.checkpointFilter- ? { "checkpoint-filter": formValues.checkpointFilter }- : formValues.checkpointFilterUniqueKey- ? { "checkpoint-filter-unique-key": formValues.checkpointFilterUniqueKey }- : {};-return {...source,- "source-incremental-strategy": {- type: formValues.sourceStrategy,- ...strategyFields,- },+ "source-incremental-strategy":+ formValues.checkpointFilterFieldId != null+ ? {+ type: formValues.sourceStrategy,+ "checkpoint-filter-field-id": Number(+ formValues.checkpointFilterFieldId,+ ),+ }+ : undefined,};};diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.tsnew file mode 100644index 00000000000..27d91e8eccf--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts@@ -0,0 +1,36 @@+import { type IncrementalSettingsFormValues, VALIDATION_SCHEMA } from "./form";++const getValues = (+ values: Partial<IncrementalSettingsFormValues> = {},+): IncrementalSettingsFormValues => ({+ incremental: true,+ sourceStrategy: "checkpoint",+ checkpointFilterFieldId: "123",+ targetStrategy: "append",+ ...values,+});++describe("IncrementalTransform form validation", () => {+ it("requires checkpoint field for incremental transforms", async () => {+ const values = getValues({ checkpointFilterFieldId: null });++ await expect(VALIDATION_SCHEMA.validate(values)).rejects.toMatchObject({+ message: "required",+ });+ });++ it("accepts checkpoint field when incremental transforms are enabled", async () => {+ const values = getValues({ checkpointFilterFieldId: "42" });++ await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);+ });++ it("does not require checkpoint field when incremental transforms are disabled", async () => {+ const values = getValues({+ incremental: false,+ checkpointFilterFieldId: null,+ });++ await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);+ });+});diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/index.tsindex 1f51a5b5a9c..aecd1a27dfe 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts@@ -1,5 +1,4 @@-export * from "./hooks";export * from "./form";export * from "./IncrementalTransformSettings";export * from "./KeysetColumnSelect";-export * from "./NativeQueryColumnSelect";+export { useUpdateIncrementalSettings } from "./useUpdateIncrementalSettings";diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.tsnew file mode 100644index 00000000000..d4cb2ee6e41--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.ts@@ -0,0 +1,36 @@+import { useFormikContext } from "formik";+import { useEffect } from "react";++import type { SelectOption } from "metabase/ui";++type UseAutoSelectFirstOptionProps = {+ name: string;+ options: SelectOption[];+ disabled?: boolean;+};++export function useAutoSelectFirstOption({+ name,+ options,+ disabled,+}: UseAutoSelectFirstOptionProps) {+ const { setFieldValue, values } =+ useFormikContext<Record<string, string | null>>();+ const currentValue = values[name];++ useEffect(() => {+ if (disabled || options.length === 0) {+ return;+ }++ const hasValidCurrentValue =+ currentValue != null &&+ options.some((option) => option.value === currentValue);++ if (hasValidCurrentValue) {+ return;+ }++ setFieldValue(name, options[0].value);+ }, [currentValue, disabled, name, options, setFieldValue]);+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.tsnew file mode 100644index 00000000000..4a784950e54--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts@@ -0,0 +1,88 @@+import { match } from "ts-pattern";++import {+ skipToken,+ useGetAdhocQueryMetadataQuery,+ useGetTableQueryMetadataQuery,+} from "metabase/api";+import { useSelector } from "metabase/lib/redux";+import { getMetadata } from "metabase/selectors/metadata";+import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";+import * as Lib from "metabase-lib";+import type { TransformSource } from "metabase-types/api";+import { isConcreteTableId } from "metabase-types/api/table";++import { getSourceFieldOptions } from "./KeysetColumnSelect";+import { useNativeHasCheckpointFieldOptions } from "./useNativeCheckpointFieldOptions";++export const useHasCheckpointOptions = (source: TransformSource) => {+ const metadata = useSelector(getMetadata);+ const libQuery = getLibQuery(source, metadata);++ // Trigger query metadata fetch for MBQL/native query sources so metadata is populated+ // before we compute hasCheckpointOptions. Without this, getSourceFieldOptions(libQuery)+ // can return [] when the query references a card or needs schema from the API.+ useGetAdhocQueryMetadataQuery(+ source.type === "query" ? source.query : skipToken,+ );++ const isPythonTransform = source.type === "python";+ const transformType = match({+ isMbqlQuery: isMbqlQuery(source, metadata),+ isPythonTransform,+ })+ .with({ isMbqlQuery: true }, () => "mbql" as const)+ .with({ isPythonTransform: true }, () => "python" as const)+ .otherwise(() => "native" as const);++ const hasNativeCheckpointOptions =+ useNativeHasCheckpointFieldOptions(libQuery);++ const pythonTableId = isPythonTransform+ ? Object.values(source["source-tables"]).find(isConcreteTableId)+ : null;+ const { data: pythonTable } = useGetTableQueryMetadataQuery(+ pythonTableId ? { id: pythonTableId } : skipToken,+ );++ const getHasCheckpointOptions = () => {+ try {+ return match(transformType)+ .with("mbql", () =>+ libQuery ? getSourceFieldOptions(libQuery).length > 0 : false,+ )+ .with("python", () => {+ if (!pythonTable?.db_id) {+ return false;+ }++ const metadataProvider = Lib.metadataProvider(+ pythonTable.db_id,+ metadata,+ );+ const tableMetadata = Lib.tableOrCardMetadata(+ metadataProvider,+ pythonTable.id,+ );+ if (!tableMetadata) {+ return false;+ }++ const tableQuery = Lib.queryFromTableOrCardMetadata(+ metadataProvider,+ tableMetadata,+ );+ return getSourceFieldOptions(tableQuery).length > 0;+ })+ .with("native", () => hasNativeCheckpointOptions)+ .exhaustive();+ } catch {+ return false;+ }+ };++ return {+ hasCheckpointOptions: getHasCheckpointOptions(),+ transformType,+ };+};diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.tsnew file mode 100644index 00000000000..a23834b79ce--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts@@ -0,0 +1,90 @@+import { useMemo } from "react";++import { useSelector } from "metabase/lib/redux";+import { getMetadata } from "metabase/selectors/metadata";+import type { SelectOption } from "metabase/ui";+import * as Lib from "metabase-lib";+import { isConcreteTableId } from "metabase-types/api/table";++import { getSourceFieldOptions } from "./KeysetColumnSelect/KeysetColumnSelect";+import { useTableQueryMetadataResults } from "./useTableQueryMetadataResults";++export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {+ const metadata = useSelector(getMetadata);++ const tableIds = useMemo(() => {+ if (!query) {+ return [];+ }+ try {+ const templateTags = Lib.templateTags(query);+ const tableTags = Object.values(templateTags).filter(+ (tag) => tag.type === "table" && tag["table-id"] != null,+ );+ return tableTags.map((tag) => tag["table-id"]).filter(isConcreteTableId);+ } catch {+ return [];+ }+ }, [query]);++ const { tables, isLoading, hasError } =+ useTableQueryMetadataResults(tableIds);++ const fieldOptions = useMemo((): Array<SelectOption> => {+ if (tables.length === 0) {+ return [];+ }++ try {+ const allOptions: Array<SelectOption> = [];+ const seenFieldIds = new Set<number>();+ const showTablePrefix = tables.length > 1;++ for (const table of tables) {+ const metadataProvider = Lib.metadataProvider(table.db_id, metadata);+ const tableMetadata = Lib.tableOrCardMetadata(+ metadataProvider,+ table.id,+ );+ if (!tableMetadata) {+ continue;+ }++ const tableQuery = Lib.queryFromTableOrCardMetadata(+ metadataProvider,+ tableMetadata,+ );++ const options = getSourceFieldOptions(tableQuery, {+ labelPrefix: showTablePrefix+ ? table.display_name || table.name+ : undefined,+ seenFieldIds,+ });++ allOptions.push(...options);+ }++ return allOptions;+ } catch (error) {+ console.error(+ "NativeQueryTableTagFieldSelect: Error extracting fields:",+ error,+ );+ return [];+ }+ }, [tables, metadata]);++ return {+ fieldOptions,+ isLoading,+ tableIds,+ hasError,+ };+}++export function useNativeHasCheckpointFieldOptions(query: Lib.Query | null) {+ const { fieldOptions } = useNativeCheckpointFieldOptions(query);++ return fieldOptions.length > 0;+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.tsnew file mode 100644index 00000000000..02222efe574--- /dev/null+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.ts@@ -0,0 +1,40 @@+import { useEffect, useMemo } from "react";++import { tableApi } from "metabase/api";+import { useDispatch, useSelector } from "metabase/lib/redux";+import type { Table } from "metabase-types/api";++const selectTableQueryMetadata =+ tableApi.endpoints.getTableQueryMetadata.select;++/**+ * Fetches table query metadata for a dynamic list of table IDs.+ * Uses RTK Query's initiate/select pattern to avoid the "hooks in a loop" problem.+ */+export function useTableQueryMetadataResults(tableIds: number[]) {+ const dispatch = useDispatch();++ useEffect(() => {+ const subscriptions = tableIds.map((id) =>+ dispatch(tableApi.endpoints.getTableQueryMetadata.initiate({ id })),+ );+ return () => {+ subscriptions.forEach((sub) => sub.unsubscribe());+ };+ }, [dispatch, tableIds]);++ const selectors = useMemo(+ () => tableIds.map((id) => selectTableQueryMetadata({ id })),+ [tableIds],+ );++ return useSelector((state) => {+ const results = selectors.map((sel) => sel(state));+ const isLoading = results.some((r) => r.isLoading);+ const hasError = results.some((r) => r.isError);+ const tables = results+ .map((r) => r.data)+ .filter((t): t is Table => t != null);+ return { isLoading, hasError, tables };+ });+}diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/hooks.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.tssimilarity index 91%rename from frontend/src/metabase/transforms/components/IncrementalTransform/hooks.tsrename to frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.tsindex 3c8ee1f1c3a..38fdb3a59ac 100644--- a/frontend/src/metabase/transforms/components/IncrementalTransform/hooks.ts+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts@@ -10,7 +10,7 @@ import {getIncrementalSettingsFromTransform,} from "./form";-export const useUpdateIncrementalSettings = (transform: Transform) => {+export function useUpdateIncrementalSettings(transform: Transform) {const [updateTransform] = useUpdateTransformMutation();const initialValues = useMemo(() => getIncrementalSettingsFromTransform(transform),@@ -29,4 +29,4 @@ export const useUpdateIncrementalSettings = (transform: Transform) => {validationSchema: VALIDATION_SCHEMA,updateIncrementalSettings,};-};+}diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css b/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.cssdeleted file mode 100644index 86f3a93b537..00000000000--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css+++ /dev/null@@ -1,6 +0,0 @@-.bordered {- background: var(--mb-color-background-secondary);- border-radius: var(--mantine-radius-md);- padding: var(--mantine-spacing-md);- border: 1px solid var(--mb-color-border);-}diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx b/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsxdeleted file mode 100644index d99543e2305..00000000000--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx+++ /dev/null@@ -1,39 +0,0 @@-import cx from "classnames";-import { t } from "ttag";--import { CHECKPOINT_TEMPLATE_TAG } from "metabase/transforms/constants";-import { Stack, Text } from "metabase/ui";--import S from "./QueryComplexityWarning.module.css";--type QueryComplexityWarningProps = {- variant?: "default" | "standout";-};--export const QueryComplexityWarning = ({- variant = "default",-}: QueryComplexityWarningProps) => {- return (- <Stack- gap={variant === "standout" ? "sm" : "md"}- className={cx({ [S.bordered]: variant === "standout" })}- data-testid="query-complexity-warning"- >- {variant === "standout" && (- <Text fw="bold">{t`We couldn’t add the check automatically.`}</Text>- )}- <Text lh="1.25rem">{t`This query is too complicated for us to automatically add a checkpoint. You can turn off incremental transformation, or add an explicit filter to the query, like this:`}</Text>- <Text- lh="1.25rem"- py="md"- className={cx({ [S.bordered]: variant === "default" })}- >- <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>- </Text>- <Text- lh="1.25rem"- c="saturated-red"- >{t`If you save anyway, this transform will likely not process new data, or might only process partial quantities of new data.`}</Text>- </Stack>- );-};diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.ts b/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.tsdeleted file mode 100644index 638c5b5972a..00000000000--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.ts+++ /dev/null@@ -1,2 +0,0 @@-export { QueryComplexityWarning } from "./QueryComplexityWarning";-export { useQueryComplexityChecks } from "./use-query-complexity-checks";diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx b/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsxdeleted file mode 100644index f7f5142ce5e..00000000000--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx+++ /dev/null@@ -1,78 +0,0 @@-import { useCallback } from "react";-import { t } from "ttag";--import { useLazyCheckQueryComplexityQuery } from "metabase/api";-import { useConfirmation } from "metabase/common/hooks";-import { useSelector } from "metabase/lib/redux";-import { getMetadata } from "metabase/selectors/metadata";-import { CHECKPOINT_TEMPLATE_TAG } from "metabase/transforms/constants";-import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";-import * as Lib from "metabase-lib";-import type Metadata from "metabase-lib/v1/metadata/Metadata";-import type { DraftTransformSource } from "metabase-types/api";--import { QueryComplexityWarning } from "./QueryComplexityWarning";--const getQueryForComplexityCheck = (- source: DraftTransformSource,- metadata: Metadata,-): Lib.Query | undefined => {- const query = getLibQuery(source, metadata);- if (!query || isMbqlQuery(source, metadata)) {- return;- }-- const tags = Lib.templateTags(query);- if (CHECKPOINT_TEMPLATE_TAG in tags) {- return;- }-- return query;-};--export const useQueryComplexityChecks = () => {- const [checkQueryComplexity] = useLazyCheckQueryComplexityQuery();- const metadata = useSelector(getMetadata);- const { modalContent: modal, show } = useConfirmation();-- const checkComplexity = useCallback(- async (source: DraftTransformSource) => {- const query = getQueryForComplexityCheck(source, metadata);- if (!query) {- return;- }- const rawQuery = Lib.rawNativeQuery(query);- const complexity = await checkQueryComplexity(rawQuery, true).unwrap();- if (complexity.is_simple) {- return;- }- return complexity;- },- [metadata, checkQueryComplexity],- );-- const confirmIfQueryIsComplex = useCallback(- async (- source: DraftTransformSource,- confirmButtonText: string = t`Save anyway`,- ) => {- const complexity = await checkComplexity(source);- if (!complexity) {- return true;- }- return new Promise<boolean>((resolve) => {- show({- title: t`Can't automatically run this transform incrementally`,- message: <QueryComplexityWarning />,- onConfirm: () => resolve(true),- onCancel: () => resolve(false),- confirmButtonText,- confirmButtonProps: { color: "saturated-red", variant: "filled" },- });- });- },- [checkComplexity, show],- );-- return { confirmIfQueryIsComplex, checkComplexity, modal };-};diff --git a/frontend/src/metabase/transforms/constants.ts b/frontend/src/metabase/transforms/constants.tsindex 18107133265..d3604777372 100644--- a/frontend/src/metabase/transforms/constants.ts+++ b/frontend/src/metabase/transforms/constants.ts@@ -9,10 +9,9 @@ export const POLLING_INTERVAL = isCypressActive ? 200 : 3000;export const FILTER_WIDGET_MIN_WIDTH = 300;export const FILTER_WIDGET_MAX_HEIGHT = 400;-export const CHECKPOINT_TEMPLATE_TAG = "checkpoint";export const SOURCE_STRATEGY_OPTIONS = [{- value: CHECKPOINT_TEMPLATE_TAG,+ value: "checkpoint",get label() {return t`Checkpoint`;},diff --git a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.tsindex 7dac5342e17..e035d7601c4 100644--- a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts+++ b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts@@ -1,4 +1,4 @@-import { useState } from "react";+import { useEffect, useRef, useState } from "react";import { skipToken, useGetTransformQuery } from "metabase/api";import type { Transform } from "metabase-types/api";@@ -18,11 +18,13 @@ const isPollingNeeded = (transform?: Transform) =>export const useTransformWithPolling = (transformId: number | undefined) => {const [isPolling, setIsPolling] = useState(false);+ const wasPolling = useRef(false);const {data: transform,isLoading,error,+ refetch,} = useGetTransformQuery(transformId ?? skipToken, {pollingInterval: isPolling ? POLLING_INTERVAL : undefined,});@@ -31,5 +33,15 @@ export const useTransformWithPolling = (transformId: number | undefined) => {setIsPolling(isPollingNeeded(transform));}+ // When polling stops (run just finished), do a delayed refetch to pick up+ // post-execution changes like the updated checkpoint watermark.+ useEffect(() => {+ if (wasPolling.current && !isPolling) {+ const timer = setTimeout(() => refetch(), POLLING_INTERVAL);+ return () => clearTimeout(timer);+ }+ wasPolling.current = isPolling;+ }, [isPolling, refetch]);+return { transform, isLoading, error };};diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsxindex 567430118d6..c3cc44322aa 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx@@ -1,5 +1,5 @@import { useFormikContext } from "formik";-import { useMemo, useState } from "react";+import { useMemo } from "react";import { t } from "ttag";import _ from "underscore";import type * as Yup from "yup";@@ -20,13 +20,8 @@ import {FormTextInput,} from "metabase/forms";import { IncrementalTransformSettings } from "metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings";-import {- QueryComplexityWarning,- useQueryComplexityChecks,-} from "metabase/transforms/components/QueryComplexityWarning";import { Box, Button, Group, Modal, Stack } from "metabase/ui";import type {- QueryComplexity,SchemaName,Transform,TransformSource,@@ -158,17 +153,9 @@ function CreateTransformForm({showIncrementalSettings = true,}: CreateTransformFormFieldsProps) {const { values, setFieldValue } = useFormikContext<NewTransformValues>();- const { checkComplexity } = useQueryComplexityChecks();- const [complexity, setComplexity] = useState<QueryComplexity | undefined>();- const handleIncrementalChange = async (value: boolean) => {+ const handleIncrementalChange = (value: boolean) => {setFieldValue("incremental", value);- if (value) {- const complexity = await checkComplexity(source);- setComplexity(complexity);- } else {- setComplexity(undefined);- }};return (@@ -201,17 +188,12 @@ function CreateTransformForm({onIncrementalChange={handleIncrementalChange}/>)}- {complexity && <QueryComplexityWarning variant="standout" />}<Group><Box flex={1}><FormErrorMessage /></Box><Button onClick={onClose}>{t`Back`}</Button>- <FormSubmitButton- label={complexity ? t`Save anyway` : t`Save`}- variant="filled"- color={complexity ? "saturated-red" : undefined}- />+ <FormSubmitButton label={t`Save`} variant="filled" /></Group></Stack></Form>diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.tsindex 1bbde70ef0b..8bb4fa16d58 100644--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts@@ -32,7 +32,10 @@ export const useCreateTransform = (databaseId,);const transform = await createTransformMutation(request).unwrap();- trackTransformCreated({ transformId: transform.id });+ trackTransformCreated({+ transformId: transform.id,+ isIncremental: transform.target.type === "table-incremental",+ });return transform;};diff --git a/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx b/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsxindex 47c09a2e3f9..7d060d032c2 100644--- a/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx+++ b/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx@@ -1,11 +1,13 @@-import type { ReactNode } from "react";+import { type ReactNode, useId } from "react";import { t } from "ttag";+import { skipToken, useGetFieldQuery } from "metabase/api";import { DateTime } from "metabase/common/components/DateTime";import CS from "metabase/css/core/index.css";import { Box, Card, Stack } from "metabase/ui";import type { TransformRun } from "metabase-types/api";+import { CheckpointValue } from "../../../../components/CheckpointValue";import {formatRunMethod,formatStatus,@@ -19,6 +21,12 @@ type InfoSectionProps = {};export function InfoSection({ run }: InfoSectionProps) {+ const { data: checkpointField } = useGetFieldQuery(+ run.checkpoint_filter_field_id+ ? { id: run.checkpoint_filter_field_id }+ : skipToken,+ );+return (<Card p={0} shadow="none" withBorder role="region" aria-label={t`Info`}><InfoSectionItem label={t`Started at`}>@@ -37,6 +45,27 @@ export function InfoSection({ run }: InfoSectionProps) {<InfoSectionItem label={t`Trigger`}>{formatRunMethod(run.run_method)}</InfoSectionItem>+ {checkpointField != null && (+ <InfoSectionItem label={t`Checkpoint field`}>+ {checkpointField.display_name}+ </InfoSectionItem>+ )}+ {run.checkpoint_lo_value != null && (+ <InfoSectionItem label={t`Checkpoint from`}>+ <CheckpointValue+ value={run.checkpoint_lo_value}+ checkpointField={checkpointField}+ />+ </InfoSectionItem>+ )}+ {run.checkpoint_hi_value != null && (+ <InfoSectionItem label={t`Checkpoint to`}>+ <CheckpointValue+ value={run.checkpoint_hi_value}+ checkpointField={checkpointField}+ />+ </InfoSectionItem>+ )}</Card>);}@@ -47,9 +76,23 @@ type InfoSectionItemProps = {};function InfoSectionItem({ label, children }: InfoSectionItemProps) {+ const labelId = useId();+return (- <Stack className={S.section} p="md" gap="xs">- <Box className={CS.textWrap} c="text-secondary" fz="sm" lh="h5">+ <Stack+ className={S.section}+ p="md"+ gap="xs"+ role="group"+ aria-labelledby={labelId}+ >+ <Box+ id={labelId}+ className={CS.textWrap}+ c="text-secondary"+ fz="sm"+ lh="h5"+ >{label}</Box><Box lh="h4">{children}</Box>diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx b/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsxindex bec3eba95da..87c92543a59 100644--- a/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx@@ -1,6 +1,5 @@import type { Row } from "@tanstack/react-table";import {- type ComponentProps,type PropsWithChildren,useCallback,useEffect,@@ -31,6 +30,7 @@ import { useSelector } from "metabase/lib/redux";import * as Urls from "metabase/lib/urls";import { type NamedUser, getUserName } from "metabase/lib/user";import { PLUGIN_REMOTE_SYNC, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";+import { getMetadata } from "metabase/selectors/metadata";import { CreateTransformMenu } from "metabase/transforms/components/CreateTransformMenu";import { ListEmptyState } from "metabase/transforms/components/ListEmptyState";import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";@@ -52,7 +52,11 @@ import {import { CollectionRowMenu } from "./CollectionRowMenu";import S from "./TransformListPage.module.css";import { type TreeNode, getCollectionNodeId, isCollectionNode } from "./types";-import { buildTreeData, getDefaultExpandedIds } from "./utils";+import {+ buildTreeData,+ getDefaultExpandedIds,+ getIncrementalWarning,+} from "./utils";const getNodeId = (node: TreeNode) => node.id;const getSubRows = (node: TreeNode) => node.children;@@ -137,10 +141,22 @@ export const TransformListPage = ({const isLoading =isLoadingCollections || isLoadingTransforms || isLoadingDatabases;const error = collectionsError ?? transformsError;+ const metadata = useSelector(getMetadata);const shouldShowPythonTransformsUpsell = useSelector(getShouldShowPythonTransformsUpsell,);+ const warningsByTransformId = useMemo(() => {+ const warnings = new Map<number, string>();+ for (const transform of transforms ?? []) {+ const warning = getIncrementalWarning(transform, metadata);+ if (warning) {+ warnings.set(transform.id, warning);+ }+ }+ return warnings;+ }, [transforms, metadata]);+const treeData = useMemo(() => {const data = buildTreeData(collections, transforms);// Only show Python library item if there's at least one item in the table@@ -176,16 +192,6 @@ export const TransformListPage = ({);const columnDefs = useMemo<TreeTableColumnDef<TreeNode>[]>(() => {- type EllipsifiedProps = ComponentProps<- typeof EntityNameCell- >["ellipsifiedProps"];- const unreadableTransformEllipsifiedProps: EllipsifiedProps = {- alwaysShowTooltip: true,- tooltipProps: {- openDelay: 300,- label: t`Sorry, you don’t have permission to see that.`,- },- };return [{id: "name",@@ -194,26 +200,12 @@ export const TransformListPage = ({minWidth: 280,maxAutoWidth: 800,enableSorting: true,- cell: ({ row }) => {- const isLibraryWithoutFeature =- row.original.nodeType === "library" && !hasPythonTransformsFeature;- return (- <Group gap="sm" wrap="nowrap" miw={0}>- <EntityNameCell- data-testid="tree-node-name"- icon={row.original.icon}- iconColor={getNodeIconColor(row.original)}- name={row.original.name}- ellipsifiedProps={- isRowDisabled(row)- ? unreadableTransformEllipsifiedProps- : undefined- }- />- {isLibraryWithoutFeature && <UpsellGem.New size={14} />}- </Group>- );- },+ cell: ({ row }) =>+ getNameCell({+ row,+ hasPythonTransformsFeature,+ warningsByTransformId,+ }),},{id: "owner",@@ -287,7 +279,7 @@ export const TransformListPage = ({) : null,},];- }, [hasPythonTransformsFeature]);+ }, [hasPythonTransformsFeature, warningsByTransformId]);const getRowHref = useCallback((row: Row<TreeNode>) => {if (isRowDisabled(row)) {@@ -388,3 +380,56 @@ export const TransformListPage = ({</PageContainer>);};++function getNameCell({+ row,+ hasPythonTransformsFeature,+ warningsByTransformId,+}: {+ row: Row<TreeNode>;+ hasPythonTransformsFeature: boolean;+ warningsByTransformId: Map<number, string>;+}) {+ const getTooltipProps = (message: string | undefined) => {+ if (!message) {+ return undefined;+ }++ return {+ alwaysShowTooltip: true,+ tooltipProps: {+ openDelay: 300,+ label: message,+ },+ };+ };++ const getWarningMessage = () => {+ if (isRowDisabled(row)) {+ return t`Sorry, you don’t have permission to see that.`;+ }++ if (row.original.transformId) {+ return warningsByTransformId.get(row.original.transformId);+ }+ return undefined;+ };++ const isLibraryWithoutFeature =+ row.original.nodeType === "library" && !hasPythonTransformsFeature;++ const hasWarning = !!getWarningMessage();++ return (+ <Group gap="sm" wrap="nowrap" miw={0}>+ <EntityNameCell+ data-testid="tree-node-name"+ icon={hasWarning ? "warning" : row.original.icon}+ iconColor={hasWarning ? "warning" : getNodeIconColor(row.original)}+ name={row.original.name}+ ellipsifiedProps={{ ...getTooltipProps(getWarningMessage()) }}+ />+ {isLibraryWithoutFeature && <UpsellGem.New size={14} />}+ </Group>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/types.ts b/frontend/src/metabase/transforms/pages/TransformListPage/types.tsindex 1c9851f4496..8748cfe2b08 100644--- a/frontend/src/metabase/transforms/pages/TransformListPage/types.ts+++ b/frontend/src/metabase/transforms/pages/TransformListPage/types.ts@@ -17,6 +17,7 @@ export type TreeNode = {collectionId?: number;url?: string;source_readable?: boolean;+ warningTooltip?: string;};export function isCollectionNode(diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformListPage/utils.tsindex 69cd7a40776..92499ff1308 100644--- a/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts+++ b/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts@@ -1,4 +1,9 @@+import { t } from "ttag";+import { getCollectionIcon } from "metabase/entities/collections/utils";+import { getLibQuery } from "metabase/transforms/utils";+import * as Lib from "metabase-lib";+import type Metadata from "metabase-lib/v1/metadata/Metadata";import type { Collection, CollectionId, Transform } from "metabase-types/api";import {@@ -7,6 +12,40 @@ import {getTransformNodeId,} from "./types";+export function getIncrementalWarning(+ transform: Transform,+ metadata: Metadata,+): string | undefined {+ const isIncremental = transform.target.type === "table-incremental";+ if (!isIncremental) {+ return undefined;+ }++ const isNative = transform.source_type === "native";+ if (isNative) {+ const libQuery = getLibQuery(transform.source, metadata);+ const hasTableTag = libQuery+ ? Object.values(Lib.templateTags(libQuery)).some(+ (tag) => tag.type === "table" && tag["table-id"] != null,+ )+ : false;++ if (!hasTableTag) {+ return t`Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field.`;+ }+ }++ const checkpointFieldId =+ transform.source["source-incremental-strategy"]?.[+ "checkpoint-filter-field-id"+ ];+ if (!checkpointFieldId) {+ return t`Incremental transform is enabled but no checkpoint field is selected. Please select a checkpoint field in the transform settings.`;+ }++ return undefined;+}+export function buildTreeData(collections: Collection[] | undefined,transforms: Transform[] | undefined,diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.tsindex ebbbe241f4d..6773ebd78b3 100644--- a/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts+++ b/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts@@ -1,10 +1,32 @@-import type { TransformOwner } from "metabase-types/api";+import { createMockMetadata } from "__support__/metadata";+import { getLibQuery } from "metabase/transforms/utils";+import * as Lib from "metabase-lib";+import { DEFAULT_TEST_QUERY, SAMPLE_PROVIDER } from "metabase-lib/test-helpers";import {+ createMockStructuredDatasetQuery,+ createMockTemplateTag,createMockTransform,createMockTransformOwner,+ createMockTransformTarget,} from "metabase-types/api/mocks";-import { buildTreeData } from "./utils";+import { buildTreeData, getIncrementalWarning } from "./utils";++jest.mock("metabase/transforms/utils", () => ({+ getLibQuery: jest.fn(),+}));++jest.mock("metabase-lib", () => ({+ ...jest.requireActual<typeof Lib>("metabase-lib"),+ templateTags: jest.fn(),+}));++const query = Lib.createTestQuery(SAMPLE_PROVIDER, DEFAULT_TEST_QUERY);++const mockGetLibQuery = getLibQuery as jest.MockedFunction<typeof getLibQuery>;+const mockTemplateTags = Lib.templateTags as jest.MockedFunction<+ typeof Lib.templateTags+>;describe("buildTreeData", () => {it("should return empty array when no collections or transforms", () => {@@ -38,13 +60,13 @@ describe("buildTreeData", () => {id: 1,name: "Test Transform",owner_email: "[email protected]",- owner: { email: "[email protected]" } as TransformOwner,+ owner: createMockTransformOwner({ email: "[email protected]" }),});const result = buildTreeData([], [transform]);expect(result).toHaveLength(1);- expect(result[0].owner).toEqual({ email: "[email protected]" });+ expect(result[0].owner).toMatchObject({ email: "[email protected]" });expect(result[0].owner_email).toBe("[email protected]");});@@ -82,7 +104,7 @@ describe("buildTreeData", () => {id: 2,name: "Transform with email owner",owner_email: "[email protected]",- owner: { email: "[email protected]" } as TransformOwner,+ owner: createMockTransformOwner({ email: "[email protected]" }),}),createMockTransform({id: 3,@@ -95,8 +117,129 @@ describe("buildTreeData", () => {expect(result).toHaveLength(3);expect(result[0].owner).toEqual(userOwner);- expect(result[1].owner).toEqual({ email: "[email protected]" });+ expect(result[1].owner).toMatchObject({ email: "[email protected]" });expect(result[1].owner_email).toBe("[email protected]");expect(result[2].owner).toBeUndefined();});});++describe("getIncrementalWarning", () => {+ const metadata = createMockMetadata({});++ it("should return undefined for non-incremental transform", () => {+ const transform = createMockTransform({+ target: createMockTransformTarget({ type: "table" }),+ });++ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();+ });++ it("should warn when native query has no table variables", () => {+ mockGetLibQuery.mockReturnValue(query);+ mockTemplateTags.mockReturnValue({+ snippet1: createMockTemplateTag({+ id: "1",+ name: "snippet1",+ type: "snippet",+ }),+ });++ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ const result = getIncrementalWarning(transform, metadata);+ expect(result).toMatch(/table variable/);+ });++ it("should warn when native query has no lib query (getLibQuery returns null)", () => {+ mockGetLibQuery.mockReturnValue(null);++ const transform = createMockTransform({+ source_type: "native",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ const result = getIncrementalWarning(transform, metadata);+ expect(result).toMatch(/table variable/);+ });++ it("should warn when incremental transform has no checkpoint field", () => {+ mockGetLibQuery.mockReturnValue(query);+ mockTemplateTags.mockReturnValue({+ my_table: createMockTemplateTag({+ id: "1",+ name: "my_table",+ type: "table",+ "table-id": 42,+ }),+ });++ const transform = createMockTransform({+ source_type: "native",+ source: {+ type: "query",+ query: createMockStructuredDatasetQuery(),+ },+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ const result = getIncrementalWarning(transform, metadata);+ expect(result).toMatch(/checkpoint field/);+ });++ it("should return undefined for properly configured native incremental transform", () => {+ mockGetLibQuery.mockReturnValue(query);+ mockTemplateTags.mockReturnValue({+ my_table: createMockTemplateTag({+ id: "1",+ name: "my_table",+ type: "table",+ "table-id": 42,+ }),+ });++ const transform = createMockTransform({+ source_type: "native",+ source: {+ type: "query",+ query: createMockStructuredDatasetQuery(),+ "source-incremental-strategy": {+ type: "checkpoint",+ "checkpoint-filter-field-id": 99,+ },+ },+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();+ });++ it("should warn when mbql incremental transform has no checkpoint field", () => {+ const transform = createMockTransform({+ source_type: "mbql",+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ const result = getIncrementalWarning(transform, metadata);+ expect(result).toMatch(/checkpoint field/);+ });++ it("should return undefined for properly configured mbql incremental transform", () => {+ const transform = createMockTransform({+ source_type: "mbql",+ source: {+ type: "query",+ query: createMockStructuredDatasetQuery(),+ "source-incremental-strategy": {+ type: "checkpoint",+ "checkpoint-filter-field-id": 99,+ },+ },+ target: createMockTransformTarget({ type: "table-incremental" }),+ });++ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();+ });+});diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsxindex eac53b84235..ae8d3b8a4c8 100644--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx@@ -32,7 +32,6 @@ import type {Transform,} from "metabase-types/api";-import { useQueryComplexityChecks } from "../../components/QueryComplexityWarning";import {TransformEditor,type TransformEditorProps,@@ -131,8 +130,6 @@ function TransformQueryPageBody({useRegisterMetabotTransformContext(transform, source, lastRunError);- const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();-const {checkData,isCheckingDependencies,@@ -186,13 +183,6 @@ function TransformQueryPageBody({if (!isCompleteSource(source)) {return;}- if ("source-incremental-strategy" in source) {- const confirmed = await confirmIfQueryIsComplex(source);- if (!confirmed) {- return;- }- }-await handleInitialSave({ id: transform.id, source });};@@ -289,7 +279,6 @@ function TransformQueryPageBody({isEnabled={isDirty && !isSaving && !isCheckingDependencies}onConfirm={rejectProposed}/>- {modal}</>);}diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsxindex 240466e8722..30bae62d951 100644--- a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx@@ -101,7 +101,7 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {<Stack gap={0}><RunStatusrun={last_run ?? null}- neverRunMessage={t`This transform hasn’t been run before.`}+ neverRunMessage={t`This transform hasn't been run before.`}runInfo={<Anchorkey="link"diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsxnew file mode 100644index 00000000000..4453a532a05--- /dev/null+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsx@@ -0,0 +1,78 @@+import { useDisclosure } from "@mantine/hooks";+import { useId } from "react";+import { t } from "ttag";++import {+ skipToken,+ useGetFieldQuery,+ useResetCheckpointMutation,+} from "metabase/api";+import { ConfirmModal } from "metabase/common/components/ConfirmModal";+import { useMetadataToasts } from "metabase/metadata/hooks";+import { CheckpointValue } from "metabase/transforms/components/CheckpointValue";+import { isTransformRunning } from "metabase/transforms/utils";+import { Box, Button, Group, Icon, Text } from "metabase/ui";+import type { Transform } from "metabase-types/api";++export function ResetCheckpointSection({+ transform,+}: {+ transform: Transform;+}) {+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();+ const [resetCheckpoint, { isLoading }] = useResetCheckpointMutation();++ const checkpointFieldId =+ transform.source?.["source-incremental-strategy"]?.[+ "checkpoint-filter-field-id"+ ];+ const { data: checkpointField } = useGetFieldQuery(+ checkpointFieldId ? { id: checkpointFieldId } : skipToken,+ );++ const labelId = useId();++ const handleConfirm = async () => {+ const { error } = await resetCheckpoint(transform.id);+ closeModal();+ if (error) {+ sendErrorToast(t`Failed to reset checkpoint`);+ } else {+ sendSuccessToast(t`Checkpoint has been reset`);+ }+ };++ if (transform.last_checkpoint_value == null) {+ return null;+ }++ return (+ <Group gap="md" align="center">+ <Box c="text-secondary" role="group" aria-labelledby={labelId}>+ <span id={labelId}>{t`Current checkpoint`}: </span>+ <Text component="span" fw="bold" c="text-primary">+ <CheckpointValue+ value={transform.last_checkpoint_value}+ checkpointField={checkpointField}+ />+ </Text>+ </Box>+ <Button+ leftSection={<Icon name="revert" aria-hidden />}+ disabled={isTransformRunning(transform) || isLoading}+ onClick={openModal}+ >+ {t`Reset checkpoint`}+ </Button>+ <ConfirmModal+ title={t`Reset checkpoint?`}+ message={t`This will cause the next run to reprocess all data from scratch instead of only new rows.`}+ opened={isModalOpen}+ onClose={closeModal}+ onConfirm={handleConfirm}+ confirmButtonText={t`Reset`}+ />+ </Group>+ );+}diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsxindex 66250768345..d3b729d48ed 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx@@ -28,6 +28,17 @@ import { createMockState } from "metabase-types/store/mocks";import { TransformSettingsSection } from "./TransformSettingsSection";+jest.mock(+ "metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions",+ () => ({+ useHasCheckpointOptions: jest.fn().mockReturnValue({+ hasCheckpointOptions: true,+ hasNativeCheckpointOptions: true,+ transformType: "mbql",+ }),+ }),+);+type SetupOpts = {remoteSyncType?: EnterpriseSettings["remote-sync-type"];transform?: Transform;diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsxindex c80ae6db9f7..0ce20d285e5 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx@@ -32,6 +32,17 @@ type SetupOpts = {transform?: Transform;};+jest.mock(+ "metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions",+ () => ({+ useHasCheckpointOptions: jest.fn().mockReturnValue({+ hasCheckpointOptions: true,+ hasNativeCheckpointOptions: true,+ transformType: "mbql",+ }),+ }),+);+function setup({transform = createMockTransform(),remoteSyncType,diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsxindex f74c4c28d17..e8c1a02bc5d 100644--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx@@ -1,7 +1,10 @@+import { useDisclosure } from "@mantine/hooks";import { useFormikContext } from "formik";+import { useCallback, useRef } from "react";import { t } from "ttag";import _ from "underscore";+import { ConfirmModal } from "metabase/common/components/ConfirmModal";import { Form, FormInlineUpdater, FormProvider } from "metabase/forms";import { useMetadataToasts } from "metabase/metadata/hooks";import type { IncrementalSettingsFormValues } from "metabase/transforms/components/IncrementalTransform";@@ -9,9 +12,10 @@ import {IncrementalTransformSettings,useUpdateIncrementalSettings,} from "metabase/transforms/components/IncrementalTransform";-import { useQueryComplexityChecks } from "metabase/transforms/components/QueryComplexityWarning";import type { Transform } from "metabase-types/api";+import { ResetCheckpointSection } from "./ResetCheckpointSection";+type UpdateIncrementalSettingsProps = {transform: Transform;readOnly?: boolean;@@ -23,44 +27,72 @@ const IncrementalTransformSettingsWrapper = ({}: UpdateIncrementalSettingsProps) => {const { values, setFieldValue } =useFormikContext<IncrementalSettingsFormValues>();- const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();- const handleIncrementalChange = async (value: boolean) => {- if (value) {- const confirmed = await confirmIfQueryIsComplex(transform.source);- if (!confirmed) {- return;- }- }+ const handleIncrementalChange = (value: boolean) => {setFieldValue("incremental", value);++ // If incremental is turned off, reset the checkpoint filter field+ if (!value) {+ setFieldValue("checkpointFilterFieldId", null);+ }};return (- <>- <IncrementalTransformSettings- source={transform.source}- incremental={values.incremental}- onIncrementalChange={handleIncrementalChange}- variant="standalone"- readOnly={readOnly}- />- {modal}- </>+ <IncrementalTransformSettings+ source={transform.source}+ incremental={values.incremental}+ onIncrementalChange={handleIncrementalChange}+ variant="standalone"+ readOnly={readOnly}+ extraActions={+ !readOnly && transform.last_checkpoint_value != null ? (+ <ResetCheckpointSection transform={transform} />+ ) : undefined+ }+ />);};-export const UpdateIncrementalSettings = ({+function UpdateIncrementalSettingsContent({transform,readOnly,-}: UpdateIncrementalSettingsProps) => {+ updateIncrementalSettings,+}: UpdateIncrementalSettingsProps & {+ updateIncrementalSettings: (+ values: IncrementalSettingsFormValues,+ ) => Promise<unknown>;+}) {const { sendSuccessToast, sendErrorToast } = useMetadataToasts();+ const { onBeforeUpdate, modalProps } =+ useCheckpointChangeInterceptor(transform);- const showSuccessToast = () =>- sendSuccessToast(t`Incremental transformation settings updated`);-- const showErrorToast = () =>- sendErrorToast(t`Failed to update incremental transformation settings`);+ return (+ <>+ <FormInlineUpdater+ update={updateIncrementalSettings}+ onBeforeUpdate={onBeforeUpdate}+ onSuccess={() =>+ sendSuccessToast(t`Incremental transformation settings updated`)+ }+ onError={() =>+ sendErrorToast(+ t`Failed to update incremental transformation settings`,+ )+ }+ />+ <ConfirmModal {...modalProps} />+ <IncrementalTransformSettingsWrapper+ transform={transform}+ readOnly={readOnly}+ />+ </>+ );+}+export function UpdateIncrementalSettings({+ transform,+ readOnly,+}: UpdateIncrementalSettingsProps) {const { initialValues, validationSchema, updateIncrementalSettings } =useUpdateIncrementalSettings(transform);@@ -72,16 +104,82 @@ export const UpdateIncrementalSettings = ({enableReinitialize><Form>- <FormInlineUpdater- update={updateIncrementalSettings}- onSuccess={showSuccessToast}- onError={showErrorToast}- />- <IncrementalTransformSettingsWrapper+ <UpdateIncrementalSettingsContenttransform={transform}readOnly={readOnly}+ updateIncrementalSettings={updateIncrementalSettings}/></Form></FormProvider>);-};+}++/**+ * Hook that intercepts checkpoint field changes to show a confirmation modal+ * when a checkpoint value already exists. Must be used inside FormProvider.+ */+function useCheckpointChangeInterceptor(transform: Transform) {+ const { resetForm } = useFormikContext<IncrementalSettingsFormValues>();+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();+ const pendingResolve = useRef<((shouldProceed: boolean) => void) | null>(+ null,+ );++ const currentFieldId =+ transform.source?.["source-incremental-strategy"]?.[+ "checkpoint-filter-field-id"+ ];++ const onBeforeUpdate = useCallback(+ async (values: IncrementalSettingsFormValues): Promise<boolean> => {+ const waitingForCheckpointSelection =+ values.incremental &&+ values.sourceStrategy === "checkpoint" &&+ values.checkpointFilterFieldId == null;++ if (waitingForCheckpointSelection) {+ return false;+ }++ const fieldChanged =+ currentFieldId != null &&+ values.checkpointFilterFieldId != null &&+ String(currentFieldId) !== values.checkpointFilterFieldId;++ if (fieldChanged && transform.last_checkpoint_value != null) {+ return new Promise<boolean>((resolve) => {+ pendingResolve.current = resolve;+ openModal();+ });+ }++ return true;+ },+ [transform, currentFieldId, openModal],+ );++ const handleConfirm = useCallback(() => {+ pendingResolve.current?.(true);+ pendingResolve.current = null;+ closeModal();+ }, [closeModal]);++ const handleCancel = useCallback(() => {+ resetForm();+ pendingResolve.current?.(false);+ pendingResolve.current = null;+ closeModal();+ }, [resetForm, closeModal]);++ return {+ onBeforeUpdate,+ modalProps: {+ title: t`Change checkpoint field?`,+ message: t`Changing the checkpoint field will reset the stored checkpoint value. The next run will reprocess all data from scratch.`,+ confirmButtonText: t`Change checkpoint field`,+ opened: isModalOpen,+ onClose: handleCancel,+ onConfirm: handleConfirm,+ },+ };+}diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.tsindex 05a2e5592cf..7ac1df46c8b 100644--- a/frontend/src/metabase/transforms/utils.ts+++ b/frontend/src/metabase/transforms/utils.ts@@ -21,8 +21,6 @@ import type {TransformSource,} from "metabase-types/api";-import { CHECKPOINT_TEMPLATE_TAG } from "./constants";-export function parseTimestampWithTimezone(timestamp: string,systemTimezone: string | undefined,@@ -258,11 +256,7 @@ export function getValidationResult(query: Lib.Query): ValidationResult {return { isValid: Lib.canSave(query, "question") };}-const ALLOWED_TEMPLATE_TYPES = new Set([- "card",- "snippet",- CHECKPOINT_TEMPLATE_TAG,-]);+const ALLOWED_TEMPLATE_TYPES = new Set(["card", "snippet", "table"]);function validateTemplateTag(tag: TemplateTag): ValidationResult {// Allow snippets, cards, and the special transform variables ({checkpoint})diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 26976ff61a2..526cb863c9f 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -3384,6 +3384,98 @@ databaseChangeLog:);rollback: # nothing to undo - these rows should not exist in v59+ - changeSet:+ id: v59.2026-03-12T12:00:00+ author: wotbrew+ comment: Add last_checkpoint_value to transform table+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform+ columnName: last_checkpoint_value+ changes:+ - addColumn:+ tableName: transform+ columns:+ - column:+ name: last_checkpoint_value+ type: ${text.type}+ remarks: Serialized checkpoint watermark value for incremental transforms, temporal as ISO8601+ constraints:+ nullable: true++ - changeSet:+ id: v59.2026-03-13T00:00:00+ author: bronsa+ comment: Remove legacy incremental transform strategies that use checkpoint-filter instead of checkpoint-filter-field-id+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.RemoveLegacyIncrementalStrategies"++ - changeSet:+ id: v59.2026-03-13T00:00:01+ author: bronsa+ comment: Add checkpoint_filter_field_id column to transform_run+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_run+ columnName: checkpoint_filter_field_id+ changes:+ - addColumn:+ tableName: transform_run+ columns:+ - column:+ name: checkpoint_filter_field_id+ type: int+ remarks: The field ID used for incremental checkpoint filtering+ constraints:+ nullable: true++ - changeSet:+ id: v59.2026-03-13T00:00:02+ author: bronsa+ comment: Add checkpoint_lo_value column to transform_run+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_run+ columnName: checkpoint_lo_value+ changes:+ - addColumn:+ tableName: transform_run+ columns:+ - column:+ name: checkpoint_lo_value+ type: ${text.type}+ remarks: The low checkpoint value for this run (exclusive lower bound)+ constraints:+ nullable: true++ - changeSet:+ id: v59.2026-03-13T00:00:03+ author: bronsa+ comment: Add checkpoint_hi_value column to transform_run+ preConditions:+ - onFail: MARK_RAN+ - not:+ - columnExists:+ tableName: transform_run+ columnName: checkpoint_hi_value+ changes:+ - addColumn:+ tableName: transform_run+ columns:+ - column:+ name: checkpoint_hi_value+ type: ${text.type}+ remarks: The high checkpoint value for this run (inclusive upper bound)+ constraints:+ nullable: true+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 5d6f1d70da5..7eedb67395e 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1993,3 +1993,61 @@:schema uploads_schema_name}:where [:= :id id]}))](run! retire-and-revive-upload-table! inactive-upload-tables))))))++(defn- legacy-checkpoint-column-name+ "Extract the column name from a legacy column-unique-key string.+ Uses the same format as [[metabase.lib.metadata.column/unpack-unique-key]]:+ `column-unique-key-v<version>$<column-key>`."+ [unique-key]+ (when-let [[_match _version column-key] (re-find #"^column-unique-key-v(\d+)\$(.+$)" unique-key)]+ column-key))++(defn- legacy-checkpoint-source-table-id+ "Extract the source table ID from a parsed transform source for checkpoint migration.+ Returns nil for native transforms (which can't be migrated)."+ [parsed]+ (case (keyword (:type parsed))+ :query (get-in parsed [:query :stages 0 :source-table])+ :python (let [source-tables (:source-tables parsed)]+ (cond+ ;; vec format: [{:alias "t" :table_id 42}]+ (sequential? source-tables)+ (when (= 1 (count source-tables))+ (:table_id (first source-tables)))+ ;; map format: {"t" 42} or {"t" {:table_id 42}}+ (map? source-tables)+ (when (= 1 (count source-tables))+ (let [v (first (vals source-tables))]+ (if (map? v) (:table_id v) v)))))+ nil))++(defn- resolve-checkpoint-field-id+ "Try to resolve a legacy checkpoint-filter-unique-key to a field ID."+ [parsed]+ (when-let [unique-key (get-in parsed [:source-incremental-strategy :checkpoint-filter-unique-key])]+ (when-let [column-name (legacy-checkpoint-column-name unique-key)]+ (when-let [table-id (legacy-checkpoint-source-table-id parsed)]+ (when (int? table-id)+ (t2/select-one-pk :model/Field+ :table_id table-id+ :name column-name+ :active true))))))++(define-migration RemoveLegacyIncrementalStrategies+ (doseq [{:keys [id source]} (t2/select [:transform :id :source])]+ (let [parsed (json-out source true)]+ (when (:source-incremental-strategy parsed)+ (if-let [field-id (resolve-checkpoint-field-id parsed)]+ ;; Migrate: replace legacy keys with checkpoint-filter-field-id+ (let [migrated (update parsed :source-incremental-strategy+ (fn [s]+ (-> s+ (dissoc :checkpoint-filter-unique-key :checkpoint-filter)+ (assoc :checkpoint-filter-field-id field-id))))]+ (t2/query {:update :transform+ :set {:source (json-in migrated)}+ :where [:= :id id]}))+ ;; Can't migrate: strip incremental strategy+ (t2/query {:update :transform+ :set {:source (json-in (dissoc parsed :source-incremental-strategy))}+ :where [:= :id id]}))))))diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex 93bdc5cf98b..06c50b9a61b 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -123,14 +123,21 @@(when-let [new-collection (:collection_id (t2/changes transform))](collection/check-collection-namespace :model/Transform new-collection)(collection/check-allowed-content :model/Transform new-collection))- (cond-> transform- source- (assoc :source_type (transforms-base.u/transform-source-type source)- :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))-- (or (:source (t2/changes transform)) (:target (t2/changes transform)))- ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.- (assoc :target_db_id (transforms-base.i/target-db-id transform))))+ (let [changes (t2/changes transform)]+ (cond-> transform+ source+ (assoc :source_type (transforms-base.u/transform-source-type source)+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))++ (or (:source changes) (:target changes))+ ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.+ (assoc :target_db_id (transforms-base.i/target-db-id transform))++ ;; Reset checkpoint when the incremental filter field changes+ (let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])+ new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]+ (and old-field-id (not= old-field-id new-field-id)))+ (assoc :last_checkpoint_value nil))))(t2/define-after-select :model/Transform[{:keys [source] :as transform}]@@ -323,7 +330,7 @@(defmethod serdes/make-spec "Transform"[_model-name opts]{:copy [:name :description :entity_id :owner_email]- :skip [:dependency_analysis_version :source_type :target_db_id]+ :skip [:dependency_analysis_version :source_type :target_db_id :last_checkpoint_value]:transform {:created_at (serdes/date):creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User)diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 88b43649a68..801ad9b2c78 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -34,8 +34,6 @@source-tables-vec->map-for-fecheck-database-featurecheck-feature-enabled!- extract-all-columns-from-query- extract-incremental-filter-columns-from-queryvalidate-incremental-column-type!validate-transform-query!get-transformsdiff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex 38d21a3e6b5..e612fb2d9a3 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -5,24 +5,17 @@(:require[metabase.api.common :as api][metabase.database-routing.core :as database-routing]- [metabase.driver :as driver]- [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.util :as driver.u][metabase.events.core :as events]- [metabase.lib.core :as lib][metabase.models.interface :as mi][metabase.models.transforms.transform :as transform.model]- [metabase.query-processor.compile :as qp.compile][metabase.transforms-base.interface :as transforms-base.i][metabase.transforms-base.ordering :as transforms-base.ordering][metabase.transforms-base.util :as transforms-base.u][metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.i18n :refer [deferred-tru]]- [metabase.util.log :as log]- [toucan2.core :as t2])- (:import- (java.sql PreparedStatement)))+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -65,82 +58,21 @@(assoc error:status-code 400)))))-(defn extract-all-columns-from-query- "Extracts column metadata (name and type) from a query.-- Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (try- (let [{:keys [query]} (qp.compile/compile query)]- (sql-jdbc.execute/do-with-connection-with-options- driver- database-id- {}- (fn [conn]- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]- (when-let [rsmeta (.getMetaData stmt)]- (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))- (catch Exception e- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))- nil)))--(defn extract-incremental-filter-columns-from-query- "Extracts column names suitable for incremental transform checkpoint filtering.-- This function is specifically for incremental transform checkpoint column selection.- It only returns columns with types supported for checkpoint filtering:- - Temporal types (timestamp, timestamp with timezone)- - Numeric types (integer, float, decimal)-- Text, boolean, and other types are filtered out as they are not supported for- incremental checkpointing.-- Returns a vector of column names (as strings), or nil if extraction fails.-- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()- to inspect the query structure. This works for most modern JDBC drivers but may not- be supported by all drivers or for all query types."- [driver database-id query]- (some->> (extract-all-columns-from-query driver database-id query)- (filter (comp transforms-base.u/supported-incremental-filter-type? :base_type))- (mapv :name)))-(defn validate-incremental-column-type!"Validates that the checkpoint column for an incremental transform has a supported type.- For MBQL/Python transforms, resolves the column from the query using the unique key.- For native queries, extracts columns from the query and checks the checkpoint-filter column.-- Throws a 400 error if the column type is not supported or cannot be resolved."+ Resolves the field by ID and checks that its base-type is numeric or temporal.+ Throws a 400 error if the field cannot be found or its type is not supported."[{:keys [source]}]- (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}+ (when-let [{:keys [checkpoint-filter-field-id] strategy-type :type}(:source-incremental-strategy source)]- (when (and (= :query (:type source)) (= "checkpoint" strategy-type))- (let [{:keys [query]} source- database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))- driver-name (driver/the-initialized-driver (:engine database))]- (cond- ;; For MBQL, resolve column from query metadata- checkpoint-filter-unique-key- (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]- (api/check-400 column (deferred-tru "Checkpoint column not found in query."))- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base-type column))- (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."- (pr-str (:base-type column)))))-- ;; For native query with checkpoint-filter, validate type if we can extract the column metadata- checkpoint-filter- (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]- (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type column))- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."- checkpoint-filter- (pr-str (:base_type column)))))))))))+ (when (= "checkpoint" strategy-type)+ (let [field (t2/select-one :model/Field checkpoint-filter-field-id)]+ (api/check-400 field (deferred-tru "Checkpoint field not found."))+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."+ (:name field)+ (pr-str (:base_type field))))))))(defn get-transforms"Get a list of transforms."diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex de3ed6bb1aa..bce1e9f32be 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -35,25 +35,28 @@:db/id (:id db):db/engine (name driver)}(let [conn-spec (driver/connection-spec driver db)- transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}]- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]- (transforms.u/run-cancelable-transform!- run-id driver transform-details- (fn [cancel-chan]- (let [result (transforms-base.i/execute-base!- transform- {:cancelled? #(boolean (a/poll! cancel-chan))- :run-id run-id- :with-stage-timing-fn (fn [rid stage thunk]- (transforms.instrumentation/with-stage-timing [rid stage]- (thunk)))})]- ;; Bridge result-map to exception-based flow for run-cancelable-transform!- (when-not (= :succeeded (:status result))- (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))- result))))+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}+ exec-result+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]+ (transforms.u/run-cancelable-transform!+ run-id transform driver transform-details+ (fn [cancel-chan source-range-params]+ (let [result (transforms-base.i/execute-base!+ transform+ {:cancelled? #(boolean (a/poll! cancel-chan))+ :run-id run-id+ :source-range-params source-range-params+ :with-stage-timing-fn (fn [rid stage thunk]+ (transforms.instrumentation/with-stage-timing [rid stage]+ (thunk)))})]+ ;; Bridge result-map to exception-based flow for run-cancelable-transform!+ (when-not (= :succeeded (:status result))+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))+ result))))];; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)(transforms-base.u/complete-execution! transform{:run-id run-id+ :source-range-params (:source-range-params exec-result):with-stage-timing-fn (fn [rid stage thunk](transforms.instrumentation/with-stage-timing [rid stage](thunk)))})))))diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.cljindex a235243e4b1..46679487d71 100644--- a/src/metabase/transforms/schema.clj+++ b/src/metabase/transforms/schema.clj@@ -1,7 +1,7 @@(ns metabase.transforms.schema(:require- [metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.id :as lib.schema.id][metabase.queries.schema :as queries.schema][metabase.transforms-base.util :as transforms-base.u][metabase.util.malli.registry :as mr]@@ -10,11 +10,7 @@(mr/def ::checkpoint-strategy[:map[:type [:= "checkpoint"]]- ;; for native- [:checkpoint-filter {:optional true} :string]- ;; for mbql and python- [:checkpoint-filter-unique-key {:optional true}- ::lib.schema/column-unique-key]])+ [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]])(mr/def ::source-incremental-strategy[:multi {:dispatch :type}diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex 7692aa73a69..e302b750786 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -106,20 +106,25 @@(defn run-cancelable-transform!"Execute a transform with cancellation support and proper error handling.+ Computes `source-range-params` once upfront, saves them to the run record, and passes them+ to `run-transform!` which receives `cancel-chan` and `source-range-params` as arguments.+Options:- `:ex-message-fn` change how caught exceptions are presented to the user in run logs, by default the same as clojure.core/ex-message"- [run-id driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}]+ [run-id transform driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}];; local run is responsible for status, using canceling lifecycle(try(when-not (driver/schema-exists? driver db-id output-schema)(driver/create-schema-if-needed! driver conn-spec output-schema))- (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))- (let [cancel-chan (a/promise-chan)- ret (binding [qp.pipeline/*canceled-chan* cancel-chan]- (canceling/chan-start-run! run-id cancel-chan)- (run-transform! cancel-chan))]- (transform-run/succeed-started-run! run-id)- ret)+ (let [source-range-params (transforms-base.u/get-source-range-params transform)]+ (transforms-base.u/save-run-checkpoint-range! run-id source-range-params)+ (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))+ (let [cancel-chan (a/promise-chan)+ ret (binding [qp.pipeline/*canceled-chan* cancel-chan]+ (canceling/chan-start-run! run-id cancel-chan)+ (run-transform! cancel-chan source-range-params))]+ (transform-run/succeed-started-run! run-id)+ ret))(catch Throwable t(if (:timeout (ex-data t))(transform-run/timeout-run! run-id {:message (ex-message-fn t)})diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.cljindex df3f8e654d7..84926b47f17 100644--- a/src/metabase/transforms_base/core.clj+++ b/src/metabase/transforms_base/core.clj@@ -42,5 +42,6 @@([transform opts](let [result (transforms-base.i/execute-base! transform opts)](when (= :succeeded (:status result))- (transforms-base.u/complete-execution! transform opts))+ (transforms-base.u/complete-execution! transform+ (assoc opts :source-range-params (:source-range-params result))))result)))diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.cljindex cfb9347745f..aa8ab55e614 100644--- a/src/metabase/transforms_base/query.clj+++ b/src/metabase/transforms_base/query.clj@@ -79,7 +79,7 @@:result <driver result>:error <exception if failed>}"[{:keys [id source target] :as transform} :- ::transforms-base.schema/transform- {:keys [cancelled?] :as _opts} :- [:maybe ::transforms-base.schema/execute-base-options]]+ {:keys [cancelled? source-range-params] :as _opts} :- [:maybe ::transforms-base.schema/execute-base-options]](try;; Check cancellation before starting(when (and cancelled? (cancelled?))@@ -88,12 +88,18 @@(let [db (get-in source [:query :database]){driver :engine :as database} (t2/select-one :model/Database db)_ (transforms-base.u/throw-if-db-routing-enabled! transform database)+ ;; First incremental run (no checkpoint) should behave like non-incremental+ ;; to drop and recreate the table rather than appending to existing data.+ effective-transform-type (if (and (= :table-incremental (keyword (:type target)))+ (nil? (:last_checkpoint_value transform)))+ :table+ (keyword (:type target)))transform-details {:db-id db:database database:transform-id id- :transform-type (keyword (:type target))+ :transform-type effective-transform-type:conn-spec (driver/connection-spec driver database)- :query (transforms-base.u/compile-source transform)+ :query (transforms-base.u/compile-source transform source-range-params):output-schema (:schema target):output-table (transforms-base.u/qualified-table-name driver target)}opts (transform-opts transform-details)@@ -121,7 +127,8 @@(throw (ex-info "Transform cancelled after query execution" {:status :cancelled}))){:status :succeeded- :result result}))+ :result result+ :source-range-params source-range-params}))(catch Exception e(let [data (ex-data e)]diff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.cljindex 0604a3137ce..cb0a0aed8cb 100644--- a/src/metabase/transforms_base/schema.clj+++ b/src/metabase/transforms_base/schema.clj@@ -5,6 +5,8 @@The full transforms module (metabase.transforms.schema) extends these withadditional fields like :id (required for scheduled execution)."(:require+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms])(:import@@ -39,6 +41,22 @@[:name {:optional true} :string][:description {:optional true} [:maybe :string]]])+;;; ----------------------------------------- Source Range Params -----------------------------------------------++(mr/def ::checkpoint-bound+ "A bound (lo or hi) for incremental checkpoint filtering."+ [:map+ [:value :any]])++(mr/def ::source-range-params+ "Parameters for incremental range filtering on a source query.+ Returned by get-source-range-params."+ [:map+ [:column ::lib.schema.metadata/column]+ [:checkpoint-filter-field-id ::lib.schema.id/field]+ [:lo {:optional true} [:maybe ::checkpoint-bound]]+ [:hi {:optional true} [:maybe ::checkpoint-bound]]])+;;; ------------------------------------------------- Options -------------------------------------------------(mr/def ::execute-base-options@@ -49,7 +67,8 @@[:with-stage-timing-fn {:optional true} ifn?][:publish-events? {:optional true} :boolean][:message-log {:optional true} [:maybe ::atom]]- [:cancel-chan {:optional true} [:maybe ::chan]]])+ [:cancel-chan {:optional true} [:maybe ::chan]]+ [:source-range-params {:optional true} [:maybe ::source-range-params]]]);;; ------------------------------------------------- Result -------------------------------------------------@@ -59,4 +78,5 @@[:status [:enum :succeeded :failed :cancelled :timeout]][:result {:optional true} :any][:error {:optional true} [:maybe (ms/InstanceOfClass Throwable)]]- [:logs {:optional true} [:maybe :string]]])+ [:logs {:optional true} [:maybe :string]]+ [:source-range-params {:optional true} [:maybe ::source-range-params]]])diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 94ecc69dafb..a62fa0cff3c 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -4,18 +4,15 @@This namespace contains functions that are used by both query and python transformsand do NOT require transform_run database access."(:require- [buddy.core.codecs :as codecs]- [buddy.core.hash :as buddy-hash][clojure.string :as str][java-time.api :as t][metabase.driver :as driver]- [metabase.driver.sql.query-processor :as sql.qp]- [metabase.driver.util :as driver.u][metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.util :as lib.util][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.add-remaps :as remap]@@ -24,9 +21,9 @@[metabase.query-processor.preprocess :as qp.preprocess][metabase.sync.core :as sync][metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.schema :as transforms-base.schema][metabase.util :as u][metabase.util.date-2 :as u.date]- [metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :as i18n][metabase.util.log :as log][metabase.util.malli :as mu]@@ -145,11 +142,6 @@;;; ------------------------------------------------- Incremental/Checkpoint Helpers --------------------------------------------------(defn- checkpoint-incremental?- "Returns true if `source` uses checkpoint-based incremental strategy."- [source]- (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))-(defn supported-incremental-filter-type?"Returns true if the given base-type is supported for incremental filtering.@@ -158,118 +150,144 @@(or (isa? base-type :type/Temporal)(isa? base-type :type/Number)))-(defn- source->checkpoint-filter-unique-key- "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."- [query source-incremental-strategy]- (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))--(defn- source->checkpoint-filter-column- "Resolve the checkpoint filter column for an incremental transform.-- Tries to resolve the column using the unique key first.- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.-- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).- Throws an exception if the column type is not supported."- [query source-incremental-strategy table metadata-provider]- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]- (when-some [{column-name :name- :keys [base-type]- :as column}- (cond- checkpoint-filter-unique-key- (source->checkpoint-filter-unique-key query source-incremental-strategy)- checkpoint-filter- (when-some [field-id (t2/select-one-pk :model/Field- :table_id (:id table)- :name checkpoint-filter)]- (lib.metadata/field metadata-provider field-id)))]- (when-not (supported-incremental-filter-type? base-type)- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "- "Only numeric and temporal columns are supported for incremental filtering.")- {:column-name column-name- :base-type base-type})))- column)))--(declare target-table)--(defn next-checkpoint- "Build a query to compute the MAX of the checkpoint column from the target table.-- Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),- or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."- [{:keys [source target] :as transform}]- (let [db-id (transforms-base.i/target-db-id transform)]- (when (checkpoint-incremental? source)- (when-let [table (target-table db-id target)]- (let [metadata-provider (lib-be/application-database-metadata-provider db-id)- table-metadata (lib.metadata/table metadata-provider (:id table))- query (lib/query metadata-provider table-metadata)]- (when-let [filter-column (source->checkpoint-filter-column query- (:source-incremental-strategy source)- table metadata-provider)]- {:query (-> query (lib/aggregate (lib/max filter-column)))- :filter-column filter-column}))))))--(defn- next-checkpoint-value- "Execute the checkpoint query and normalize the result for database insertion.- Returns `nil` if the target table is empty."- [{:keys [query filter-column]}]- (let [{:keys [base-type]} filter-column- v (some-> query qp/process-query :data :rows first first)]- ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right- ;; shape for reinsertion- (cond- (nil? v)- nil-- (isa? base-type :type/Integer)- (bigint v)-- ;; any other number that's not an integer, should be a decimal/float- (number? v)- (bigdec v)+(defn- encode-checkpoint-value [v]+ (if (number? v)+ (str v)+ (u.date/format v)))++(defn save-watermark!+ "Commits the incremental transforms :hi watermark value to the appdb."+ [transform-id source-range-params]+ (t2/update! :model/Transform+ transform-id+ {:last_checkpoint_value (some-> source-range-params :hi :value encode-checkpoint-value)}))++(defn save-run-checkpoint-range!+ "Persist the checkpoint range (lo/hi) on a transform run record.+ This is called early in execution so the range is recorded even if the run fails."+ [run-id source-range-params]+ (when (and run-id source-range-params)+ (let [{:keys [checkpoint-filter-field-id lo hi]} source-range-params]+ (t2/update! :model/TransformRun run-id+ (cond-> {:checkpoint_filter_field_id checkpoint-filter-field-id}+ lo (assoc :checkpoint_lo_value (encode-checkpoint-value (:value lo)))+ hi (assoc :checkpoint_hi_value (encode-checkpoint-value (:value hi))))))))++(defn- parse-checkpoint-value+ "Parse a serialized checkpoint value string according to its base-type keyword."+ [base-type s]+ (cond+ (not (string? s)) s+ (isa? base-type :type/Float) (bigdec s)+ (isa? base-type :type/Number) (biginteger s)+ (isa? base-type :type/Temporal) (u.date/parse s)+ :else (throw (ex-info (str "Unsupported checkpoint type: " (pr-str base-type))+ {:base-type base-type}))))++(defn- tag-checkpoint-value+ "Wrap a raw checkpoint value from the QP into a map `{:value parsed}`."+ [base-type raw-value]+ {:value (parse-checkpoint-value base-type raw-value)})++(defn- inject-filters-into-table-tag+ "Inject `:source-filters` into the table template tag matching the checkpoint field's table.++ Instead of manually expanding {{tag}} to a subquery SQL string, this adds filter metadata+ to the template tag so the QP's existing pipeline handles substitution. The filters specify+ field-id-based comparisons (e.g. :> lo, :<= hi) that get rendered as a filtered subquery+ in `->replacement-snippet-info`."+ [query source-range-params]+ (let [{:keys [checkpoint-filter-field-id lo hi column]} source-range-params+ table-id (:table-id column)+ tags (lib/template-tags query)+ tag-name (some (fn [[k v]]+ (when (and (#{:table "table"} (:type v))+ (= table-id (:table-id v)))+ k))+ tags)+ _ (when-not tag-name+ (throw (ex-info "No table variable found for checkpoint field's table"+ {:checkpoint-table-id table-id+ :template-tags tags})))+ filters (cond-> []+ lo (conj {:field-id checkpoint-filter-field-id :op :> :value (:value lo)})+ hi (conj {:field-id checkpoint-filter-field-id :op :<= :value (:value hi)}))]+ (lib.util/update-query-stage+ query 0+ #(assoc-in % [:template-tags tag-name :source-filters] filters))))++(mu/defn get-source-range-params :- [:maybe ::transforms-base.schema/source-range-params]+ "Returns information on the incremental range filters that ought to be applied to a source query.- :else v)))+ Returns a map:+ :column (the lib column value of the incremental filter column)+ :checkpoint-filter-field-id (the field ID of the checkpoint column)+ Range predicate terms (maps :type, :value), can be nil (in which case the filter clause should be omitted):+ :lo values in the source table must be > this :value.+ :hi values in the source table must be <= this :value."+ [{:keys [source target] :as transform}]+ (let [{:keys [source-incremental-strategy]} source+ {:keys [checkpoint-filter-field-id]} source-incremental-strategy]+ (when (and (= "table-incremental" (:type target))+ (native-query-transform? transform)+ (not (some (fn [[_k v]] (#{:table "table"} (:type v)))+ (lib/template-tags (:query source)))))+ (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "+ "Please add a table variable to the query and update the checkpoint field."))]+ (throw (ex-info msg {:transform-message msg}))))+ (when (and (= "table-incremental" (:type target))+ (not checkpoint-filter-field-id))+ (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "+ "Please select a checkpoint field in the transform settings."))]+ (throw (ex-info msg {:transform-message msg}))))+ (when checkpoint-filter-field-id+ (let [{:keys [last_checkpoint_value]} transform+ db-id (transforms-base.i/target-db-id transform)+ metadata-provider (lib-be/application-database-metadata-provider db-id)+ column (lib.metadata/field metadata-provider checkpoint-filter-field-id)+ _ (when (or (nil? column) (not (:active column)))+ (throw (ex-info "Checkpoint field does not exist or is not active"+ {:checkpoint-filter-field-id checkpoint-filter-field-id})))+ _ (when-not (supported-incremental-filter-type? (:base-type column))+ (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type " (pr-str (:base-type column)) ". "+ "Only numeric and temporal columns are supported for incremental filtering.")+ {:column column})))+ base-type (:base-type column)+ lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))++ max-value+ (let [table-id (:table-id column)+ table-metadata (lib.metadata/table metadata-provider table-id)+ base-query (lib/query metadata-provider table-metadata)+ filtered-query (if lo (lib/filter base-query (lib/> column lo)) base-query)+ query (lib/aggregate (lib/append-stage filtered-query) (lib/max column))+ query-result (qp/process-query query)]+ (ffirst (get-in query-result [:data :rows])))]+ {:column column+ :checkpoint-filter-field-id checkpoint-filter-field-id+ :lo (when lo {:value lo})+ :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)+ lo {:value lo})}))))(defn preprocess-incremental-query- "Add checkpoint checkpoint filtering to a query for incremental execution.-- For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.- For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.- Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."- [query source-incremental-strategy checkpoint]- (if-let [checkpoint-value (next-checkpoint-value checkpoint)]+ "Add checkpoint filtering to a query for incremental execution.++ For native queries, injects `:source-filters` into the table template tag matching the checkpoint+ field's table. The QP's substitution pipeline renders these as a filtered subquery.+ For MBQL queries, adds filter clauses `WHERE checkpoint_column > lo AND checkpoint_column <= hi`.+ Returns the query unchanged when source-range-params is nil."+ [query source-range-params]+ (if source-range-params(if (lib/native? query)- ;; native query with explicit checkpoint filter- (if (get-in query [:stages 0 :template-tags "checkpoint"])- (update query :parameters conj- {:type (if (number? checkpoint-value) :number :text)- :target [:variable [:template-tag "checkpoint"]]- :value checkpoint-value})- query)- ;; mbql query- (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))+ ;; Native: inject source-filters into the table template tag — QP handles the rest+ (inject-filters-into-table-tag query source-range-params)+ ;; MBQL: add filter clauses+ (cond-> query+ (:lo source-range-params) (lib/filter 0 (lib/> (:column source-range-params) (:value (:lo source-range-params))))+ (:hi source-range-params) (lib/filter 0 (lib/<= (:column source-range-params) (:value (:hi source-range-params))))))+ ;; No range params - return unchangedquery))-(defn- post-process-incremental-query- "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.-- Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "- [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]- (let [{:keys [checkpoint-filter]} source-incremental-strategy]- (if (and (lib/native? (:query source))- (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))- (next-checkpoint-value checkpoint))- (let [wrap-query (fn [query]- (let [honeysql-query {:select [:*]- :from [[[:raw (str "(" query ")")] :subquery]]- :where [:> (h2x/identifier :field checkpoint-filter)- [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]- (first (sql.qp/format-honeysql driver honeysql-query))))]- (update outer-query :query wrap-query))- outer-query)))-(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]"Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."[{:keys [source]}]@@ -283,21 +301,18 @@(defn compile-source"Compile the source query of a transform to SQL, applying incremental filtering if required."- [{:keys [source] :as transform}]- (let [{:keys [source-incremental-strategy] query-type :type} source]- (case (keyword query-type)- :query- (let [checkpoint (next-checkpoint transform)- query (:query source)- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]- (binding [driver/*compile-with-inline-parameters*- (or (= :clickhouse driver)- driver/*compile-with-inline-parameters*)]- (-> query- (preprocess-incremental-query source-incremental-strategy checkpoint)- massage-sql-query- qp.compile/compile- (post-process-incremental-query driver source checkpoint)))))))+ [{:keys [source]} source-range-params]+ (let [{query-type :type} source]+ (assert (= :query (keyword query-type)))+ (let [query (:query source)+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]+ (binding [driver/*compile-with-inline-parameters*+ (or (= :clickhouse driver)+ driver/*compile-with-inline-parameters*)]+ (-> query+ (preprocess-incremental-query source-range-params)+ massage-sql-query+ qp.compile/compile)))));;; ------------------------------------------------- Target Table Management -------------------------------------------------@@ -420,106 +435,13 @@(log/infof "Renaming tables: %s" (pr-str rename-map))(driver/rename-tables! driver database-id rename-map))-;;; ------------------------------------------------- Secondary Index Management ---------------------------------------------------(def ^:private metabase-index-prefix "mb_transform_idx_")--(defn- incremental-filter-index-name [schema table-name filter-column-name]- (let [prefix metabase-index-prefix- suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]- (str prefix suffix)))--(defn- should-index-incremental-filter-column? [driver database]- (and (driver.u/supports? driver :describe-indexes database)- (driver.u/supports? driver :transforms/index-ddl database)))--(defn- metabase-owned-index? [index]- (str/starts-with? (:index-name index) metabase-index-prefix))--(defn- metabase-incremental-filter-index [filter-column target]- (let [table-name (:name target)- schema (:schema target)- filter-name (:name filter-column)]- ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)- {:index-name (incremental-filter-index-name schema table-name filter-name)- :value filter-name}))--(defn- decide-secondary-index-ddl- "Decides which indexes should be dropped/created on the target table.-- e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.-- Indexes are represented as maps with at least the keys :index-name, :value.- This matches the form used by [[metabase.driver/describe-table-indexes]].-- Returns a map:- :drop - a vector of indexes that are redundant and should be dropped.- :create - a vector of desirable indexes that do not exist and that should be created.-- Notes:- - If user covering indexes exist they should be reused.- - Will never drop user indexes.- - Indexes we previously created and are no longer required are dropped."- [{:keys [filter-column indexes target database]}]- (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)- default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))- column-name (:name filter-column)- existing-user-index (first (filter #(= (:value %) column-name) user-indexes))- existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))- driver (:engine database)- intended-index (when (should-index-incremental-filter-column? driver database)- ;; Prefer reuse to not create redundant indexes- (or existing-user-index default-mb-index))- drop (if intended-index- (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)- mb-indexes)- create (when (and intended-index- (not= (:index-name intended-index) (:index-name existing-user-index))- (not= (:index-name intended-index) (:index-name existing-mb-index)))- [intended-index])]- {:drop (vec drop)- :create (vec create)}))--(defn execute-secondary-index-ddl-if-required!- "If target table index modifications are required, executes those CREATE/DROP commands.- See [[decide-secondary-index-ddl]] for details.-- `run-id` is optional - if nil, instrumentation/timing is skipped.- `with-stage-timing-fn` is optional - if provided, will be called as (with-stage-timing-fn run-id stage thunk)."- ([transform database target]- (execute-secondary-index-ddl-if-required! transform nil database target nil))- ([transform run-id database target with-stage-timing-fn]- (when (driver.u/supports? (:engine database) :describe-indexes database)- (let [driver (:engine database)- indexes (driver/describe-table-indexes driver database target)- checkpoint (next-checkpoint transform)- {:keys [drop create]}- (decide-secondary-index-ddl- {:filter-column (:filter-column checkpoint)- :database database- :target target- :indexes indexes})- maybe-timed (fn [stage thunk]- (if (and run-id with-stage-timing-fn)- (with-stage-timing-fn run-id stage thunk)- (thunk)))]- (doseq [{:keys [index-name value]} drop]- (maybe-timed [:import :drop-incremental-filter-index]- (fn []- (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))- (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name))))- (doseq [{:keys [index-name value]} create]- (maybe-timed [:import :create-incremental-filter-index]- (fn []- (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))- (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))))-;;; ------------------------------------------------- Post-Execution Completion -------------------------------------------------(defn complete-execution!"Post-processing steps after a transform has been executed successfully.Performs:+ - Save watermark (if source-range-params provided)- Sync target table to AppDB- Set `transform_id` on the target table- Publish Metabase events (unless `:publish-events?` is false)@@ -530,10 +452,13 @@to preserve the correct order of operations."[transform opts](let [{:keys [target]} transform- {:keys [run-id with-stage-timing-fn publish-events?]+ {:keys [publish-events? source-range-params]:or {publish-events? true}} optsdb-id (transforms-base.i/target-db-id transform)database (t2/select-one :model/Database db-id)]+ ;; Save watermark if source-range-params available+ (when source-range-params+ (save-watermark! (:id transform) source-range-params));; Sync target table(sync-target! target database);; Mark the table as owned by this transform@@ -549,10 +474,7 @@:transform-id (:id transform):transform-type (keyword (:type target)):output-schema (:schema target)- :output-table (qualified-table-name (:engine database) target)}}))- ;; Create secondary indexes if needed- (execute-secondary-index-ddl-if-required!- transform run-id database target with-stage-timing-fn)))+ :output-table (qualified-table-name (:engine database) target)}}))));;; ------------------------------------------------- Source Table Schemas -------------------------------------------------diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 4810afc61a6..159d7b636b4 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -4,7 +4,6 @@[metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers]- [metabase.driver :as driver][metabase.events.core :as events][metabase.query-processor :as qp][metabase.query-processor.middleware.constraints :as qp.constraints]@@ -12,7 +11,6 @@[metabase.query-processor.streaming :as qp.streaming][metabase.request.core :as request][metabase.server.core :as server]- [metabase.sql-tools.core :as sql-tools][metabase.transforms-base.util :as transforms-base.u][metabase.transforms-inspector.core :as inspector][metabase.transforms-inspector.schema :as inspector.schema]@@ -74,7 +72,10 @@[:message [:maybe :string]][:user_id [:maybe pos-int?]][:transform_name {:optional true} [:maybe :string]]- [:transform_entity_id {:optional true} [:maybe :string]]])+ [:transform_entity_id {:optional true} [:maybe :string]]+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]+ [:checkpoint_lo_value {:optional true} [:maybe :string]]+ [:checkpoint_hi_value {:optional true} [:maybe :string]]])(def ^:private TransformResponse[:map {:closed true}@@ -100,7 +101,8 @@[:table {:optional true} [:maybe :map]][:owner_user_id {:optional true} [:maybe pos-int?]][:owner_email {:optional true} [:maybe :string]]- [:owner {:optional true} [:maybe OwnerResponse]]])+ [:owner {:optional true} [:maybe OwnerResponse]]+ [:last_checkpoint_value {:optional true} [:maybe :string]]])(def ^:private TransformRunResponse[:map {:closed true}@@ -115,6 +117,9 @@[:user_id [:maybe pos-int?]][:transform_name {:optional true} [:maybe :string]][:transform_entity_id {:optional true} [:maybe :string]]+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]+ [:checkpoint_lo_value {:optional true} [:maybe :string]]+ [:checkpoint_hi_value {:optional true} [:maybe :string]];; Transform can have id/name when exists, or be nil when deleted[:transform {:optional true} [:maybe [:map {:closed true}[:id {:optional true} pos-int?]@@ -299,6 +304,13 @@(transforms.core/cancel-run! (:id run))))nil)+(api.macros/defendpoint :post "/:id/reset-checkpoint" :- :nil+ "Reset the stored checkpoint for an incremental transform."+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]]+ (api/write-check :model/Transform id)+ (t2/update! :model/Transform id {:last_checkpoint_value nil})+ nil)+(defn run-transform!"Run a transform. Returns a 202 response with run_id.The transform must already be fetched and validated."@@ -328,49 +340,6 @@[:id ms/PositiveInt]]](run-transform! (api/write-check :model/Transform id)))-(defn- simple-native-query?- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion.- Delegates to sql-tools which dispatches to the configured parser backend (macaw or sqlglot)."- [sql-string]- (sql-tools/simple-query? sql-string))--(api.macros/defendpoint :post "/is-simple-query" :- [:map- [:is_simple :boolean]- [:reason {:optional true} :string]]- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion"- [_route-params- _query-params- {:keys [query]} :- [:map [:query string?]]]- (api/check-superuser)- (simple-native-query? query))--(api.macros/defendpoint :post "/extract-columns"- :- [:map [:columns [:maybe [:sequential :string]]]]- "Extract column names suitable for incremental transform checkpoint filtering.-- This endpoint is specifically for populating the checkpoint column dropdown in- incremental transforms. It only returns columns with types supported for checkpoint- filtering: temporal (timestamp/tz) and numeric (int/float) types.-- Text, boolean, and other unsupported column types are filtered out.-- The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],- which handles parameterized queries with template tags. Then extracts column names- and types using PreparedStatement metadata.-- Returns a map with a :columns key containing a vector of column names (strings).- If extraction fails, returns nil for :columns."- [_route-params- _query-params- {:keys [query]} :- [:map- [:query ::qp.schema/any-query]]]- (api/check-superuser)- (let [database-id (:database query)- database (api/check-404 (t2/select-one :model/Database :id database-id))- driver-name (driver/the-initialized-driver (:engine database))- columns (transforms.core/extract-incremental-filter-columns-from-query driver-name database-id query)]- {:columns columns}))-;;; -------------------------------------------------- Inspector API --------------------------------------------------(api.macros/defendpoint :get "/:id/inspect"diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex fe841e17733..3240c6277c7 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2848,6 +2848,93 @@:active true:is_upload false} unrelated)))))))+(deftest remove-legacy-incremental-strategies-test+ (testing "v59.2026-03-13T00:00:00: migrate legacy checkpoint-filter to checkpoint-filter-field-id"+ (impl/test-migrations ["v59.2026-03-13T00:00:00"] [migrate!]+ (let [db-id (:id (new-instance-with-default :metabase_database))+ table-id (:id (new-instance-with-default :metabase_table {:db_id db-id}))+ field-id (t2/insert-returning-pk! :metabase_field+ {:name "updated_at"+ :table_id table-id+ :base_type "type/DateTimeWithLocalTZ"+ :database_type "timestamptz"+ :active true+ :created_at :%now+ :updated_at :%now})+ target (json/encode {:type "table" :schema "public" :name "out"})+ insert-transform!+ (fn [source]+ (t2/insert-returning-pk!+ :transform {:name (mt/random-name)+ :source source+ :target target+ :source_type "mbql"+ :source_database_id db-id+ :created_at :%now+ :updated_at :%now}))+ ;; Transform with resolvable legacy checkpoint strategy+ resolvable-source+ (json/encode {:type "query"+ :query {:stages [{:source-table table-id}]}+ :source-incremental-strategy+ {:type "checkpoint"+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"+ :checkpoint-filter {:some "filter"}}})+ resolvable-id (insert-transform! resolvable-source)+ ;; Transform with unresolvable legacy checkpoint strategy (no matching field)+ unresolvable-source+ (json/encode {:type "query"+ :query {:stages [{:source-table table-id}]}+ :source-incremental-strategy+ {:type "checkpoint"+ :checkpoint-filter-unique-key "column-unique-key-v1$nonexistent_column"+ :checkpoint-filter {:some "filter"}}})+ unresolvable-id (insert-transform! unresolvable-source)+ ;; Transform without any incremental strategy — should be untouched+ no-strategy-source+ (json/encode {:type "query"+ :query {:stages [{:source-table table-id}]}})+ no-strategy-id (insert-transform! no-strategy-source)+ ;; Python transform with resolvable checkpoint strategy+ python-resolvable-source+ (json/encode {:type "python"+ :body "x=1"+ :source-tables [{"alias" "t" "table_id" table-id}]+ :source-incremental-strategy+ {:type "checkpoint"+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"+ :checkpoint-filter {:some "filter"}}})+ python-resolvable-id (insert-transform! python-resolvable-source)+ ;; Native transform — can't resolve source table, strategy stripped+ native-source+ (json/encode {:type "native"+ :source-incremental-strategy+ {:type "checkpoint"+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"+ :checkpoint-filter {:some "filter"}}})+ native-id (insert-transform! native-source)+ get-source (fn [id]+ (json/decode+ (t2/select-one-fn :source :transform :id id)+ keyword))]+ (migrate!)+ (testing "Resolvable legacy strategy is migrated to checkpoint-filter-field-id"+ (let [strategy (:source-incremental-strategy (get-source resolvable-id))]+ (is (= field-id (:checkpoint-filter-field-id strategy)))+ (is (not (contains? strategy :checkpoint-filter-unique-key)))+ (is (not (contains? strategy :checkpoint-filter)))))+ (testing "Unresolvable legacy strategy is stripped entirely"+ (is (not (contains? (get-source unresolvable-id) :source-incremental-strategy))))+ (testing "Transform without strategy is untouched"+ (is (not (contains? (get-source no-strategy-id) :source-incremental-strategy))))+ (testing "Python transform with single source-table resolves correctly"+ (let [strategy (:source-incremental-strategy (get-source python-resolvable-id))]+ (is (= field-id (:checkpoint-filter-field-id strategy)))+ (is (not (contains? strategy :checkpoint-filter-unique-key)))+ (is (not (contains? strategy :checkpoint-filter)))))+ (testing "Native transform strategy is stripped (can't resolve source table)"+ (is (not (contains? (get-source native-id) :source-incremental-strategy))))))))+(deftest unify-source-tables-format-test(testing "v60.2026-03-03T12:00:00: convert source-tables from map to vec format"(impl/test-migrations ["v60.2026-03-03T12:00:00"] [migrate!]diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.cljindex e0bc81bc168..2312a1a352b 100644--- a/test/metabase/transforms/models_test.clj+++ b/test/metabase/transforms/models_test.clj@@ -325,3 +325,51 @@(let [run (transform-run/start-run! transform-id {:run_method "manual"})](is (= transform-name (:transform_name run)))(is (= entity-id (:transform_entity_id run))))))))++(deftest checkpoint-reset-on-filter-field-change-test+ (testing "Changing checkpoint-filter-field-id resets stored checkpoint"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-temp [:model/Database {db-id :id} {}+ :model/Transform {transform-id :id}+ {:source {:type "query"+ :query {:database db-id+ :type "native"+ :native {:query "SELECT 1"+ :template-tags {}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id 100}}+ :target {:type "table-incremental"+ :schema "public"+ :name "test_incr"+ :db_id db-id}+ :last_checkpoint_value "42"}]+ (testing "checkpoint is present before update"+ (let [t (t2/select-one :model/Transform transform-id)]+ (is (= "42" (:last_checkpoint_value t)))))++ (testing "changing checkpoint-filter-field-id resets checkpoint"+ (t2/update! :model/Transform transform-id+ {:source {:type "query"+ :query {:database db-id+ :type "native"+ :native {:query "SELECT 1"+ :template-tags {}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id 200}}})+ (let [t (t2/select-one :model/Transform transform-id)]+ (is (nil? (:last_checkpoint_value t)))))++ (testing "updating without changing checkpoint-filter-field-id preserves checkpoint"+ ;; Set checkpoint again+ (t2/update! :model/Transform transform-id+ {:last_checkpoint_value "99"})+ (t2/update! :model/Transform transform-id+ {:source {:type "query"+ :query {:database db-id+ :type "native"+ :native {:query "SELECT 2"+ :template-tags {}}}+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id 200}}})+ (let [t (t2/select-one :model/Transform transform-id)]+ (is (= "99" (:last_checkpoint_value t)))))))))diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex e89f0529809..45ae3057eb2 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -363,9 +363,8 @@;; so it sets transform_id on the target table.(with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})transforms-base.u/sync-target! (constantly nil)- transforms-base.u/execute-secondary-index-ddl-if-required! (constantly nil)- transforms.u/run-cancelable-transform! (fn [_run-id _driver _details run-fn & _opts]- (run-fn (a/promise-chan)))]+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]+ (run-fn (a/promise-chan) nil))](transforms.execute/execute! transform {:run-method :manual})(is (= transform-id(t2/select-one-fn :transform_id :model/Table :id table-id))))))))))@@ -399,7 +398,7 @@(mt/with-premium-features #{:transforms-basic}(let [transform {:source {:type "query":query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}- {:keys [query]} (transforms-base.u/compile-source transform)]+ {:keys [query]} (transforms-base.u/compile-source transform (transforms-base.u/get-source-range-params transform))](is (string? query))(is (not (re-find #"(?i)\bLIMIT\b" query))(str "Expected no LIMIT clause in compiled SQL, got: " query)))))))diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 0c1a1949f09..63c6f839090 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -3,7 +3,6 @@(:require[clojure.test :refer :all][medley.core :as m]- [metabase.driver :as driver][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.transforms.transform :as transform.model]@@ -14,7 +13,6 @@[metabase.test :as mt][metabase.transforms-base.util :as transforms-base.u][metabase.transforms-rest.api.transform]- [metabase.transforms.crud][metabase.transforms.query-test-util :as query-test-util][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :refer [get-test-schema@@ -1189,94 +1187,6 @@(mt/user-http-request :lucky :get 200 "transform")(mt/user-http-request :lucky :get 200 (str "transform/" (:id transform)))))))))-(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]- [_driver _feature _database]- true)--(doseq [driver [:clickhouse :redshift :bigquery-cloud-sdk :snowflake]]- (defmethod driver/database-supports? [driver ::extract-columns-from-query]- [_driver _feature _database]- false))--(deftest ^:parallel extract-columns-from-query-test- (mt/with-premium-features #{}- (testing "POST /api/transform/extract-columns"- (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table- ::extract-columns-from-query]})- (mt/dataset transforms-dataset/transforms-test- (letfn [(make-native-query [sql]- {:lib/type :mbql/query- :database (mt/id)- :stages [{:lib/type :mbql.stage/native- :native sql}]})]- (testing "Successfully extracts columns from a simple SELECT query"- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"- {:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]- (is (= ["id" "price"] (:columns response))- "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))- (testing "Returns nil for invalid SQL"- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"- {:query (make-native-query "SELECT * FORM invalid_table")})]- (is (nil? (:columns response)))))- (testing "Extracts columns from query with aliases"- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"- {:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]- (is (= ["product_id"] (:columns response))- "Should only return numeric column (id), filtering out text column (name)")))- (testing "Filters columns by type - only returns numeric and temporal columns"- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"- {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]- (is (= ["id" "price" "created_at"] (:columns response))- "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))- (testing "Requires superuser permissions"- (is (= "You don't have permissions to do that."- (mt/user-http-request :rasta :post 403 "transform/extract-columns"- {:query (make-native-query "SELECT * FROM transforms_products")}))))- (testing "Returns 404 for non-existent database"- (is (= "Not found."- (mt/user-http-request :crowberto :post 404 "transform/extract-columns"- {:query {:lib/type :mbql/query- :database 999999- :stages [{:lib/type :mbql.stage/native- :native "SELECT * FROM transforms_products"}]}}))))))))))--(deftest ^:parallel is-simple-query-test- (mt/with-premium-features #{}- (testing "POST /api/transform/is-simple-query"- (testing "Returns true for simple SELECT queries"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT id, name FROM products"})]- (is (true? (:is_simple response)))- (is (nil? (:reason response)))))- (testing "Returns true for simple SELECT with WHERE clause"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT id, name FROM products WHERE category = 'Electronics'"})]- (is (true? (:is_simple response)))))- (testing "Returns true for simple SELECT with JOIN"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT p.id, p.name, c.name FROM products p JOIN categories c ON p.category_id = c.id"})]- (is (true? (:is_simple response)))))- (testing "Returns false for query with LIMIT"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT id, name FROM products LIMIT 10"})]- (is (false? (:is_simple response)))- (is (= "Contains a LIMIT" (:reason response)))))- (testing "Returns false for query with OFFSET"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT id, name FROM products OFFSET 5"})]- (is (false? (:is_simple response)))- (is (= "Contains an OFFSET" (:reason response)))))- (testing "Returns false for query with LIMIT and OFFSET"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "SELECT id, name FROM products LIMIT 10 OFFSET 5"})]- (is (false? (:is_simple response)))- (is (= "Contains a LIMIT" (:reason response)))))- (testing "Returns false for query with CTE"- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"- {:query "WITH category_counts AS (SELECT category, COUNT(*) as cnt FROM products GROUP BY category) SELECT * FROM category_counts"})]- (is (false? (:is_simple response)))- (is (= "Contains a CTE" (:reason response))))))))-;;; ------------------------------------------------------------;;; User Attribution Tests;;; ------------------------------------------------------------@@ -1640,12 +1550,13 @@(is (= ["collection"] (:here child-coll)))(is (= ["transform"] (:below child-coll)))))))))-(deftest native-incremental-column-type-validated-on-create-test+(deftest incremental-column-type-validated-on-create-test(mt/with-premium-features #{}- (testing "POST /api/transform column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (testing "POST /api/transform column type validation with checkpoint-filter-field-id"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]+ (let [schema (get-test-schema)+ name-field-id (t2/select-one-pk :model/Field :name "name" :table_id (mt/id :transforms_products))](testing "Rejects unsupported checkpoint column type (text)"(let [response (mt/user-http-request :crowberto :post 400 "transform"{:name "Invalid Incremental Transform"@@ -1653,20 +1564,21 @@:query (lib/native-query (mt/metadata-provider)"SELECT id, name, category, price FROM transforms_products"):source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "name"}}+ :checkpoint-filter-field-id name-field-id}}:target {:type "table-incremental":schema schema:name "invalid_incremental":target-incremental-strategy {:type "append"}}})](is (string? response))- (is (re-find #"Only numeric and temporal" response))))))))))+ (is (re-find #"unsupported type" response))))))))))-(deftest native-incremental-column-type-validated-on-update-test+(deftest incremental-column-type-validated-on-update-test(mt/with-premium-features #{}- (testing "PUT /api/transform column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)+ (testing "PUT /api/transform column type validation with checkpoint-filter-field-id"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)(mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]+ (let [schema (get-test-schema)+ category-field-id (t2/select-one-pk :model/Field :name "category" :table_id (mt/id :transforms_products))](with-transform-cleanup! [table-name "update_incremental_test"](let [;; Create a non-incremental transform firstcreated (mt/user-http-request :crowberto :post 200 "transform"@@ -1684,60 +1596,13 @@:query (lib/native-query (mt/metadata-provider)"SELECT id, name, category FROM transforms_products"):source-incremental-strategy {:type "checkpoint"- :checkpoint-filter "category"}}+ :checkpoint-filter-field-id category-field-id}}:target {:type "table-incremental":schema schema:name table-name:target-incremental-strategy {:type "append"}}})](is (string? response))- (is (re-find #"Only numeric and temporal" response))))))))))))--(deftest mbql-incremental-column-type-validated-on-create-test- (mt/with-premium-features #{}- (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (testing "Rejects unsupported checkpoint column type (text)"- (let [query (mt/mbql-query transforms_products)- response (mt/user-http-request :crowberto :post 400 "transform"- {:name "Invalid MBQL Incremental"- :source {:type "query"- :query query- :source-incremental-strategy {:type "checkpoint"- :checkpoint-filter-unique-key "column-unique-key-v1$name"}}- :target {:type "table-incremental"- :schema schema- :name "invalid_mbql_incremental"- :target-incremental-strategy {:type "append"}}})]- (is (string? response))- (is (re-find #"not supported" response))))))))))--(deftest native-incremental-column-validation-when-not-extractable-test- (mt/with-premium-features #{}- (testing "Native query checkpoint column validation with text input fallback"- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)- (mt/dataset transforms-dataset/transforms-test- (let [schema (get-test-schema)]- (testing "Accepts any column if they were not extractable"- (with-transform-cleanup! [table-name "fallback_test_unextracted"]- (with-redefs [metabase.transforms.crud/extract-all-columns-from-query- ;; simulate lack of driver support for extraction- (fn [_driver _database-id _query] nil)]- (let [response (mt/user-http-request :crowberto :post 200 "transform"- {:name "Unextracted Column - Text Input Fallback"- :source {:type "query"- :query (lib/native-query (mt/metadata-provider)- "SELECT id, name, created_at FROM transforms_products")- :source-incremental-strategy {:type "checkpoint"- ;; created_at is in the query but not in our stubbed extraction- :checkpoint-filter "created_at"}}- :target {:type "table-incremental"- :schema schema- :name table-name- :target-incremental-strategy {:type "append"}}})]- (is (some? (:id response))- "Should accept column not in extracted metadata, allowing text input fallback")))))))))))+ (is (re-find #"unsupported type" response))))))))))))(deftest search-filters-transform-source-types-test(mt/with-premium-features #{}