Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

incremental transforms with source filter (#70573)

Nicola Mometto525b09b87b35
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 5970b7e728e..2495f0f114d 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1699,13 +1699,11 @@
metabase.transforms-rest.api.transform-job
metabase.transforms-rest.api.transform-tag}
:uses #{api
- driver
events
models
query-processor
request
server
- sql-tools
transforms
transforms-base
transforms-inspector
diff --git a/e2e/support/helpers/e2e-transform-helpers.ts b/e2e/support/helpers/e2e-transform-helpers.ts
index ec203205767..21a27362dae 100644
--- a/e2e/support/helpers/e2e-transform-helpers.ts
+++ b/e2e/support/helpers/e2e-transform-helpers.ts
@@ -4,6 +4,7 @@ import type {
CollectionId,
PythonTransformTableAliases,
SchemaName,
+ TemplateTags,
TransformId,
TransformRun,
TransformRunStatus,
@@ -130,6 +131,10 @@ export function createSqlTransform({
sourceCheckpointStrategy,
name = "SQL transform",
wrapId = true,
+ templateTags,
+ tableVariableTable,
+ tableVariableSchema,
+ databaseId = WRITABLE_DB_ID,
}: {
name?: string;
sourceQuery: string;
@@ -139,31 +144,67 @@ export function createSqlTransform({
visitTransform?: boolean;
sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;
wrapId?: boolean;
+ templateTags?: TemplateTags;
+ /** When set, adds a template variable of type "table" named "table" for the given table */
+ tableVariableTable?: string;
+ tableVariableSchema?: string;
+ databaseId?: number;
}) {
- return createTransform(
- {
- name,
- source: {
- type: "query",
- query: {
- database: WRITABLE_DB_ID,
- type: "native",
- native: {
- query: sourceQuery,
+ const create = (native: {
+ query: string;
+ "template-tags"?: TemplateTags;
+ }) => {
+ return createTransform(
+ {
+ name,
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "native",
+ native,
},
+ "source-incremental-strategy": sourceCheckpointStrategy,
},
- "source-incremental-strategy": sourceCheckpointStrategy,
- },
- target: {
- type: "table",
- database: WRITABLE_DB_ID,
- name: targetTable,
- schema: targetSchema,
+ target: {
+ type: "table",
+ database: WRITABLE_DB_ID,
+ name: targetTable,
+ schema: targetSchema,
+ },
+ tag_ids: tagIds,
},
- tag_ids: tagIds,
- },
- { wrapId, visitTransform },
- );
+ { wrapId, visitTransform },
+ );
+ };
+
+ if (tableVariableTable) {
+ return getTableId({
+ databaseId,
+ name: tableVariableTable,
+ schema: tableVariableSchema,
+ }).then((tableId) => {
+ const resolvedTemplateTags: TemplateTags = {
+ ...(templateTags ?? {}),
+ table: {
+ id: "table-tag-id",
+ name: "table",
+ "display-name": "Table",
+ type: "table",
+ "table-id": tableId,
+ },
+ };
+ return create({
+ query: sourceQuery,
+ "template-tags": resolvedTemplateTags,
+ });
+ });
+ }
+
+ return create({
+ query: sourceQuery,
+ ...(templateTags && { "template-tags": templateTags }),
+ });
}
export function createPythonTransform({
diff --git a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
index 0c6d92cc8ab..172a4ace9e2 100644
--- a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
@@ -95,25 +95,30 @@ describe("issue GDGT-1774", () => {
it("should display field options in the incremental update field picker (GDGT-1774)", () => {
H.getTableId({ name: SOURCE_TABLE })
.then((tableId) =>
- H.createTransform({
- name: "Incremental MBQL transform",
- source: {
- type: "query",
- query: {
- database: WRITABLE_DB_ID,
+ H.getFieldId({ tableId, name: "score" }).then((fieldId) =>
+ H.createTransform({
+ name: "Incremental MBQL transform",
+ source: {
type: "query",
- query: { "source-table": tableId },
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: { "source-table": tableId },
+ },
+ "source-incremental-strategy": {
+ type: "checkpoint",
+ "checkpoint-filter-field-id": fieldId,
+ },
},
- "source-incremental-strategy": { type: "checkpoint" },
- },
- target: {
- type: "table-incremental",
- database: WRITABLE_DB_ID,
- name: TARGET_TABLE,
- schema: TARGET_SCHEMA,
- "target-incremental-strategy": { type: "append" },
- },
- }),
+ target: {
+ type: "table-incremental",
+ database: WRITABLE_DB_ID,
+ name: TARGET_TABLE,
+ schema: TARGET_SCHEMA,
+ "target-incremental-strategy": { type: "append" },
+ },
+ }),
+ ),
)
.then((res) => H.DataStudio.Transforms.visitSettingsTab(res.body.id));
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.ts
new file mode 100644
index 00000000000..e66f9efd1f7
--- /dev/null
+++ b/e2e/test/scenarios/data-studio/transforms/transforms-incremental.cy.spec.ts
@@ -0,0 +1,416 @@
+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
+
+const { H } = cy;
+
+const DB_NAME = "Writable Postgres12";
+const SOURCE_TABLE = "Animals";
+const TARGET_TABLE = "transform_table";
+const TARGET_SCHEMA = "Schema A";
+
+describe("scenarios > admin > transforms incremental", () => {
+ beforeEach(() => {
+ H.restore("postgres-writable");
+ H.resetTestTable({ type: "postgres", table: "many_schemas" });
+ H.resetSnowplow();
+ cy.signInAsAdmin();
+ H.activateToken("bleeding-edge");
+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });
+
+ cy.intercept("PUT", "/api/field/*").as("updateField");
+ cy.intercept("POST", "/api/transform").as("createTransform");
+ cy.intercept("PUT", "/api/transform/*").as("updateTransform");
+ cy.intercept("DELETE", "/api/transform/*").as("deleteTransform");
+ cy.intercept("DELETE", "/api/transform/*/table").as("deleteTransformTable");
+ cy.intercept("POST", "/api/transform-tag").as("createTag");
+ cy.intercept("PUT", "/api/transform-tag/*").as("updateTag");
+ cy.intercept("DELETE", "/api/transform-tag/*").as("deleteTag");
+ cy.intercept("POST", "/api/ee/dependencies/check-transform").as(
+ "checkTransformDependencies",
+ );
+ cy.intercept("POST", "/api/transform/*/reset-checkpoint").as(
+ "resetCheckpoint",
+ );
+ });
+
+ afterEach(() => {
+ H.expectNoBadSnowplowEvents();
+ });
+
+ describe("creation", () => {
+ it("should be able to create and run an mbql incremental transform", () => {
+ cy.log("create a new transform");
+ visitTransformListPage();
+ cy.button("Create a transform").click();
+ H.popover().findByText("Query builder").click();
+
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_create",
+ event_detail: "query",
+ });
+
+ H.miniPicker().within(() => {
+ cy.findByText(DB_NAME).click();
+ cy.findByText(TARGET_SCHEMA).click();
+ cy.findByText(SOURCE_TABLE).click();
+ });
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("MBQL");
+
+ cy.log("should auto-populate table name based on transform name...");
+ cy.findByLabelText("Table name").should("have.value", "mbql");
+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+
+ cy.log("...unless user has manually modified the table name");
+ cy.findByLabelText("Name").type(" transform");
+ cy.findByLabelText("Table name").should("have.value", TARGET_TABLE);
+ cy.findByRole("switch", {
+ name: /Only process new and changed data/i,
+ }).click({ force: true });
+
+ cy.button("Save").click();
+ cy.wait("@createTransform").then(({ response }) => {
+ const transformId = response?.body?.id;
+ if (transformId != null) {
+ cy.wrap(transformId).as("transformId");
+ }
+ });
+ });
+
+ cy.log("run the transform and make sure its table can be queried");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_trigger_manual_run",
+ });
+
+ cy.findByRole("link", { name: "See all runs" }).click();
+
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findByRole("row", { name: /MBQL transform/i }).click();
+ });
+
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/30/).should("be.visible");
+ });
+ });
+
+ cy.log(
+ "add one element to the source table and run incremental transform again",
+ );
+ H.queryWritableDB(
+ 'INSERT INTO "Schema A"."Animals" (name, score) VALUES (\'NewRow\', 31)',
+ );
+
+ cy.go("back");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+
+ cy.log(
+ "verify the new element was picked up in the incremental transfer",
+ );
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /MBQL transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+
+ cy.log("go to Transform Settings and reset checkpoint");
+ cy.go("back");
+ H.DataStudio.Transforms.settingsTab().click();
+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {
+ cy.findByText(/31/).should("exist");
+ });
+ cy.button("Reset checkpoint").click();
+ H.modal().within(() => {
+ cy.button("Reset").click();
+ });
+ cy.wait("@resetCheckpoint");
+
+ cy.log(
+ "go to Runs tab, run transform again and check new run has checkpoint to 31",
+ );
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /MBQL transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+ });
+
+ it(
+ "should be able to create and run a Python incremental transform",
+ { tags: ["@python"] },
+ () => {
+ H.setPythonRunnerSettings();
+ cy.log("create a new transform");
+ visitTransformListPage();
+ cy.button("Create a transform").click();
+ H.popover().findByText("Python script").click();
+
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_create",
+ event_detail: "python",
+ });
+
+ cy.findByTestId("python-transform-top-bar")
+ .findByText("Writable Postgres12")
+ .click();
+ H.popover().findByText(DB_NAME).click();
+
+ getPythonDataPicker().findByText("Select a table…").click();
+ H.entityPickerModal().findByText(SOURCE_TABLE).click();
+
+ H.PythonEditor.clear().paste(
+ `import pandas as pd
+
+def transform(animals):
+ return pd.DataFrame([{"name": "test", "score": 0}])`,
+ );
+
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("Python transform");
+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+ cy.findByRole("switch", {
+ name: /Only process new and changed data/i,
+ }).click({ force: true });
+ cy.button("Save").click();
+ cy.wait("@createTransform").then(({ response }) => {
+ const transformId = response?.body?.id;
+ if (transformId != null) {
+ cy.wrap(transformId).as("transformId");
+ }
+ });
+ });
+
+ cy.log("run the transform and make sure its table can be queried");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_trigger_manual_run",
+ });
+
+ cy.findByRole("link", { name: "See all runs" }).click();
+
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findByRole("row", { name: /Python transform/i }).click();
+ });
+
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/30/).should("be.visible");
+ });
+ });
+
+ cy.log(
+ "add one element to the source table and run incremental transform again",
+ );
+ H.queryWritableDB(
+ 'INSERT INTO "Schema A"."Animals" (name, score) VALUES (\'NewRow\', 31)',
+ );
+
+ cy.go("back");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+
+ cy.log(
+ "verify the new element was picked up in the incremental transfer",
+ );
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /Python transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+
+ cy.log("go to Transform Settings and reset checkpoint");
+ cy.go("back");
+ H.DataStudio.Transforms.settingsTab().click();
+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {
+ cy.findByText(/31/).should("exist");
+ });
+ cy.button("Reset checkpoint").click();
+ H.modal().within(() => {
+ cy.button("Reset").click();
+ });
+ cy.wait("@resetCheckpoint");
+
+ cy.log(
+ "go to Runs tab, run transform again and check new run has checkpoint to 31",
+ );
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /Python transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+ },
+ );
+
+ it("should be able to create and run a native SQL incremental transform", () => {
+ const SCHEMA_B = "Schema B";
+ cy.log("create a new transform");
+ visitTransformListPage();
+ cy.button("Create a transform").click();
+ H.popover().findByText("SQL query").click();
+
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_create",
+ event_detail: "native",
+ });
+
+ H.popover().findByText(DB_NAME).click();
+ H.NativeEditor.type("SELECT * FROM {{t}}", {
+ allowFastSet: true,
+ }).blur();
+
+ cy.log("configure table variable for incremental support");
+ cy.findByTestId("native-query-top-bar")
+ .findByLabelText("Variables")
+ .click();
+ editorSidebar().findByLabelText("Variable type").click();
+ H.popover().findByText("Table").click();
+ H.popover().findByText(SCHEMA_B).click();
+ H.popover().findByText(SOURCE_TABLE).click();
+
+ getQueryEditor().button("Save").click();
+
+ H.modal().within(() => {
+ cy.findByLabelText("Name").clear().type("SQL transform");
+ cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
+ cy.findByRole("switch", {
+ name: /Only process new and changed data/i,
+ }).click({ force: true });
+ cy.button("Save").click();
+ cy.wait("@createTransform");
+ });
+
+ cy.log("run the transform and make sure its table can be queried");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ H.expectUnstructuredSnowplowEvent({
+ event: "transform_trigger_manual_run",
+ });
+
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findByRole("row", { name: /SQL transform/i }).click();
+ });
+
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/30/).should("be.visible");
+ });
+ });
+
+ cy.log(
+ "add one element to the source table and run incremental transform again",
+ );
+ H.queryWritableDB(
+ `INSERT INTO "${SCHEMA_B}"."Animals" (name, score) VALUES (\'NewRow\', 31)`,
+ );
+
+ cy.go("back");
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+
+ cy.log(
+ "verify the new element was picked up in the incremental transfer",
+ );
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /SQL transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+
+ cy.log("go to Transform Settings and reset checkpoint");
+ cy.go("back");
+ H.DataStudio.Transforms.settingsTab().click();
+ cy.findByRole("group", { name: /Current checkpoint/i }).within(() => {
+ cy.findByText(/31/).should("exist");
+ });
+ cy.button("Reset checkpoint").click();
+ H.modal().within(() => {
+ cy.button("Reset").click();
+ });
+ cy.wait("@resetCheckpoint");
+
+ cy.log(
+ "go to Runs tab, run transform again and check new run has checkpoint to 31",
+ );
+ H.DataStudio.Transforms.runTab().click();
+ runTransformAndWaitForSuccess();
+ cy.findByRole("link", { name: "See all runs" }).click();
+ cy.findByRole("treegrid", { name: "Transform runs" }).within(() => {
+ cy.findAllByRole("row", { name: /SQL transform/i })
+ .first()
+ .click();
+ });
+ cy.findByRole("region", { name: "Info" }).within(() => {
+ cy.findByRole("group", { name: "Checkpoint to" }).within(() => {
+ cy.findByText(/31/).should("be.visible");
+ });
+ });
+ });
+ });
+});
+
+function visitTransformListPage() {
+ return cy.visit("/data-studio/transforms");
+}
+
+function runTransformAndWaitForSuccess() {
+ getRunButton().click();
+ getRunButton().should("have.text", "Ran successfully");
+}
+
+function getRunButton(options: { timeout?: number } = {}) {
+ return cy.findAllByTestId("run-button").eq(0, options);
+}
+
+function getQueryEditor() {
+ return cy.findByTestId("transform-query-editor");
+}
+
+function editorSidebar() {
+ return cy.findByTestId("editor-sidebar");
+}
+
+function getPythonDataPicker() {
+ return cy.findByTestId("python-data-picker");
+}
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index aa0d8edb714..453c2cd0b0e 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -925,7 +925,17 @@ LIMIT
});
it("should handle sequential changes correctly when first update is in progress", () => {
- createMbqlTransform({ visitTransform: true });
+ // Use composite_pk_table which has at least two numeric fields (id1, score)
+ // so we can reliably select a second checkpoint field
+ H.resetTestTable({ type: "postgres", table: "composite_pk_table" });
+ H.resyncDatabase({
+ dbId: WRITABLE_DB_ID,
+ tableName: "composite_pk_table",
+ });
+ createMbqlTransform({
+ sourceTable: "composite_pk_table",
+ visitTransform: true,
+ });
H.DataStudio.Transforms.settingsTab().click();
cy.log("Verify initial state");
@@ -958,12 +968,11 @@ LIMIT
cy.wait(400);
// Make a second change while first is still in progress
- // Select any available checkpoint field
+ // Select the second checkpoint field (composite_pk_table has id1 and score)
getFieldPicker().scrollIntoView().should("be.visible");
getFieldPicker().click();
- // Click the first available option in the popover
- H.popover().findAllByRole("option").first().click();
+ H.popover().findAllByRole("option").eq(1).click();
cy.log("Wait for both requests to complete");
cy.wait("@updateTransformDelayed");
@@ -983,7 +992,15 @@ LIMIT
});
it("should update source strategy and checkpoint field inline", () => {
- createMbqlTransform({ visitTransform: true });
+ H.resetTestTable({ type: "postgres", table: "composite_pk_table" });
+ H.resyncDatabase({
+ dbId: WRITABLE_DB_ID,
+ tableName: "composite_pk_table",
+ });
+ createMbqlTransform({
+ sourceTable: "composite_pk_table",
+ visitTransform: true,
+ });
H.DataStudio.Transforms.settingsTab().click();
cy.log("Enable incremental transformation");
@@ -999,7 +1016,7 @@ LIMIT
cy.log("Select a checkpoint field");
getFieldPicker().click();
// Click the first available option in the popover
- H.popover().findAllByRole("option").first().click();
+ H.popover().findAllByRole("option").eq(1).click();
cy.wait("@updateTransform");
H.undoToast().should(
"contain.text",
@@ -1658,98 +1675,6 @@ LIMIT
H.DataStudio.Transforms.pythonResults().should("not.exist");
},
);
-
- describe("query complexity warning", () => {
- it("should show complexity warning modal when saving a complex SQL query", () => {
- cy.log("create a simple SQL transform");
- createSqlTransform({
- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
- visitTransform: true,
- sourceCheckpointStrategy: { type: "checkpoint" },
- });
-
- cy.log("visit edit mode and change to a complex query with LIMIT");
- H.DataStudio.Transforms.clickEditDefinition();
- cy.url().should("include", "/edit");
-
- H.NativeEditor.type(" LIMIT 10");
- getQueryEditor().button("Save").click();
-
- handleQueryComplexityWarningModal("cancel");
- cy.log("verify modal is closed and still in edit mode");
- H.modal().should("not.exist");
- cy.url().should("include", "/edit");
- cy.get("@updateTransform.all").should("have.length", 0);
-
- cy.log("Save anyway");
- getQueryEditor().button("Save").click();
- handleQueryComplexityWarningModal("save");
-
- cy.wait("@updateTransform");
- cy.url().should("not.include", "/edit");
- });
-
- it("should confirm incremental settings change if query is complex", () => {
- createSqlTransform({
- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,
- sourceCheckpointStrategy: { type: "checkpoint" },
- visitTransform: true,
- });
- H.DataStudio.Transforms.settingsTab().click();
-
- cy.log("Toggle incremental on");
- isIncrementalSwitchDisabled();
- getIncrementalSwitch().click();
-
- handleQueryComplexityWarningModal("cancel");
-
- cy.log("Verify that the switch is still off");
- isIncrementalSwitchDisabled();
-
- cy.log("Toggle incremental on");
- getIncrementalSwitch().click();
- handleQueryComplexityWarningModal("save");
-
- cy.wait("@updateTransform");
- isIncrementalSwitchEnabled();
- H.undoToast().should(
- "contain.text",
- "Incremental transformation settings updated",
- );
- });
-
- it("should show complexity warning with danger button in create transform modal when enabling incremental with complex query", () => {
- cy.log("create a new SQL transform with a complex query");
- visitTransformListPage();
- cy.button("Create a transform").click();
- H.popover().findByText("SQL query").click();
- H.popover().findByText(DB_NAME).click();
-
- H.NativeEditor.type(
- `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}" LIMIT 10`,
- );
-
- getQueryEditor().button("Save").click();
-
- H.modal().within(() => {
- cy.findByLabelText("Name").clear().type("Complex SQL transform");
- cy.findByLabelText("Table name").clear().type(TARGET_TABLE);
-
- cy.log("Enable incremental transformation");
- getIncrementalSwitch().click();
-
- cy.log("Verify complexity warning appears inline");
- cy.findByTestId("query-complexity-warning")
- .scrollIntoView()
- .should("be.visible");
-
- cy.log("Verify the submit button is styled as danger (red)");
- cy.findByRole("button", { name: "Save anyway" })
- .scrollIntoView()
- .should("have.css", "background-color", "rgb(209, 44, 41)");
- });
- });
- });
});
describe("runs", () => {
@@ -2686,7 +2611,9 @@ LIMIT
beforeEach(() => {
cy.log("create a transform");
createSqlTransform({
- sourceQuery: `SELECT * FROM "${TARGET_SCHEMA}"."${SOURCE_TABLE}"`,
+ sourceQuery: "SELECT * FROM {{ table }}",
+ tableVariableTable: SOURCE_TABLE,
+ tableVariableSchema: TARGET_SCHEMA,
});
cy.log("set up remote sync");
H.setupGitSync();
@@ -3788,18 +3715,6 @@ function isIncrementalSwitchDisabled() {
return getIncrementalSwitch().findByRole("switch").should("not.be.checked");
}
-function handleQueryComplexityWarningModal(action: "cancel" | "save") {
- cy.log(`Verify complexity warning modal appears and ${action} it`);
- return H.modal().within(() => {
- cy.findByTestId("query-complexity-warning").should("be.visible");
- if (action === "save") {
- cy.button("Save anyway").click();
- } else {
- cy.button("Cancel").click();
- }
- });
-}
-
function getJobTransformTable() {
return cy.findByLabelText("Job transforms");
}
@@ -3891,6 +3806,8 @@ function createSqlTransform(opts: {
tagIds?: TransformTagId[];
visitTransform?: boolean;
sourceCheckpointStrategy?: TransformSourceCheckpointStrategy;
+ tableVariableTable?: string;
+ tableVariableSchema?: string;
}) {
return H.createSqlTransform({
targetTable: TARGET_TABLE,
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index 9ec54e49cd0..1f501795297 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -165,18 +165,23 @@
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
(defmethod transfer-file-to-db :table-incremental
- [driver {db-id :id}
+ [driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
- (let [table-name (transforms-base.u/qualified-table-name driver target)
- table-exists? (transforms-base.u/target-table-exists? transform)
- data-source {:type :jsonl-file
- :file temp-file}]
- (if (not table-exists?)
- (do
- (log/info "New table")
- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
- (insert-data! driver db-id (table-schema table-name metadata) data-source))))
+ ;; First incremental run: no checkpoint exists yet, behave like non-incremental
+ ;; to drop and recreate the table rather than appending to existing data.
+ (if (nil? (:last_checkpoint_value transform))
+ ((get-method transfer-file-to-db :table) driver db transform metadata temp-file)
+ ;; Normal incremental: append if table exists, create if it doesn't
+ (let [table-name (transforms-base.u/qualified-table-name driver target)
+ table-exists? (transforms-base.u/target-table-exists? transform)
+ data-source {:type :jsonl-file
+ :file temp-file}]
+ (if (not table-exists?)
+ (do
+ (log/info "New table")
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+ (insert-data! driver db-id (table-schema table-name metadata) data-source)))))
(defmethod transfer-file-to-db :table
[driver {db-id :id :as db}
@@ -216,18 +221,19 @@
Options:
- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"
- [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn]}]
+ [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn source-range-params]}]
;; Resolve name-based source table refs to table IDs (throws if any not found)
(let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))]
(with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
(let [driver (:engine db)
server-url (transforms-python.settings/python-runner-url)
- _ (python-runner/copy-tables-to-s3! {:run-id run-id
- :shared-storage @shared-storage-ref
- :source (assoc source :source-tables resolved-source-tables)
- :cancel-chan cancel-chan
- :limit (:limit source)
- :transform-id (:id transform)})
+ _ (python-runner/copy-tables-to-s3! {:run-id run-id
+ :shared-storage @shared-storage-ref
+ :source (assoc source :source-tables resolved-source-tables)
+ :cancel-chan cancel-chan
+ :limit (:limit source)
+ :transform-id (:id transform)
+ :source-range-params source-range-params})
_ (start-cancellation-process! server-url run-id cancel-chan)
{:keys [status body] :as response}
(python-runner/execute-python-code-http-call!
@@ -305,7 +311,7 @@
:logs <string>
:error <exception if failed>}"
[transform :- ::transforms-base.schema/transform
- {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan]} :- [:maybe ::transforms-base.schema/execute-base-options]]
+ {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan source-range-params]} :- [:maybe ::transforms-base.schema/execute-base-options]]
(assert (transforms-base.u/python-transform? transform) "Transform must be a python transform")
(let [message-log (or message-log (empty-message-log))]
(try
@@ -336,7 +342,8 @@
(log/info "Executing Python transform" transform-id "with target" (pr-str target))
(let [result (run-python-transform-impl! transform db effective-run-id cancel-chan message-log
- {:with-stage-timing-fn with-stage-timing-fn})]
+ {:with-stage-timing-fn with-stage-timing-fn
+ :source-range-params source-range-params})]
(log! message-log (i18n/tru "Python execution finished successfully in {0}"
(u.format/format-milliseconds (u/since-ms start-ms))))
@@ -346,7 +353,8 @@
{:status :succeeded
:result result
- :logs (message-log->string message-log)}))
+ :logs (message-log->string message-log)
+ :source-range-params source-range-params}))
(catch Exception e
(let [data (ex-data e)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 1a88ad71a40..f4f680b03d4 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -110,12 +110,13 @@
(driver.conn/with-write-connection
(let [conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
- run-fn (fn [cancel-chan]
+ run-fn (fn [cancel-chan source-range-params]
(let [result (transforms-base.i/execute-base!
transform
{:cancelled? #(boolean (a/poll! cancel-chan))
:cancel-chan cancel-chan
:run-id run-id
+ :source-range-params source-range-params
:message-log message-log
:with-stage-timing-fn (fn [rid stage thunk]
(transforms.instrumentation/with-stage-timing [rid stage]
@@ -126,10 +127,11 @@
result))
ex-message-fn #(exceptional-run-message message-log %)
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
- (transforms.u/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
+ (transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]
;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
(transforms-base.u/complete-execution! transform
{:run-id run-id
+ :source-range-params (:source-range-params result)
:with-stage-timing-fn (fn [rid stage thunk]
(transforms.instrumentation/with-stage-timing [rid stage]
(thunk)))})
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 49691b82aa6..aea8c43f44b 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -270,14 +270,13 @@
(defn- build-table-query
"Build a mbql query for table, might add a proper filter for incremental transforms."
- [table-id source-incremental-strategy transform-id limit]
+ [table-id source-incremental-strategy source-range-params limit]
(let [db-id (t2/select-one-fn :db_id (t2/table-name :model/Table) :id table-id)
metadata-provider (lib-be/application-database-metadata-provider db-id)
- table-metadata (lib.metadata/table metadata-provider table-id)
- transform (t2/select-one :model/Transform transform-id)]
+ table-metadata (lib.metadata/table metadata-provider table-id)]
(cond-> (-> (lib/query metadata-provider table-metadata)
lib/disable-default-limit)
- source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-incremental-strategy (transforms-base.u/next-checkpoint transform))
+ source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-range-params)
limit (lib/limit limit))))
;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
@@ -289,7 +288,7 @@
source
cancel-chan
limit
- transform-id]}]
+ source-range-params]}]
(when (and (:source-incremental-strategy source)
(> (count (:source-tables source)) 1))
(throw (ex-info "Incremental transforms for python only supports one source table" {})))
@@ -307,7 +306,7 @@
fields-meta (fields-metadata driver table-id)
manifest (generate-manifest table-id fields-meta)]
(transforms.instrumentation/with-stage-timing [run-id [:export :dwh-to-file]]
- (let [query (build-table-query table-id (:source-incremental-strategy source) transform-id limit)]
+ (let [query (build-table-query table-id (:source-incremental-strategy source) source-range-params limit)]
(write-query-data-to-file!
{:query query
:fields-meta fields-meta
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index c527f347b13..ab502aea2a7 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -1,7 +1,6 @@
(ns ^:mb/driver-tests ^:mb/transforms-python-test metabase-enterprise.transforms.incremental-test
"Tests for incremental transforms functionality."
(:require
- [clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
[honey.sql :as sql]
@@ -26,25 +25,22 @@
(def checkpoint-configs
{:integer {:field-name "id"
:template-tag-type :number
- :lib-column-key "column-unique-key-v1$id"
- :expected-initial-checkpoint 10
- :expected-second-checkpoint 16}
+ :expected-initial-checkpoint 16
+ :expected-second-checkpoint 17}
:float {:field-name "price"
:template-tag-type :number
- :lib-column-key "column-unique-key-v1$price"
- :expected-initial-checkpoint 49.99
- :expected-second-checkpoint 199.99}
+ :expected-initial-checkpoint 199.99
+ :expected-second-checkpoint 379.99}
:temporal {:field-name "created_at"
:template-tag-type :text
- :lib-column-key "column-unique-key-v1$created_at"
- :expected-initial-checkpoint "2024-01-10"
- :expected-second-checkpoint "2024-01-16"}})
+ :expected-initial-checkpoint #t "2024-01-16T10:00:00Z"
+ :expected-second-checkpoint #t "2024-01-20T10:00:00Z"}})
(defn- valid-checkpoint-transform-combo?
- "Check if a checkpoint type and transform type combination is valid. "
- [checkpoint-type transform-type]
- (not (and (= checkpoint-type :temporal)
- (= transform-type :native))))
+ "Check if a checkpoint type and transform type combination is valid.
+ With table tag expansion, all combinations are now supported."
+ [_checkpoint-type _transform-type]
+ true)
(defn- build-table-query-with-order-by
"Wraps the original build-table-query to add ORDER BY for deterministic test results."
@@ -77,49 +73,33 @@
(transforms.execute/execute! transform run-opts))
(transforms.execute/execute! transform run-opts)))
-(defn- make-incremental-source-query
- "Create a native query with optional checkpoint template tag."
- [schema checkpoint-config]
- (let [{:keys [field-name template-tag-type]} checkpoint-config
+(defn- make-incremental-source-query-with-table-tag
+ "Create a native query with a table tag for testing subquery expansion."
+ [checkpoint-config]
+ (let [{:keys [field-name]} checkpoint-config
+ table-id (mt/id :transforms_products)
timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
- query (format "SELECT *, %s AS %s FROM %s [[WHERE %s > {{checkpoint}}]] ORDER BY %s LIMIT 10"
+ query (format "SELECT *, %s AS %s FROM {{source_table}} AS %s ORDER BY %s"
timestamp-sql
(sql.u/quote-name driver/*driver* :field "load_timestamp")
- (if schema
- (sql.u/quote-name driver/*driver* :table schema "transforms_products")
- "transforms_products")
- (sql.u/quote-name driver/*driver* :field field-name)
+ (sql.u/quote-name driver/*driver* :field "source_table")
(sql.u/quote-name driver/*driver* :field field-name))]
{:database (mt/id)
:type :native
:native {:query query
- :template-tags {"checkpoint" {:id "checkpoint"
- :name "checkpoint"
- :display-name "Checkpoint"
- :type template-tag-type
- :required false}}}}))
-
-(defn- make-incremental-source-query-without-template-tag
- "Create a native query without template tags for testing automatic checkpoint insertion. "
- [schema]
- (let [timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
- query (format "SELECT *, %s AS %s FROM %s"
- timestamp-sql
- (sql.u/quote-name driver/*driver* :field "load_timestamp")
- (if schema
- (sql.u/quote-name driver/*driver* :table schema "transforms_products")
- "transforms_products"))]
- {:database (mt/id)
- :type :native
- :native {:query query}}))
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}))
(defn- make-incremental-mbql-query
"Create an MBQL query for incremental transforms. "
[checkpoint-config]
(let [{:keys [field-name]} checkpoint-config]
(mt/mbql-query transforms_products {:expressions {"load_timestamp" [:now]}
- :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]
- :limit 10})))
+ :order-by [[:asc [:field (mt/id :transforms_products (keyword field-name))]]]})))
(def incremental-python-body
(str "import pandas as pd\n"
@@ -132,32 +112,28 @@
(defn- make-incremental-transform-payload
"Create a transform payload for incremental transform testing.
- transform-type can be :native, :native-auto-wrap, :mbql, or :python
+ transform-type can be :native, :mbql, or :python
checkpoint-config should be from checkpoint-configs"
[transform-name target-table transform-type checkpoint-config]
- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- {:keys [field-name lib-column-key]} checkpoint-config]
+ (let [{:keys [field-name]} checkpoint-config
+ checkpoint-filter-field (fn [] (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products)))]
{:name transform-name
:source_database_id (mt/id)
:source (case transform-type
+ ;; Native queries now use table tags with checkpoint-filter-field-id
:native {:type "query"
- :query (make-incremental-source-query schema checkpoint-config)
+ :query (make-incremental-source-query-with-table-tag checkpoint-config)
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter field-name}}
- :native-auto-wrap {:type "query"
- :query (make-incremental-source-query-without-template-tag schema)
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter field-name}}
+ :checkpoint-filter-field-id (checkpoint-filter-field)}}
:mbql {:type "query"
:query (make-incremental-mbql-query checkpoint-config)
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter-unique-key lib-column-key}}
+ :checkpoint-filter-field-id (checkpoint-filter-field)}}
:python {:type "python"
:source-tables [{:alias "transforms_products" :table_id (mt/id :transforms_products)}]
- :limit 10
:body incremental-python-body
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter-unique-key lib-column-key}})
+ :checkpoint-filter-field-id (checkpoint-filter-field)}})
:target (merge target-table {:type "table-incremental"
:target-incremental-strategy {:type "append"}})}))
@@ -182,9 +158,14 @@
result (qp/process-query native-query)]
(some-> result :data :rows first first bigint)))
-(defn get-checkpoint-value [transform]
- (#'transforms-base.u/next-checkpoint-value
- (transforms-base.u/next-checkpoint transform)))
+(defn- get-checkpoint-value [transform-id]
+ (let [transform (t2/select-one :model/Transform transform-id)
+ {:keys [last_checkpoint_value]} transform]
+ (when last_checkpoint_value
+ (let [source-incremental-strategy (get-in transform [:source :source-incremental-strategy])
+ field-id (:checkpoint-filter-field-id source-incremental-strategy)
+ field (t2/select-one :model/Field field-id)]
+ (#'transforms-base.u/parse-checkpoint-value (:base_type field) last_checkpoint_value)))))
(defn- compare-checkpoint-values
"Compare two checkpoint values with type-appropriate logic. "
@@ -193,8 +174,7 @@
:integer (= (bigint expected) (bigint actual))
:float (and (number? actual)
(< (Math/abs (double (- expected actual))) 0.01))
- :temporal (and (string? actual)
- (str/starts-with? actual expected))))
+ :temporal (= expected actual)))
(defn- insert-test-products!
"Insert new products into the transforms_products table."
@@ -235,7 +215,7 @@
(disj (mt/normal-drivers-with-feature :transforms/table) :redshift :clickhouse :sqlserver))
(defn- target-table-gen [prefix]
- {:type "table"
+ {:type :table
:name prefix
:schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
:database (mt/id)})
@@ -251,6 +231,7 @@
(with-transform-cleanup! [target-table (target-table-gen "incremental_test")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
{:keys [field-name]} checkpoint-config
+ expected-field-id (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products))
transform-payload (make-incremental-transform-payload "Test Incremental Transform" target-table :native checkpoint-config)]
(testing "Transform is created successfully"
(mt/with-temp [:model/Transform transform transform-payload]
@@ -258,10 +239,10 @@
(is (= "Test Incremental Transform" (:name transform)))
(is (= "table-incremental" (-> transform :target :type)))
(is (= "checkpoint" (-> transform :source :source-incremental-strategy :type)))
- (is (= field-name (-> transform :source :source-incremental-strategy :checkpoint-filter)))
+ (is (= expected-field-id (-> transform :source :source-incremental-strategy :checkpoint-filter-field-id)))
(testing "No checkpoint exists initially"
- (is (nil? (get-checkpoint-value transform))))
+ (is (nil? (get-checkpoint-value (:id transform)))))
(testing "Can retrieve transform via API"
(let [retrieved (mt/user-http-request :crowberto :get 200 (format "transform/%d" (:id transform)))]
@@ -293,23 +274,44 @@
(transforms.tu/wait-for-table (:name target-table) 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 10 row-count) "First run should process the first 10 products")
+ (is (= 16 row-count) "First run should process all 16 products")
(is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
(testing "Checkpoint is created after first run"
- (let [checkpoint (get-checkpoint-value transform)]
+ (let [checkpoint (get-checkpoint-value (:id transform))]
(is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)
- (format "Checkpoint should be MAX(%s) from first 10 rows" (:field-name checkpoint-config)))))))
+ (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config)))))))
- (testing "Second run should process the remaining rows"
- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
- distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 16 row-count) "Second run should add remaining 6 rows")
- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")
- (let [checkpoint (get-checkpoint-value transform)]
- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
- (format "Checkpoint should be MAX(%s) from all 16 rows" (:field-name checkpoint-config))))))))))))))))
+ (testing "Second run with no new data adds nothing"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows, no new data")
+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))
+
+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)
+ (not= driver/*driver* :clickhouse)
+ (not= driver/*driver* :snowflake))
+ (testing "After inserting new data, incremental run appends only new rows"
+ (with-insert-test-products!
+ [{:name "Incremental Twice Product"
+ :category "Gadget"
+ :price 379.99
+ :created-at "2024-01-20T10:00:00"}]
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
+ (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")
+ (if (= checkpoint-type :integer)
+ ;; Integer checkpoint uses auto-increment ID, just verify it advanced
+ (is (> checkpoint expected-initial-checkpoint)
+ "Integer checkpoint should advance past initial value")
+ (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from all 17 rows" (:field-name checkpoint-config)))))))))))))))))))
(deftest switch-incremental-to-non-incremental-test
(testing "Switching an incremental transform to non-incremental overwrites data"
@@ -325,47 +327,48 @@
{:keys [expected-initial-checkpoint]} checkpoint-config
initial-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)]
(mt/with-temp [:model/Transform transform initial-payload]
- (testing "Initial incremental run processes first batch"
- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (transforms.tu/wait-for-table (:name target-table) 10000)
- (let [row-count (get-table-row-count target-table)
- distinct-timestamps (get-distinct-timestamp-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 10 row-count) "Initial run should process first 10 products")
- (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
- (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))
-
- (testing "Second incremental run processes remaining data"
- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
- distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 16 row-count) "Should have 16 rows after second incremental run")
- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")))
-
- (testing "Switch to non-incremental via PUT API"
- (let [non-incremental-payload (-> initial-payload
- (update :source dissoc :source-incremental-strategy)
- (update :target dissoc :source-incremental-strategy)
- (update :target assoc :type "table"))
- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))
- non-incremental-payload)]
- (is (= "table" (-> updated :target :type)))
- (is (nil? (-> updated :source :source-incremental-strategy)))))
-
- (testing "Non-incremental run overwrites all data"
- (let [transform (t2/select-one :model/Transform (:id transform))]
+ (let [get-transform #(t2/select-one :model/Transform (:id transform))]
+ (testing "Initial incremental run processes all data"
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "Initial run should process all 16 products")
+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be created")))
+
+ (testing "Second incremental run with no new data"
+ (execute-transform-with-ordering! (get-transform) transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 10 row-count) "Should overwrite to 10 rows")
- (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))
-
- (testing "Running again still overwrites"
+ (is (= 16 row-count) "Should still have 16 rows, no new data")
+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))
+
+ (testing "Switch to non-incremental via PUT API"
+ (let [non-incremental-payload (-> initial-payload
+ (update :source dissoc :source-incremental-strategy)
+ (update :target dissoc :source-incremental-strategy)
+ (update :target assoc :type "table"))
+ updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))
+ non-incremental-payload)]
+ (is (= "table" (-> updated :target :type)))
+ (is (nil? (-> updated :source :source-incremental-strategy)))))
+
+ (testing "Non-incremental run overwrites all data"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
+ (let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 10 row-count) "Should still have 10 rows after another run")
- (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp")))))))))))))))
+ (is (= 16 row-count) "Should overwrite to 16 rows")
+ (is (= 1 distinct-timestamps) "All rows should have same timestamp after non-incremental overwrite"))
+
+ (testing "Running again still overwrites"
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows after another run")
+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))))))))))))))
(deftest switch-non-incremental-to-incremental-test
(testing "Switching a non-incremental transform to incremental computes checkpoint from existing data"
@@ -378,7 +381,7 @@
(mt/dataset transforms-dataset/transforms-test
(with-transform-cleanup! [target-table (target-table-gen "switch_non_incr_to_incr")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
- {:keys [expected-second-checkpoint]} checkpoint-config
+ {:keys [expected-initial-checkpoint]} checkpoint-config
incremental-payload (make-incremental-transform-payload "Switch Transform" target-table transform-type checkpoint-config)
initial-payload (-> incremental-payload
(update :source dissoc :source-incremental-strategy)
@@ -390,11 +393,10 @@
(transforms.tu/wait-for-table (:name target-table) 10000)
(let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)]
- (is (= 10 row-count) "Initial run should process 10 products")
+ (is (= 16 row-count) "Initial run should process all 16 products")
(is (= 1 distinct-timestamps) "All rows should have same timestamp from non-incremental run")
-
(testing "No checkpoint exists"
- (let [checkpoint (get-checkpoint-value transform)]
+ (let [checkpoint (get-checkpoint-value (:id transform))]
(is (nil? checkpoint) "No checkpoint for non-incremental transform")))))
(testing "Switch to incremental via PUT API"
@@ -403,15 +405,23 @@
(is (= "table-incremental" (-> updated :target :type)))
(is (= "checkpoint" (-> updated :source :source-incremental-strategy :type)))))
- (testing "First incremental run after switch processes no new data"
+ (testing "First incremental run after switch recreates table with checkpoint"
(let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
+ (let [row-count (get-table-row-count target-table)
distinct-timestamps (get-distinct-timestamp-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 16 row-count) "Should process remaining 6 entries")
- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamp")
- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "Should have all 16 rows")
+ (is (= 1 distinct-timestamps) "Should have 1 distinct timestamp")
+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint) "Checkpoint should be computed from existing data"))))
+
+ (testing "Second incremental run with no new data"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows, no new data")
+ (is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))
(when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
(not= driver/*driver* :clickhouse)
@@ -426,80 +436,34 @@
(let [transform (t2/select-one :model/Transform (:id transform))]
(execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
(let [row-count (get-table-row-count target-table)
- checkpoint (get-checkpoint-value transform)]
+ checkpoint (get-checkpoint-value (:id transform))]
(is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
;; For integer checkpoints, we can verify exact value >= 17
;; For float/temporal, just verify checkpoint exists
(is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))
-(deftest native-query-without-template-tag-test
- (testing "Native query without template tags uses automatic checkpoint wrapping"
- (doseq [checkpoint-type [:integer :float :temporal]]
- (testing (format "with %s checkpoint" (name checkpoint-type))
- (mt/test-drivers (test-drivers)
- (mt/with-premium-features #{:transforms-basic}
- (mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table (target-table-gen "native_no_template")]
- (let [checkpoint-config (get checkpoint-configs checkpoint-type)
- {:keys [expected-second-checkpoint field-name]} checkpoint-config
- schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- transform-payload {:name "Native Without Template Tag"
- :source {:type "query"
- :query (make-incremental-source-query-without-template-tag schema)
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter field-name}}
- :target (merge target-table
- {:type "table-incremental"
- :target-incremental-strategy {:type "append"}})}]
- (mt/with-temp [:model/Transform transform transform-payload]
- (testing "First run processes all existing data"
- (transforms.execute/execute! transform {:run-method :manual})
- (transforms.tu/wait-for-table (:name target-table) 10000)
- (let [row-count (get-table-row-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 16 row-count) "First run should process all 16 products")
- (is (compare-checkpoint-values checkpoint-type expected-second-checkpoint checkpoint)
- (format "Checkpoint should be MAX(%s) = %s" (:field-name checkpoint-config) expected-second-checkpoint))))
-
- (testing "Second run without new data adds nothing"
- (transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)]
- (is (= 16 row-count) "Should still have 16 rows, no new data")))
-
- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
- (not= driver/*driver* :clickhouse)
- ;; this *should* work see #68965 for context, will plan follow-up task
- (not= driver/*driver* :snowflake))
- (testing "After inserting new data, incremental run appends only new rows"
- (with-insert-test-products!
- [{:name "New Product 1"
- :category "Electronics"
- :price 299.99
- :created-at "2024-01-21T10:00:00"}
- {:name "New Product 2"
- :category "Books"
- :price 319.99
- :created-at "2024-01-21T11:00:00"}]
-
- (transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 18 row-count) "Should append 2 new rows (16 + 2 = 18)")
- (is (some? checkpoint) "Checkpoint should be updated")))))))))))))))
-
(deftest unsupported-checkpoint-column-type-test
(testing "Transform fails at runtime with unsupported checkpoint column type"
- (mt/test-drivers #{:h2 :postgres}
+ (mt/test-drivers #{:postgres}
(mt/with-premium-features #{:transforms-basic}
(mt/dataset transforms-dataset/transforms-test
(with-transform-cleanup! [target-table "unsupported_type_test"]
- (let [schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- ;; Create transform with text column (unsupported type) as checkpoint
+ (let [name-field-id (mt/id :transforms_products :name)
+ table-id (mt/id :transforms_products)
+ schema (t2/select-one-fn :schema :model/Table table-id)
transform-payload {:name "Invalid Checkpoint Type Transform"
:source {:type "query"
- :query (make-incremental-source-query-without-template-tag schema)
+ :query {:database (mt/id)
+ :type :native
+ :native {:query "SELECT * FROM {{source_table}} AS s"
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "name"}}
+ :checkpoint-filter-field-id name-field-id}}
:target {:type "table-incremental"
:schema schema
:name target-table
@@ -510,266 +474,219 @@
(is (string? response))
(is (re-find #"unsupported type" response)))))))))))
-(deftest ^:postgres-only native-query-with-temporal-checkpoint-test
- (testing "Native query with temporal checkpoint"
- ;; we test only in postgres because it's easy to cast to ::timestamp
- (mt/test-drivers [:postgres]
- (mt/with-premium-features #{:transforms-basic}
- (mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table (target-table-gen "native_temporal_cast")]
- (let [checkpoint-config (get checkpoint-configs :temporal)
- {:keys [expected-initial-checkpoint expected-second-checkpoint field-name]} checkpoint-config
- schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- timestamp-sql (first (sql/format (sql.qp/current-datetime-honeysql-form driver/*driver*)))
- query (format "SELECT *, %s AS load_timestamp FROM %s [[WHERE %s > {{checkpoint}}::timestamp]] ORDER BY %s LIMIT 10"
- timestamp-sql
- (if schema
- (sql.u/quote-name driver/*driver* :table schema "transforms_products")
- "transforms_products")
- field-name
- field-name)
- transform-payload {:name "Native With Temporal Cast"
- :source {:type "query"
- :query {:database (mt/id)
- :type :native
- :native {:query query
- :template-tags {"checkpoint" {:id "checkpoint"
- :name "checkpoint"
- :display-name "Checkpoint"
- :type :text
- :required false}}}}
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter field-name}}
- :target (merge target-table
- {:type "table-incremental"
- :target-incremental-strategy {:type "append"}})}]
- (mt/with-temp [:model/Transform transform transform-payload]
- (testing "First run processes first batch"
- (transforms.execute/execute! transform {:run-method :manual})
- (transforms.tu/wait-for-table (:name target-table) 10000)
- (let [row-count (get-table-row-count target-table)
- distinct-timestamps (get-distinct-timestamp-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 10 row-count) "First run should process the first 10 products")
- (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
- (is (compare-checkpoint-values :temporal expected-initial-checkpoint checkpoint)
- (format "Checkpoint should be MAX(%s) from first 10 rows" field-name))))
-
- (testing "Second run processes remaining data"
- (transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
- distinct-timestamps (get-distinct-timestamp-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 16 row-count) "Second run should add remaining 6 rows")
- (is (= 2 distinct-timestamps) "Should have 2 distinct timestamps (one per incremental run)")
- (is (compare-checkpoint-values :temporal expected-second-checkpoint checkpoint)
- (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))
-
- (testing "Third run without new data adds nothing"
- (transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)]
- (is (= 16 row-count) "Should still have 16 rows, no new data")))
-
- (testing "After inserting new data, incremental run appends only new rows"
- (with-insert-test-products!
- [{:name "New Temporal Product"
- :category "Electronics"
- :price 299.99
- :created-at "2024-01-21T10:00:00"}]
-
- (transforms.execute/execute! transform {:run-method :manual})
- (let [row-count (get-table-row-count target-table)
- checkpoint (get-checkpoint-value transform)]
- (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
- (is (some? checkpoint) "Checkpoint should be updated"))))))))))))
-
(defn- pg-table-rows [db-spec table-name]
(next.jdbc/execute! db-spec [(format "SELECT * FROM %s" table-name)] {:builder-fn jdbc.rs/as-unqualified-lower-maps}))
(deftest empty-table-test
- (mt/test-drivers #{:postgres} ; no db specifics
+ (mt/test-drivers #{:postgres}
(mt/with-premium-features #{:transforms-basic}
- (with-transform-cleanup! [target-table "empty_table_target"]
- (let [db-id (mt/id)
- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT * FROM (VALUES (42)) x(id) WHERE 1 = 2"}}
- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}
- target {:type "table-incremental"
- :schema "public"
- :name target-table
- :database db-id
- :target-incremental-strategy {:type "append"}}]
- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
- (transforms.execute/execute! transform {:run-method :manual})
- (testing "still creates target table"
- (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))
- (testing "sync has picked up table"
- (is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))
- (testing "checkpoint is recognized"
- (is (some? (transforms-base.u/next-checkpoint transform))))))))))
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "empty_table_target"]
+ (let [db-id (mt/id)
+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)
+ table-id (mt/id :transforms_products)
+ checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)
+ ;; Query that returns no rows but has the checkpoint column
+ source {:type "query"
+ :query {:database db-id
+ :type :native
+ :native {:query "SELECT * FROM {{source_table}} AS s WHERE 1 = 2"
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id checkpoint-field-id}}
+ target {:type "table-incremental"
+ :schema "public"
+ :name target-table
+ :database db-id
+ :target-incremental-strategy {:type "append"}}]
+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (testing "still creates target table"
+ (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))
+ (testing "sync has picked up table"
+ (is (=? {:name target-table} (t2/select-one :model/Table :name target-table))))
+ (testing "checkpoint is set from source table MAX even though query returned no rows"
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (is (some? (:last_checkpoint_value transform))
+ "Checkpoint is computed from source table, not query output"))))))))))
(deftest checkpoint-field-does-not-exist-test
- (mt/test-drivers #{:postgres} ; no db specifics
- (mt/with-premium-features #{:transforms-basic}
- (with-transform-cleanup! [target-table "missing_field_target"]
- (let [db-id (mt/id)
- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT * FROM (VALUES (42)) x(id)"}}
- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "no_such_column"}}
- target {:type "table-incremental"
- :schema "public"
- :name target-table
- :database db-id
- :target-incremental-strategy {:type "append"}}]
- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
- (transforms.execute/execute! transform {:run-method :manual})
- (testing "still creates target table"
- (is (= 1 (count (next.jdbc/execute! db-spec ["SELECT true FROM information_schema.tables WHERE table_name = ?" target-table])))))
- (testing "sync has picked up table"
- (is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))
- (testing "target table has expected data"
- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))
- (testing "checkpoint is not recognized, so transform acts as if no checkpoint"
- (is (nil? (transforms-base.u/next-checkpoint transform))))
- ;; Maybe this is unrealistic - you cannot select a column that does not exist
- ;; But the source tables schema can change, e.g. rename: you change event_time to event_ts or something
- ;; so one would have to be careful to disable or delete transforms ahead of a schema change like this
- ;; For now asserting behaviour is-what-it-is, but this should provoke a hmmm... maybe it is better to fail early?
- (testing "running a second time will duplicate the existing data"
- (transforms.execute/execute! transform {:run-method :manual})
- (is (= [{:id 42} {:id 42}] (pg-table-rows db-spec target-table))))))))))
+ (testing "Transform with non-existent checkpoint-filter-field-id fails gracefully"
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "missing_field_target"]
+ (let [db-id (mt/id)
+ table-id (mt/id :transforms_products)
+ invalid-field-id 999999
+ source {:type "query"
+ :query {:database db-id
+ :type :native
+ :native {:query "SELECT id FROM {{source_table}}"
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id invalid-field-id}}
+ target {:type "table-incremental"
+ :schema "public"
+ :name target-table
+ :database db-id
+ :target-incremental-strategy {:type "append"}}]
+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
+ (testing "Transform execution fails with invalid checkpoint field ID"
+ (is (thrown-with-msg?
+ Exception
+ #"Checkpoint field does not exist"
+ (transforms.execute/execute! transform {:run-method :manual}))))))))))))
+
+(deftest checkpoint-field-is-not-active-test
+ (testing "Transform with inactive checkpoint field fails gracefully"
+ (mt/test-drivers #{:postgres}
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "inactive_field_target"]
+ (let [db-id (mt/id)
+ table-id (mt/id :transforms_products)
+ field-id (mt/id :transforms_products :id)
+ source {:type "query"
+ :query {:database db-id
+ :type :native
+ :native {:query "SELECT id FROM {{source_table}} as s"
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id field-id}}
+ target {:type "table-incremental"
+ :schema "public"
+ :name target-table
+ :database db-id
+ :target-incremental-strategy {:type "append"}}]
+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
+ ;; Mark the field as inactive
+ (t2/update! :model/Field field-id {:active false})
+ (try
+ (testing "Transform execution fails with inactive checkpoint field"
+ (is (thrown-with-msg?
+ Exception
+ #"Checkpoint field does not exist or is not active"
+ (transforms.execute/execute! transform {:run-method :manual}))))
+ (finally
+ ;; Restore field to active state for other tests
+ (t2/update! :model/Field field-id {:active true})))))))))))
(deftest changing-query-keeps-checkpoint-test
- (mt/test-drivers #{:postgres} ; no db specifics
+ (mt/test-drivers #{:postgres}
(mt/with-premium-features #{:transforms-basic}
- (with-transform-cleanup! [target-table "change_table_target"]
- (let [db-id (mt/id)
- db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)
- source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT * FROM (VALUES (42)) x(id)"}}
- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}
- target {:type "table-incremental"
- :schema "public"
- :name target-table
- :database db-id
- :target-incremental-strategy {:type "append"}}]
- (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
- (testing "initial run"
- (transforms.execute/execute! transform {:run-method :manual})
- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))
- (testing "initial increment"
- (transforms.execute/execute! transform {:run-method :manual})
- (is (= [{:id 42}] (pg-table-rows db-spec target-table))))
- (t2/update! :model/Transform
- (:id transform)
- {:source {:type "query"
- :query {:database db-id
- :type :native
- :native {:query "SELECT * FROM (VALUES (42), (43)) x(id)"}}
- :source-incremental-strategy {:type "checkpoint", :checkpoint-filter "id"}}})
- (testing "second increment"
- (transforms.execute/execute! (t2/select-one :model/Transform (:id transform)) {:run-method :manual})
- (is (= [{:id 42} {:id 43}] (pg-table-rows db-spec target-table))))))))))
-
-(deftest filter-column-indexed-test
- (testing "Filter column is indexed"
- (doseq [checkpoint-type [:integer]
- transform-type [:native :mbql :python]
- :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
- (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
- (mt/with-premium-features #{:transforms-basic :transforms-python}
+ (mt/dataset transforms-dataset/transforms-test
+ (with-transform-cleanup! [target-table "change_table_target"]
+ (let [db-id (mt/id)
+ db-spec (sql-jdbc.conn/db->pooled-connection-spec db-id)
+ table-id (mt/id :transforms_products)
+ checkpoint-field-id (t2/select-one-pk :model/Field :name "id" :table_id table-id)
+ make-source (fn [query-sql]
+ {:type "query"
+ :query {:database db-id
+ :type :native
+ :native {:query query-sql
+ :template-tags {"source_table" {:id "source_table"
+ :name "source_table"
+ :display-name "Source Table"
+ :type "table"
+ :table-id table-id
+ :required true}}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id checkpoint-field-id}})
+ source (make-source "SELECT id FROM {{source_table}} AS s")
+ target {:type "table-incremental"
+ :schema "public"
+ :name target-table
+ :database db-id
+ :target-incremental-strategy {:type "append"}}]
+ (mt/with-temp [:model/Transform transform {:name "test transform" :source source, :target target}]
+ (testing "initial run processes all rows"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= 16 (count (pg-table-rows db-spec target-table)))))
+ (testing "second run with no new data"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= 16 (count (pg-table-rows db-spec target-table)))))
+ (t2/update! :model/Transform
+ (:id transform)
+ {:source (make-source "SELECT id FROM {{source_table}} AS s ORDER BY id DESC")})
+ (testing "run after query change with no new data keeps same row count"
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform)) {:run-method :manual})
+ (is (= 16 (count (pg-table-rows db-spec target-table)))
+ "Checkpoint preserved across query change, no new rows added"))
+ (testing "after inserting new data, incremental run appends"
+ (with-insert-test-products!
+ [{:name "New Product" :category "Test" :price 99.99 :created-at "2024-01-21T10:00:00"}]
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= 17 (count (pg-table-rows db-spec target-table)))
+ "Should append 1 new row after query change")))))))))))
+
+(deftest native-query-with-table-tag-test
+ (testing "Native query with table tag uses subquery expansion for incremental filtering"
+ (doseq [checkpoint-type [:integer :float]]
+ (testing (format "with %s checkpoint" (name checkpoint-type))
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
(mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table (target-table-gen "incremental_index")]
+ (with-transform-cleanup! [target-table (target-table-gen "native_table_tag")]
(let [checkpoint-config (get checkpoint-configs checkpoint-type)
- transform-payload (make-incremental-transform-payload "Incremental Transform" target-table transform-type checkpoint-config)]
+ {:keys [field-name expected-initial-checkpoint]} checkpoint-config
+ checkpoint-filter-field-id (t2/select-one-pk :model/Field :name field-name :table_id (mt/id :transforms_products))
+ transform-payload {:name "Native With Table Tag"
+ :source {:type "query"
+ :query (make-incremental-source-query-with-table-tag checkpoint-config)
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id checkpoint-filter-field-id}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "append"}})}]
(mt/with-temp [:model/Transform transform transform-payload]
- (testing "First run creates index on checkpoint column"
- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
- field-name (:field-name checkpoint-config)]
- (testing "Index was created"
- (is (= 1 (count indexes)))
- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes))))))
- (testing "Data was processed correctly"
- (is (= 10 (get-table-row-count target-table))))
- (testing "Second run succeeds with existing index"
- (execute-transform-with-ordering! transform transform-type (:field-name checkpoint-config) {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
- field-name (:field-name checkpoint-config)]
- (testing "Index still exists"
- (is (= 1 (count indexes)))
- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))
- (testing "Data was processed correctly"
- (is (= 16 (get-table-row-count target-table)))))))))))))))
-
-(deftest index-cleanup-on-switch-to-non-incremental-test
- (testing "Switching to non-incremental removes metabase-owned indexes"
- (doseq [checkpoint-type [:integer]
- transform-type [:native :mbql :python]
- :when (valid-checkpoint-transform-combo? checkpoint-type transform-type)]
- (testing (format "with %s checkpoint on %s transform" (name checkpoint-type) (name transform-type))
- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
- (mt/with-premium-features #{:transforms-basic :transforms-python}
- (mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table (target-table-gen "index_cleanup_non_incr")]
- (let [checkpoint-config (get checkpoint-configs checkpoint-type)
- {:keys [field-name]} checkpoint-config
- incremental-payload (make-incremental-transform-payload "Index Cleanup Transform" target-table transform-type checkpoint-config)]
- (mt/with-temp [:model/Transform transform incremental-payload]
- (testing "First incremental run creates index"
- (execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
- (is (=? {:value field-name :index-name #"^mb_transform_idx_.*$"} (first indexes)))))
- (testing "Switch to non-incremental via API"
- (let [non-incremental-payload (-> incremental-payload
- (update :source dissoc :source-incremental-strategy))
- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))
- non-incremental-payload)]
- (is (nil? (:source-incremental-strategy (:source updated))))))
- (testing "Non-incremental run removes automatic indexes indexes"
+ (testing "First run processes all data"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (let [row-count (get-table-row-count target-table)
+ distinct-timestamps (get-distinct-timestamp-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 16 row-count) "First run should process all 16 products")
+ (is (= 1 distinct-timestamps) "All rows should have the same timestamp from first run")
+ (is (compare-checkpoint-values checkpoint-type expected-initial-checkpoint checkpoint)
+ (format "Checkpoint should be MAX(%s) from all 16 rows" field-name))))
+
+ (testing "Second run without new data adds nothing"
(let [transform (t2/select-one :model/Transform (:id transform))]
- (execute-transform-with-ordering! transform transform-type field-name {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)
- mb-indexes (filter #(str/starts-with? (:index-name %) "mb_transform_idx_") indexes)]
- (testing "Automatic indexes are dropped"
- (is (empty? mb-indexes))))))))))))))))
-
-(deftest index-cleanup-on-checkpoint-column-change-test
- (testing "Changing checkpoint column removes old index and creates new one"
- (mt/test-drivers (set/intersection (test-drivers) (mt/normal-drivers-with-feature :transforms/index-ddl))
- (mt/with-premium-features #{:transforms-basic}
- (mt/dataset transforms-dataset/transforms-test
- (with-transform-cleanup! [target-table (target-table-gen "icchange")]
- (let [integer-config (get checkpoint-configs :integer)
- float-config (get checkpoint-configs :float)
- initial-field (:field-name integer-config)
- new-field (:field-name float-config)
- transform-type :native
- initial-payload (make-incremental-transform-payload "Column Change Transform" target-table transform-type integer-config)]
- (mt/with-temp [:model/Transform transform initial-payload]
- (testing "First run with integer checkpoint creates index on id column"
- (execute-transform-with-ordering! transform transform-type initial-field {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
- (is (= 1 (count indexes)))
- (is (=? {:value initial-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))
- (testing "Switch checkpoint column via API"
- (let [new-payload (make-incremental-transform-payload "Column Change Transform" target-table transform-type float-config)
- updated (mt/user-http-request :crowberto :put 200 (format "transform/%d" (:id transform))
- new-payload)]
- (is (= new-field (-> updated :source :source-incremental-strategy :checkpoint-filter)))))
- (testing "Run with new checkpoint column updates indexes"
- (let [transform (t2/select-one :model/Transform (:id transform))]
- (execute-transform-with-ordering! transform transform-type new-field {:run-method :manual})
- (let [indexes (driver/describe-table-indexes driver/*driver* (mt/id) target-table)]
- (testing "Old index removed, new checkpoint column is now indexed"
- (is (= 1 (count indexes)))
- (is (=? {:value new-field :index-name #"^mb_transform_idx_.*$"} (first indexes)))))))))))))))
+ (transforms.execute/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)]
+ (is (= 16 row-count) "Should still have 16 rows, no new data"))))
+
+ (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)
+ (not= driver/*driver* :clickhouse)
+ (not= driver/*driver* :snowflake))
+ (testing "After inserting new data, incremental run appends only new rows"
+ (with-insert-test-products!
+ [{:name "New Table Tag Product"
+ :category "Electronics"
+ :price 299.99
+ :created-at "2024-01-21T10:00:00"}]
+
+ (let [transform (t2/select-one :model/Transform (:id transform))]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (let [row-count (get-table-row-count target-table)
+ checkpoint (get-checkpoint-value (:id transform))]
+ (is (= 17 row-count) "Should append 1 new row (16 + 1 = 17)")
+ (is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))
diff --git a/frontend/src/metabase-types/analytics/event.ts b/frontend/src/metabase-types/analytics/event.ts
index e60429e6d53..d7fd2cbf4d8 100644
--- a/frontend/src/metabase-types/analytics/event.ts
+++ b/frontend/src/metabase-types/analytics/event.ts
@@ -298,6 +298,7 @@ export type TransformCreateEvent = ValidateEvent<{
export type TransformCreatedEvent = ValidateEvent<{
event: "transform_created";
target_id: number;
+ event_detail: "incremental" | undefined;
}>;
export type TransformRunTagsUpdated = ValidateEvent<{
diff --git a/frontend/src/metabase-types/api/transform.ts b/frontend/src/metabase-types/api/transform.ts
index 11390fde65b..80df541c1f0 100644
--- a/frontend/src/metabase-types/api/transform.ts
+++ b/frontend/src/metabase-types/api/transform.ts
@@ -23,12 +23,14 @@ export type TransformOwner = Pick<
"id" | "email" | "first_name" | "last_name"
>;
+export type TransformType = "native" | "python" | "mbql";
+
export type Transform = {
id: TransformId;
name: string;
description: string | null;
source: TransformSource;
- source_type: "native" | "python" | "mbql";
+ source_type: TransformType;
target: TransformTarget;
collection_id: CollectionId | null;
created_at: string;
@@ -46,6 +48,8 @@ export type Transform = {
owner_email?: string | null;
owner?: TransformOwner | null;
+ last_checkpoint_value?: string | null;
+
// hydrated fields
collection?: Collection | null;
tag_ids?: TransformTagId[];
@@ -61,10 +65,7 @@ export type PythonTransformTableAliases = Record<string, ConcreteTableId>;
export type TransformSourceCheckpointStrategy = {
type: "checkpoint";
- // For native queries
- "checkpoint-filter"?: string;
- // For MBQL and Python queries
- "checkpoint-filter-unique-key"?: string;
+ "checkpoint-filter-field-id": number;
};
export type SourceIncrementalStrategy = TransformSourceCheckpointStrategy;
@@ -132,6 +133,10 @@ export type TransformRun = {
message: string | null;
run_method: TransformRunMethod;
+ checkpoint_filter_field_id?: number | null;
+ checkpoint_lo_value?: string | null;
+ checkpoint_hi_value?: string | null;
+
// hydrated fields
transform?: Transform;
};
@@ -293,21 +298,6 @@ export type UpdatePythonLibraryRequest = {
source: string;
};
-export type ExtractColumnsFromQueryRequest = {
- query: DatasetQuery;
-};
-
-export type ExtractColumnsFromQueryResponse = {
- columns: string[];
-};
-
-export type CheckQueryComplexityRequest = string;
-
-export type QueryComplexity = {
- is_simple: boolean;
- reason: string;
-};
-
export type InspectorFieldStats = {
distinct_count?: number;
nil_percent?: number;
diff --git a/frontend/src/metabase/api/transform.ts b/frontend/src/metabase/api/transform.ts
index d7e3f760390..f3813f04520 100644
--- a/frontend/src/metabase/api/transform.ts
+++ b/frontend/src/metabase/api/transform.ts
@@ -1,17 +1,13 @@
import { isResourceNotFoundError } from "metabase/lib/errors";
import type {
- CheckQueryComplexityRequest,
CreateTransformRequest,
Dataset,
- ExtractColumnsFromQueryRequest,
- ExtractColumnsFromQueryResponse,
GetInspectorLensRequest,
InspectorDiscoveryResponse,
InspectorLens,
ListTransformRunsRequest,
ListTransformRunsResponse,
ListTransformsRequest,
- QueryComplexity,
RunInspectorQueryRequest,
RunTransformResponse,
Transform,
@@ -202,25 +198,13 @@ export const transformApi = Api.injectEndpoints({
invalidatesTags: (_, error) =>
invalidateTags(error, [listTag("transform"), listTag("table")]),
}),
- extractColumnsFromQuery: builder.mutation<
- ExtractColumnsFromQueryResponse,
- ExtractColumnsFromQueryRequest
- >({
- query: (body) => ({
- method: "POST",
- url: "/api/transform/extract-columns",
- body,
- }),
- }),
- checkQueryComplexity: builder.query<
- QueryComplexity,
- CheckQueryComplexityRequest
- >({
- query: (queryString) => ({
+ resetCheckpoint: builder.mutation<void, TransformId>({
+ query: (id) => ({
method: "POST",
- url: "/api/transform/is-simple-query",
- body: { query: queryString },
+ url: `/api/transform/${id}/reset-checkpoint`,
}),
+ invalidatesTags: (_, error, id) =>
+ invalidateTags(error, [idTag("transform", id)]),
}),
getInspectorDiscovery: builder.query<
InspectorDiscoveryResponse,
@@ -264,7 +248,6 @@ export const {
useUpdateTransformMutation,
useDeleteTransformMutation,
useDeleteTransformTargetMutation,
- useExtractColumnsFromQueryMutation,
- useLazyCheckQueryComplexityQuery,
+ useResetCheckpointMutation,
useRunInspectorQueryQuery,
} = transformApi;
diff --git a/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx b/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx
index 1fbc7cce6b9..5814864a204 100644
--- a/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx
+++ b/frontend/src/metabase/forms/components/FormInlineUpdater/FormInlineUpdater.tsx
@@ -10,6 +10,8 @@ type FormInlineUpdaterProps<T, TSuccess> = {
onSuccess?: (value: TSuccess) => void;
onError?: (error: unknown) => void;
debounceMs?: number;
+ /** If provided, called before each update. Return false to skip the update. */
+ onBeforeUpdate?: (values: T) => Promise<boolean> | boolean;
};
const DEFAULT_INLINE_UPDATE_DEBOUNCE_MS = 300;
@@ -23,6 +25,7 @@ const DEFAULT_INLINE_UPDATE_DEBOUNCE_MS = 300;
* @param onSuccess - Optional callback called with the update result on success
* @param onError - Optional callback called with the error on failure
* @param debounceMs - Delay in ms before calling update (default: 300)
+ * @param onBeforeUpdate - Optional callback called before each update. Return false to skip the update.
*
* @example
* ```tsx
@@ -39,6 +42,7 @@ export const FormInlineUpdater = <T, TSuccess>({
onSuccess,
onError,
debounceMs = DEFAULT_INLINE_UPDATE_DEBOUNCE_MS,
+ onBeforeUpdate,
}: FormInlineUpdaterProps<T, TSuccess>) => {
const { initialValues } = useFormikContext<T>();
const updateInProgress = useRef(false);
@@ -85,6 +89,13 @@ export const FormInlineUpdater = <T, TSuccess>({
return;
}
+ if (onBeforeUpdate) {
+ const shouldProceed = await onBeforeUpdate(values);
+ if (!shouldProceed) {
+ return;
+ }
+ }
+
await processUpdate(values);
}, debounceMs);
diff --git a/frontend/src/metabase/transforms/analytics.ts b/frontend/src/metabase/transforms/analytics.ts
index c9435bf6851..86843c604c6 100644
--- a/frontend/src/metabase/transforms/analytics.ts
+++ b/frontend/src/metabase/transforms/analytics.ts
@@ -43,12 +43,15 @@ export function trackTransformCreate({
export function trackTransformCreated({
transformId,
+ isIncremental,
}: {
transformId: TransformId;
+ isIncremental: boolean;
}) {
trackSimpleEvent({
event: "transform_created",
target_id: transformId,
+ event_detail: isIncremental ? "incremental" : undefined,
});
}
diff --git a/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsx b/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsx
new file mode 100644
index 00000000000..1225926a064
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/CheckpointValue/CheckpointValue.tsx
@@ -0,0 +1,24 @@
+import { DateTime } from "metabase/common/components/DateTime";
+import { Tooltip } from "metabase/ui";
+import { isDate } from "metabase-lib/v1/types/utils/isa";
+import type { Field } from "metabase-types/api";
+
+type CheckpointValueProps = {
+ value: string;
+ checkpointField?: Field;
+};
+
+export function CheckpointValue({
+ value,
+ checkpointField,
+}: CheckpointValueProps) {
+ if (isDate(checkpointField)) {
+ return (
+ <Tooltip label={value}>
+ <DateTime value={value} unit="minute" />
+ </Tooltip>
+ );
+ }
+
+ return <>{value}</>;
+}
diff --git a/frontend/src/metabase/transforms/components/CheckpointValue/index.ts b/frontend/src/metabase/transforms/components/CheckpointValue/index.ts
new file mode 100644
index 00000000000..8c09a0a4a4e
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/CheckpointValue/index.ts
@@ -0,0 +1 @@
+export { CheckpointValue } from "./CheckpointValue";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index 6450508e0d1..fcc150acd41 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -1,5 +1,4 @@
import { useFormikContext } from "formik";
-import { match } from "ts-pattern";
import { t } from "ttag";
import { useDocsUrl } from "metabase/common/hooks";
@@ -12,7 +11,7 @@ import {
SOURCE_STRATEGY_OPTIONS,
TARGET_STRATEGY_OPTIONS,
} from "metabase/transforms/constants";
-import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";
+import { getLibQuery } from "metabase/transforms/utils";
import {
Anchor,
Box,
@@ -23,15 +22,17 @@ import {
Text,
Tooltip,
} from "metabase/ui";
-import type * as Lib from "metabase-lib";
+import * as Lib from "metabase-lib";
import type { TransformSource } from "metabase-types/api";
+import type { TransformType } from "metabase-types/api/transform";
import {
MBQLKeysetColumnSelect,
+ NativeQueryTableTagFieldSelect,
PythonKeysetColumnSelect,
} from "./KeysetColumnSelect";
-import { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";
import type { IncrementalSettingsFormValues } from "./form";
+import { useHasCheckpointOptions } from "./useHasCheckpointOptions";
type IncrementalTransformSettingsProps = {
source: TransformSource;
@@ -39,6 +40,7 @@ type IncrementalTransformSettingsProps = {
onIncrementalChange: (value: boolean) => void;
variant?: "embedded" | "standalone";
readOnly?: boolean;
+ extraActions?: React.ReactNode;
};
export const IncrementalTransformSettings = ({
@@ -47,6 +49,7 @@ export const IncrementalTransformSettings = ({
onIncrementalChange,
variant = "embedded",
readOnly,
+ extraActions,
}: IncrementalTransformSettingsProps) => {
const metadata = useSelector(getMetadata);
const libQuery = getLibQuery(source, metadata);
@@ -54,42 +57,55 @@ export const IncrementalTransformSettings = ({
PLUGIN_REMOTE_SYNC.getIsRemoteSyncReadOnly,
);
- // Check if this is a Python transform with exactly one source table
- // Incremental transforms are only supported for single-table Python transforms
- const isPythonTransform = source.type === "python";
+ const { hasCheckpointOptions, transformType } =
+ useHasCheckpointOptions(source);
+
const isMultiTablePythonTransform =
- isPythonTransform && Object.keys(source["source-tables"]).length > 1;
+ getIsPythonTransformWithMultipleTables(source);
+ const isNativeWithoutTableTags = getIsNativeWithoutTableVariables(
+ libQuery,
+ transformType,
+ );
- const transformType = match({
- isMbqlQuery: isMbqlQuery(source, metadata),
- isPythonTransform,
- })
- .with({ isMbqlQuery: true }, () => "query" as const)
- .with({ isPythonTransform: true }, () => "python" as const)
- .otherwise(() => "native" as const);
const { url: incrementalTransformsDocsUrl, showMetabaseLinks } = useDocsUrl(
- isPythonTransform
+ transformType === "python"
? "data-studio/transforms/python-transforms#incremental-python-transforms"
: "data-studio/transforms/query-transforms#incremental-query-transforms",
);
const renderIncrementalSwitch = () => {
+ const getLabel = () => {
+ if (isMultiTablePythonTransform) {
+ return t`Incremental transforms are only supported for single data source transforms.`;
+ }
+ if (isNativeWithoutTableTags) {
+ return t`Incremental transforms for native queries require at least one table variable.`;
+ }
+ if (!hasCheckpointOptions) {
+ return t`Incremental transforms require at least one numeric or temporal source field.`;
+ }
+ return t`Only process new and changed data`;
+ };
+
+ const transformHasIssues =
+ isNativeWithoutTableTags ||
+ !hasCheckpointOptions ||
+ isMultiTablePythonTransform;
+
const switchContent = (
<Switch
disabled={
- readOnly || isMultiTablePythonTransform || isRemoteSyncReadOnly
+ readOnly ||
+ isRemoteSyncReadOnly ||
+ (!incremental && transformHasIssues)
}
checked={incremental}
size="sm"
- label={
- isMultiTablePythonTransform
- ? t`Incremental transforms are only supported for single data source transforms.`
- : t`Only process new and changed data`
- }
+ label={getLabel()}
wrapperProps={{
"data-testid": "incremental-switch",
}}
- onChange={(e) => onIncrementalChange(e.target.checked)}
+ onChange={(event) => onIncrementalChange(event.target.checked)}
/>
);
@@ -109,18 +125,21 @@ export const IncrementalTransformSettings = ({
const label = t`Incremental transformation`;
const renderDescription = () => {
- const description = t`If you don’t need to reprocess all the data, incremental transforms can be faster. To use this, your transform definition should have a stable schema. `;
+ const description = t`If you don’t need to reprocess all the data, incremental transforms can be faster. To use this, your transform definition should have a stable schema.`;
return (
<>
{description}
{showMetabaseLinks && (
- <Anchor
- href={incrementalTransformsDocsUrl}
- target="_blank"
- td="underline"
- c="inherit"
- size="sm"
- >{t`Learn more.`}</Anchor>
+ <>
+ {" "}
+ <Anchor
+ href={incrementalTransformsDocsUrl}
+ target="_blank"
+ td="underline"
+ c="inherit"
+ size="sm"
+ >{t`Learn more.`}</Anchor>
+ </>
)}{" "}
</>
);
@@ -136,10 +155,16 @@ export const IncrementalTransformSettings = ({
<SourceStrategyFields
source={source}
query={libQuery}
- type={transformType}
+ transformType={transformType}
readOnly={readOnly}
/>
</Group>
+ {extraActions && (
+ <>
+ <Divider />
+ <Group p="lg">{extraActions}</Group>
+ </>
+ )}
<TargetStrategyFields variant={variant} />
</>
)}
@@ -161,7 +186,7 @@ export const IncrementalTransformSettings = ({
<SourceStrategyFields
source={source}
query={libQuery}
- type={transformType}
+ transformType={transformType}
/>
<TargetStrategyFields variant={variant} />
</>
@@ -205,17 +230,18 @@ function TargetStrategyFields({
type SourceStrategyFieldsProps = {
source: TransformSource;
query: Lib.Query | null;
- type: "query" | "native" | "python";
+ transformType: TransformType;
readOnly?: boolean;
};
function SourceStrategyFields({
source,
query,
- type,
+ transformType,
readOnly,
}: SourceStrategyFieldsProps) {
const { values } = useFormikContext<IncrementalSettingsFormValues>();
+
return (
<>
{SOURCE_STRATEGY_OPTIONS.length > 1 && (
@@ -229,9 +255,9 @@ function SourceStrategyFields({
)}
{values.sourceStrategy === "checkpoint" && (
<>
- {type === "query" && query && (
+ {transformType === "mbql" && query && (
<MBQLKeysetColumnSelect
- name="checkpointFilterUniqueKey"
+ name="checkpointFilterFieldId"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
description={t`Pick the field that we should scan to determine which records are new or changed`}
@@ -241,20 +267,20 @@ function SourceStrategyFields({
disabled={readOnly}
/>
)}
- {type === "native" && query && (
- <NativeQueryColumnSelect
- name="checkpointFilter"
- label={t`Column to check for new values`}
- placeholder={t`Pick a column`}
- description={t`Pick the column that we should scan to determine which records are new or changed`}
+ {transformType === "native" && query && (
+ <NativeQueryTableTagFieldSelect
+ name="checkpointFilterFieldId"
+ label={t`Field to check for new values`}
+ placeholder={t`Pick a field`}
+ description={t`Pick the field that we should scan to determine which records are new or changed`}
descriptionProps={{ lh: "1rem" }}
query={query}
disabled={readOnly}
/>
)}
- {type === "python" && "source-tables" in source && (
+ {transformType === "python" && "source-tables" in source && (
<PythonKeysetColumnSelect
- name="checkpointFilterUniqueKey"
+ name="checkpointFilterFieldId"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
description={t`Pick the field that we should scan to determine which records are new or changed`}
@@ -268,3 +294,30 @@ function SourceStrategyFields({
</>
);
}
+
+// Check if this is a Python transform with exactly one source table
+// Incremental transforms are only supported for single-table Python transforms
+function getIsPythonTransformWithMultipleTables(source: TransformSource) {
+ const isPythonTransform = source.type === "python";
+ const isMultiTablePythonTransform =
+ isPythonTransform && Object.keys(source["source-tables"]).length > 1;
+
+ return isMultiTablePythonTransform;
+}
+
+function getIsNativeWithoutTableVariables(
+ query: Lib.Query | null,
+ transformType: TransformType,
+) {
+ return transformType === "native" && !queryHasTableVariables(query);
+}
+
+function queryHasTableVariables(query: Lib.Query | null) {
+ const hasTableTemplateTags = query
+ ? Object.values(Lib.templateTags(query)).some(
+ (tag) => tag.type === "table" && tag["table-id"] != null,
+ )
+ : false;
+
+ return hasTableTemplateTags;
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx
new file mode 100644
index 00000000000..aeb92dcb302
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.unit.spec.tsx
@@ -0,0 +1,111 @@
+import { screen } from "@testing-library/react";
+
+import { renderWithProviders } from "__support__/ui";
+import { FormProvider } from "metabase/forms";
+import type { TransformSource } from "metabase-types/api";
+import { createMockTransformSource } from "metabase-types/api/mocks";
+
+import { IncrementalTransformSettings } from "./IncrementalTransformSettings";
+import { getInitialValues } from "./form";
+
+jest.mock("./useHasCheckpointOptions", () => ({
+ useHasCheckpointOptions: jest.fn(),
+}));
+
+jest.mock("metabase/transforms/utils", () => ({
+ getLibQuery: () => null,
+ isMbqlQuery: () => false,
+}));
+
+const { useHasCheckpointOptions } = jest.requireMock(
+ "./useHasCheckpointOptions",
+);
+
+type SetupOpts = {
+ incremental?: boolean;
+ source?: TransformSource;
+ onIncrementalChange?: () => void;
+ hookOverrides?: Partial<ReturnType<typeof useHasCheckpointOptions>>;
+};
+
+function setup({
+ incremental = false,
+ source = createMockTransformSource(),
+ onIncrementalChange = jest.fn(),
+ hookOverrides = {},
+}: SetupOpts = {}) {
+ const hookDefaults = {
+ hasCheckpointOptions: true,
+ hasNativeCheckpointOptions: true,
+ transformType: "query" as const,
+ ...hookOverrides,
+ };
+
+ useHasCheckpointOptions.mockReturnValue(hookDefaults);
+
+ const initialValues = getInitialValues({ incremental });
+
+ renderWithProviders(
+ <FormProvider initialValues={initialValues} onSubmit={jest.fn()}>
+ <IncrementalTransformSettings
+ source={source}
+ incremental={incremental}
+ onIncrementalChange={onIncrementalChange}
+ />
+ </FormProvider>,
+ );
+}
+
+function getSwitch() {
+ return screen.getByRole("switch");
+}
+
+describe("IncrementalTransformSettings", () => {
+ it("disables switch when hasCheckpointOptions is false and incremental is off", () => {
+ setup({
+ incremental: false,
+ hookOverrides: {
+ transformType: "mbql",
+ hasCheckpointOptions: false,
+ },
+ });
+
+ expect(getSwitch()).toBeDisabled();
+ });
+
+ it("enables switch when hasCheckpointOptions is false but incremental is on", () => {
+ setup({
+ incremental: true,
+ hookOverrides: {
+ transformType: "native",
+ hasCheckpointOptions: false,
+ },
+ });
+
+ expect(getSwitch()).toBeEnabled();
+ });
+
+ it("disables switch when hasNativeCheckpointOptions is false and incremental is off", () => {
+ setup({
+ incremental: false,
+ hookOverrides: {
+ transformType: "native",
+ hasNativeCheckpointOptions: false,
+ },
+ });
+
+ expect(getSwitch()).toBeDisabled();
+ });
+
+ it("enables switch when hasNativeCheckpointOptions is false but incremental is on", () => {
+ setup({
+ incremental: true,
+ hookOverrides: {
+ transformType: "native",
+ hasNativeCheckpointOptions: false,
+ },
+ });
+
+ expect(getSwitch()).toBeEnabled();
+ });
+});
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
index 5be50529d22..b45618cb29b 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
@@ -9,6 +9,58 @@ import {
} from "metabase/ui";
import * as Lib from "metabase-lib";
+import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";
+
+/**
+ * Get filterable numeric/temporal field options from the source table of a query.
+ * Returns SelectOption[] with field IDs as string values.
+ */
+export function getSourceFieldOptions(
+ query: Lib.Query,
+ opts?: { labelPrefix?: string; seenFieldIds?: Set<number> },
+): Array<SelectOption> {
+ // Stage 0 is the source stage (raw table data). Keyset pagination requires columns from the
+ // actual source table; later stages may have aggregations, joins, or expressions instead.
+ const stageIndex = 0;
+ const filterableColumns = Lib.filterableColumns(query, stageIndex);
+
+ const groups = Lib.groupColumns(filterableColumns);
+ const sourceGroup = groups.find((group) => {
+ const groupInfo = Lib.displayInfo(query, stageIndex, group);
+ return groupInfo.isSourceTable;
+ });
+
+ if (!sourceGroup) {
+ return [];
+ }
+
+ const sourceColumns = Lib.getColumnsFromColumnGroup(sourceGroup);
+ const seenFieldIds = opts?.seenFieldIds ?? new Set<number>();
+ const options: Array<SelectOption> = [];
+
+ for (const column of sourceColumns) {
+ if (!Lib.isNumeric(column) && !Lib.isTemporal(column)) {
+ continue;
+ }
+
+ const { fieldId } = Lib.fieldValuesSearchInfo(query, column);
+ if (fieldId == null || seenFieldIds.has(fieldId)) {
+ continue;
+ }
+
+ seenFieldIds.add(fieldId);
+
+ const columnInfo = Lib.displayInfo(query, stageIndex, column);
+ const label = opts?.labelPrefix
+ ? `${opts.labelPrefix}: ${columnInfo.displayName}`
+ : columnInfo.longDisplayName;
+
+ options.push({ value: String(fieldId), label });
+ }
+
+ return options;
+}
+
type KeysetColumnSelectProps = {
name: string;
label: string;
@@ -36,45 +88,8 @@ export function KeysetColumnSelect({
}
try {
- // Use -1 to get the last stage
- const stageIndex = -1;
-
- const returnedColumns = Lib.returnedColumns(query, stageIndex);
- const filterableColumns = Lib.filterableColumns(query, stageIndex);
-
- const filterableIdentifiers = new Set(
- filterableColumns.map((col) => {
- return Lib.columnKey(col);
- }),
- );
-
- const numericFilterableColumns = returnedColumns.filter((column) => {
- return (
- filterableIdentifiers.has(Lib.columnKey(column)) &&
- (Lib.isNumeric(column) || Lib.isTemporal(column))
- );
- });
-
- // Convert to select options with unique keys
- const seenKeys = new Set<string>();
- const uniqueColumns: Array<SelectOption> = [];
-
- numericFilterableColumns.forEach((column) => {
- const columnInfo = Lib.displayInfo(query, stageIndex, column);
- const uniqueKey = Lib.columnUniqueKey(column);
-
- if (!seenKeys.has(uniqueKey)) {
- seenKeys.add(uniqueKey);
-
- const label = columnInfo.longDisplayName;
-
- uniqueColumns.push({ value: uniqueKey, label });
- }
- });
-
- return uniqueColumns;
+ return getSourceFieldOptions(query);
} catch (error) {
- // If we can't extract columns (e.g., invalid query), return empty array
console.error(
"KeysetColumnSelect: Error extracting columns from query:",
error,
@@ -83,6 +98,12 @@ export function KeysetColumnSelect({
}
}, [query]);
+ useAutoSelectFirstOption({
+ name,
+ options: columnOptions,
+ disabled: disabled || isLoading,
+ });
+
return (
<FormSelect
name={name}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx
new file mode 100644
index 00000000000..46260aeda46
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/NativeQueryTableTagFieldSelect.tsx
@@ -0,0 +1,76 @@
+import { useMemo } from "react";
+import { t } from "ttag";
+
+import { FormSelect } from "metabase/forms";
+import {
+ Alert,
+ type DataAttributes,
+ Icon,
+ type InputDescriptionProps,
+ Loader,
+} from "metabase/ui";
+import type * as Lib from "metabase-lib";
+
+import { useAutoSelectFirstOption } from "../useAutoSelectFirstOption";
+import { useNativeCheckpointFieldOptions } from "../useNativeCheckpointFieldOptions";
+
+type NativeQueryTableTagFieldSelectProps = {
+ name: string;
+ label: string;
+ placeholder: string;
+ description: React.ReactNode;
+ descriptionProps?: InputDescriptionProps & DataAttributes;
+ query: Lib.Query;
+ disabled?: boolean;
+};
+
+export function NativeQueryTableTagFieldSelect({
+ name,
+ label,
+ placeholder,
+ description,
+ descriptionProps,
+ query,
+ disabled,
+}: NativeQueryTableTagFieldSelectProps) {
+ const { fieldOptions, isLoading, tableIds, hasError } =
+ useNativeCheckpointFieldOptions(query);
+
+ const noQueryMessage = useMemo(() => {
+ if (tableIds.length === 0) {
+ return t`Native queries must use at least one table variable to enable incremental transforms`;
+ }
+ if (hasError) {
+ return t`Unable to load table metadata. You may not have permission to access the table.`;
+ }
+ return null;
+ }, [tableIds, hasError]);
+
+ useAutoSelectFirstOption({
+ name,
+ options: fieldOptions,
+ disabled: disabled || isLoading,
+ });
+
+ if (noQueryMessage) {
+ return (
+ <Alert icon={<Icon name="warning" />} color="warning">
+ {noQueryMessage}
+ </Alert>
+ );
+ }
+
+ return (
+ <FormSelect
+ name={name}
+ label={label}
+ placeholder={placeholder}
+ description={description}
+ descriptionProps={descriptionProps}
+ data={fieldOptions}
+ searchable
+ disabled={disabled || fieldOptions.length === 0}
+ rightSection={isLoading ? <Loader size="xs" /> : undefined}
+ />
+ );
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
index e9c499b9ff8..0eaf25380fc 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
@@ -1,3 +1,5 @@
export { KeysetColumnSelect } from "./KeysetColumnSelect";
+export { getSourceFieldOptions } from "./KeysetColumnSelect";
export { MBQLKeysetColumnSelect } from "./MBQLKeysetColumnSelect";
+export { NativeQueryTableTagFieldSelect } from "./NativeQueryTableTagFieldSelect";
export { PythonKeysetColumnSelect } from "./PythonKeysetColumnSelect";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx
deleted file mode 100644
index d4cd9772ad4..00000000000
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/NativeQueryColumnSelect.tsx
+++ /dev/null
@@ -1,100 +0,0 @@
-import { useEffect, useState } from "react";
-import { useLatest } from "react-use";
-
-import { useExtractColumnsFromQueryMutation } from "metabase/api";
-import { FormSelect, FormTextInput } from "metabase/forms";
-import {
- type DataAttributes,
- type InputDescriptionProps,
- Loader,
-} from "metabase/ui";
-import * as Lib from "metabase-lib";
-
-type NativeQueryColumnSelectProps = {
- name: string;
- label: string;
- description: React.ReactNode;
- descriptionProps?: InputDescriptionProps & DataAttributes;
- placeholder: string;
- query: Lib.Query;
- disabled?: boolean;
-};
-
-export function NativeQueryColumnSelect({
- name,
- label,
- description,
- descriptionProps,
- placeholder,
- query,
- disabled,
-}: NativeQueryColumnSelectProps) {
- const [columns, setColumns] = useState<string[] | null>(null);
- const [extractColumns, { isLoading }] = useExtractColumnsFromQueryMutation();
-
- const queryRef = useLatest(query);
-
- useEffect(() => {
- let isCancelled = false;
- if (
- columns &&
- Lib.areLegacyQueriesEqual(
- Lib.toLegacyQuery(queryRef.current),
- Lib.toLegacyQuery(query),
- )
- ) {
- return;
- }
-
- const extract = async () => {
- try {
- const result = await extractColumns({
- query: Lib.toJsQuery(query),
- }).unwrap();
-
- if (!isCancelled) {
- // Only use columns if we got at least one
- setColumns(result.columns.length > 0 ? result.columns : null);
- }
- } catch (error) {
- // On error, fall back to text input
- if (!isCancelled) {
- setColumns(null);
- }
- }
- };
-
- extract();
-
- return () => {
- isCancelled = true;
- };
- }, [extractColumns, query, queryRef, columns]);
-
- // If we successfully extracted columns, show a selector
- if (columns && columns.length > 0) {
- return (
- <FormSelect
- name={name}
- rightSection={isLoading ? <Loader size="sm" /> : null}
- disabled={disabled || isLoading}
- label={label}
- description={description}
- placeholder={placeholder}
- data={columns.map((col) => ({ value: col, label: col }))}
- />
- );
- }
-
- // Otherwise, fall back to text input
- return (
- <FormTextInput
- name={name}
- label={label}
- placeholder={placeholder}
- description={description}
- descriptionProps={descriptionProps}
- disabled={disabled}
- />
- );
-}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts
deleted file mode 100644
index b95d8e9d607..00000000000
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/NativeQueryColumnSelect/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
index 0d63f80046a..a6e2c2a31cb 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.ts
@@ -1,5 +1,6 @@
import * as Yup from "yup";
+import * as Errors from "metabase/lib/errors";
import type {
Transform,
TransformSource,
@@ -10,18 +11,22 @@ import type {
export type IncrementalSettingsFormValues = {
incremental: boolean;
sourceStrategy: "checkpoint";
- checkpointFilter: string | null;
- checkpointFilterUniqueKey: string | null;
+ checkpointFilterFieldId: string | null; // String because Mantine Select requires string values
targetStrategy: "append";
};
export const VALIDATION_SCHEMA = Yup.object({
incremental: Yup.boolean().required(),
sourceStrategy: Yup.mixed<"checkpoint">().oneOf(["checkpoint"]).required(),
- // For native queries, use checkpointFilter (plain string)
- checkpointFilter: Yup.string().nullable().defined(),
- // For MBQL/Python queries, use checkpointFilterUniqueKey (prefixed format)
- checkpointFilterUniqueKey: Yup.string().nullable().defined(),
+ checkpointFilterFieldId: Yup.string()
+ .nullable()
+ .defined()
+ .when(["incremental", "sourceStrategy"], {
+ is: (incremental: boolean, sourceStrategy: "checkpoint") =>
+ incremental && sourceStrategy === "checkpoint",
+ then: (schema) => schema.required(Errors.required),
+ otherwise: (schema) => schema.nullable().defined(),
+ }),
targetStrategy: Yup.mixed<"append">().oneOf(["append"]).required(),
});
@@ -30,8 +35,7 @@ export const getInitialValues = (
): IncrementalSettingsFormValues => ({
incremental: false,
sourceStrategy: "checkpoint",
- checkpointFilter: null,
- checkpointFilterUniqueKey: null,
+ checkpointFilterFieldId: null,
targetStrategy: "append",
...defaults,
});
@@ -42,22 +46,18 @@ export const getIncrementalSettingsFromTransform = (
const isIncremental = transform.target.type === "table-incremental";
const strategy = transform.source["source-incremental-strategy"];
- // Read both fields from the strategy, whichever is present
- const checkpointFilter =
+ const checkpointFilterFieldId =
strategy?.type === "checkpoint"
- ? (strategy["checkpoint-filter"] ?? null)
- : null;
-
- const checkpointFilterUniqueKey =
- strategy?.type === "checkpoint"
- ? (strategy["checkpoint-filter-unique-key"] ?? null)
+ ? (strategy["checkpoint-filter-field-id"] ?? null)
: null;
return {
incremental: isIncremental,
sourceStrategy: "checkpoint",
- checkpointFilter: isIncremental ? checkpointFilter : null,
- checkpointFilterUniqueKey: isIncremental ? checkpointFilterUniqueKey : null,
+ checkpointFilterFieldId:
+ isIncremental && checkpointFilterFieldId != null
+ ? String(checkpointFilterFieldId)
+ : null,
targetStrategy: "append",
};
};
@@ -87,19 +87,17 @@ export const buildIncrementalSource = (
};
}
- // Build strategy fields based on which checkpoint field is present
- const strategyFields = formValues.checkpointFilter
- ? { "checkpoint-filter": formValues.checkpointFilter }
- : formValues.checkpointFilterUniqueKey
- ? { "checkpoint-filter-unique-key": formValues.checkpointFilterUniqueKey }
- : {};
-
return {
...source,
- "source-incremental-strategy": {
- type: formValues.sourceStrategy,
- ...strategyFields,
- },
+ "source-incremental-strategy":
+ formValues.checkpointFilterFieldId != null
+ ? {
+ type: formValues.sourceStrategy,
+ "checkpoint-filter-field-id": Number(
+ formValues.checkpointFilterFieldId,
+ ),
+ }
+ : undefined,
};
};
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
new file mode 100644
index 00000000000..27d91e8eccf
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/form.unit.spec.ts
@@ -0,0 +1,36 @@
+import { type IncrementalSettingsFormValues, VALIDATION_SCHEMA } from "./form";
+
+const getValues = (
+ values: Partial<IncrementalSettingsFormValues> = {},
+): IncrementalSettingsFormValues => ({
+ incremental: true,
+ sourceStrategy: "checkpoint",
+ checkpointFilterFieldId: "123",
+ targetStrategy: "append",
+ ...values,
+});
+
+describe("IncrementalTransform form validation", () => {
+ it("requires checkpoint field for incremental transforms", async () => {
+ const values = getValues({ checkpointFilterFieldId: null });
+
+ await expect(VALIDATION_SCHEMA.validate(values)).rejects.toMatchObject({
+ message: "required",
+ });
+ });
+
+ it("accepts checkpoint field when incremental transforms are enabled", async () => {
+ const values = getValues({ checkpointFilterFieldId: "42" });
+
+ await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);
+ });
+
+ it("does not require checkpoint field when incremental transforms are disabled", async () => {
+ const values = getValues({
+ incremental: false,
+ checkpointFilterFieldId: null,
+ });
+
+ await expect(VALIDATION_SCHEMA.validate(values)).resolves.toEqual(values);
+ });
+});
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts
index 1f51a5b5a9c..aecd1a27dfe 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/index.ts
@@ -1,5 +1,4 @@
-export * from "./hooks";
export * from "./form";
export * from "./IncrementalTransformSettings";
export * from "./KeysetColumnSelect";
-export * from "./NativeQueryColumnSelect";
+export { useUpdateIncrementalSettings } from "./useUpdateIncrementalSettings";
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.ts
new file mode 100644
index 00000000000..d4cb2ee6e41
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useAutoSelectFirstOption.ts
@@ -0,0 +1,36 @@
+import { useFormikContext } from "formik";
+import { useEffect } from "react";
+
+import type { SelectOption } from "metabase/ui";
+
+type UseAutoSelectFirstOptionProps = {
+ name: string;
+ options: SelectOption[];
+ disabled?: boolean;
+};
+
+export function useAutoSelectFirstOption({
+ name,
+ options,
+ disabled,
+}: UseAutoSelectFirstOptionProps) {
+ const { setFieldValue, values } =
+ useFormikContext<Record<string, string | null>>();
+ const currentValue = values[name];
+
+ useEffect(() => {
+ if (disabled || options.length === 0) {
+ return;
+ }
+
+ const hasValidCurrentValue =
+ currentValue != null &&
+ options.some((option) => option.value === currentValue);
+
+ if (hasValidCurrentValue) {
+ return;
+ }
+
+ setFieldValue(name, options[0].value);
+ }, [currentValue, disabled, name, options, setFieldValue]);
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts
new file mode 100644
index 00000000000..4a784950e54
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions.ts
@@ -0,0 +1,88 @@
+import { match } from "ts-pattern";
+
+import {
+ skipToken,
+ useGetAdhocQueryMetadataQuery,
+ useGetTableQueryMetadataQuery,
+} from "metabase/api";
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
+import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";
+import * as Lib from "metabase-lib";
+import type { TransformSource } from "metabase-types/api";
+import { isConcreteTableId } from "metabase-types/api/table";
+
+import { getSourceFieldOptions } from "./KeysetColumnSelect";
+import { useNativeHasCheckpointFieldOptions } from "./useNativeCheckpointFieldOptions";
+
+export const useHasCheckpointOptions = (source: TransformSource) => {
+ const metadata = useSelector(getMetadata);
+ const libQuery = getLibQuery(source, metadata);
+
+ // Trigger query metadata fetch for MBQL/native query sources so metadata is populated
+ // before we compute hasCheckpointOptions. Without this, getSourceFieldOptions(libQuery)
+ // can return [] when the query references a card or needs schema from the API.
+ useGetAdhocQueryMetadataQuery(
+ source.type === "query" ? source.query : skipToken,
+ );
+
+ const isPythonTransform = source.type === "python";
+ const transformType = match({
+ isMbqlQuery: isMbqlQuery(source, metadata),
+ isPythonTransform,
+ })
+ .with({ isMbqlQuery: true }, () => "mbql" as const)
+ .with({ isPythonTransform: true }, () => "python" as const)
+ .otherwise(() => "native" as const);
+
+ const hasNativeCheckpointOptions =
+ useNativeHasCheckpointFieldOptions(libQuery);
+
+ const pythonTableId = isPythonTransform
+ ? Object.values(source["source-tables"]).find(isConcreteTableId)
+ : null;
+ const { data: pythonTable } = useGetTableQueryMetadataQuery(
+ pythonTableId ? { id: pythonTableId } : skipToken,
+ );
+
+ const getHasCheckpointOptions = () => {
+ try {
+ return match(transformType)
+ .with("mbql", () =>
+ libQuery ? getSourceFieldOptions(libQuery).length > 0 : false,
+ )
+ .with("python", () => {
+ if (!pythonTable?.db_id) {
+ return false;
+ }
+
+ const metadataProvider = Lib.metadataProvider(
+ pythonTable.db_id,
+ metadata,
+ );
+ const tableMetadata = Lib.tableOrCardMetadata(
+ metadataProvider,
+ pythonTable.id,
+ );
+ if (!tableMetadata) {
+ return false;
+ }
+
+ const tableQuery = Lib.queryFromTableOrCardMetadata(
+ metadataProvider,
+ tableMetadata,
+ );
+ return getSourceFieldOptions(tableQuery).length > 0;
+ })
+ .with("native", () => hasNativeCheckpointOptions)
+ .exhaustive();
+ } catch {
+ return false;
+ }
+ };
+
+ return {
+ hasCheckpointOptions: getHasCheckpointOptions(),
+ transformType,
+ };
+};
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts
new file mode 100644
index 00000000000..a23834b79ce
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useNativeCheckpointFieldOptions.ts
@@ -0,0 +1,90 @@
+import { useMemo } from "react";
+
+import { useSelector } from "metabase/lib/redux";
+import { getMetadata } from "metabase/selectors/metadata";
+import type { SelectOption } from "metabase/ui";
+import * as Lib from "metabase-lib";
+import { isConcreteTableId } from "metabase-types/api/table";
+
+import { getSourceFieldOptions } from "./KeysetColumnSelect/KeysetColumnSelect";
+import { useTableQueryMetadataResults } from "./useTableQueryMetadataResults";
+
+export function useNativeCheckpointFieldOptions(query: Lib.Query | null) {
+ const metadata = useSelector(getMetadata);
+
+ const tableIds = useMemo(() => {
+ if (!query) {
+ return [];
+ }
+ try {
+ const templateTags = Lib.templateTags(query);
+ const tableTags = Object.values(templateTags).filter(
+ (tag) => tag.type === "table" && tag["table-id"] != null,
+ );
+ return tableTags.map((tag) => tag["table-id"]).filter(isConcreteTableId);
+ } catch {
+ return [];
+ }
+ }, [query]);
+
+ const { tables, isLoading, hasError } =
+ useTableQueryMetadataResults(tableIds);
+
+ const fieldOptions = useMemo((): Array<SelectOption> => {
+ if (tables.length === 0) {
+ return [];
+ }
+
+ try {
+ const allOptions: Array<SelectOption> = [];
+ const seenFieldIds = new Set<number>();
+ const showTablePrefix = tables.length > 1;
+
+ for (const table of tables) {
+ const metadataProvider = Lib.metadataProvider(table.db_id, metadata);
+ const tableMetadata = Lib.tableOrCardMetadata(
+ metadataProvider,
+ table.id,
+ );
+ if (!tableMetadata) {
+ continue;
+ }
+
+ const tableQuery = Lib.queryFromTableOrCardMetadata(
+ metadataProvider,
+ tableMetadata,
+ );
+
+ const options = getSourceFieldOptions(tableQuery, {
+ labelPrefix: showTablePrefix
+ ? table.display_name || table.name
+ : undefined,
+ seenFieldIds,
+ });
+
+ allOptions.push(...options);
+ }
+
+ return allOptions;
+ } catch (error) {
+ console.error(
+ "NativeQueryTableTagFieldSelect: Error extracting fields:",
+ error,
+ );
+ return [];
+ }
+ }, [tables, metadata]);
+
+ return {
+ fieldOptions,
+ isLoading,
+ tableIds,
+ hasError,
+ };
+}
+
+export function useNativeHasCheckpointFieldOptions(query: Lib.Query | null) {
+ const { fieldOptions } = useNativeCheckpointFieldOptions(query);
+
+ return fieldOptions.length > 0;
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.ts
new file mode 100644
index 00000000000..02222efe574
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useTableQueryMetadataResults.ts
@@ -0,0 +1,40 @@
+import { useEffect, useMemo } from "react";
+
+import { tableApi } from "metabase/api";
+import { useDispatch, useSelector } from "metabase/lib/redux";
+import type { Table } from "metabase-types/api";
+
+const selectTableQueryMetadata =
+ tableApi.endpoints.getTableQueryMetadata.select;
+
+/**
+ * Fetches table query metadata for a dynamic list of table IDs.
+ * Uses RTK Query's initiate/select pattern to avoid the "hooks in a loop" problem.
+ */
+export function useTableQueryMetadataResults(tableIds: number[]) {
+ const dispatch = useDispatch();
+
+ useEffect(() => {
+ const subscriptions = tableIds.map((id) =>
+ dispatch(tableApi.endpoints.getTableQueryMetadata.initiate({ id })),
+ );
+ return () => {
+ subscriptions.forEach((sub) => sub.unsubscribe());
+ };
+ }, [dispatch, tableIds]);
+
+ const selectors = useMemo(
+ () => tableIds.map((id) => selectTableQueryMetadata({ id })),
+ [tableIds],
+ );
+
+ return useSelector((state) => {
+ const results = selectors.map((sel) => sel(state));
+ const isLoading = results.some((r) => r.isLoading);
+ const hasError = results.some((r) => r.isError);
+ const tables = results
+ .map((r) => r.data)
+ .filter((t): t is Table => t != null);
+ return { isLoading, hasError, tables };
+ });
+}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/hooks.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
similarity index 91%
rename from frontend/src/metabase/transforms/components/IncrementalTransform/hooks.ts
rename to frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
index 3c8ee1f1c3a..38fdb3a59ac 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/hooks.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/useUpdateIncrementalSettings.ts
@@ -10,7 +10,7 @@ import {
getIncrementalSettingsFromTransform,
} from "./form";
-export const useUpdateIncrementalSettings = (transform: Transform) => {
+export function useUpdateIncrementalSettings(transform: Transform) {
const [updateTransform] = useUpdateTransformMutation();
const initialValues = useMemo(
() => getIncrementalSettingsFromTransform(transform),
@@ -29,4 +29,4 @@ export const useUpdateIncrementalSettings = (transform: Transform) => {
validationSchema: VALIDATION_SCHEMA,
updateIncrementalSettings,
};
-};
+}
diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css b/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css
deleted file mode 100644
index 86f3a93b537..00000000000
--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.module.css
+++ /dev/null
@@ -1,6 +0,0 @@
-.bordered {
- background: var(--mb-color-background-secondary);
- border-radius: var(--mantine-radius-md);
- padding: var(--mantine-spacing-md);
- border: 1px solid var(--mb-color-border);
-}
diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx b/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx
deleted file mode 100644
index d99543e2305..00000000000
--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/QueryComplexityWarning.tsx
+++ /dev/null
@@ -1,39 +0,0 @@
-import cx from "classnames";
-import { t } from "ttag";
-
-import { CHECKPOINT_TEMPLATE_TAG } from "metabase/transforms/constants";
-import { Stack, Text } from "metabase/ui";
-
-import S from "./QueryComplexityWarning.module.css";
-
-type QueryComplexityWarningProps = {
- variant?: "default" | "standout";
-};
-
-export const QueryComplexityWarning = ({
- variant = "default",
-}: QueryComplexityWarningProps) => {
- return (
- <Stack
- gap={variant === "standout" ? "sm" : "md"}
- className={cx({ [S.bordered]: variant === "standout" })}
- data-testid="query-complexity-warning"
- >
- {variant === "standout" && (
- <Text fw="bold">{t`We couldn’t add the check automatically.`}</Text>
- )}
- <Text lh="1.25rem">{t`This query is too complicated for us to automatically add a checkpoint. You can turn off incremental transformation, or add an explicit filter to the query, like this:`}</Text>
- <Text
- lh="1.25rem"
- py="md"
- className={cx({ [S.bordered]: variant === "default" })}
- >
- <code>{`[[ WHERE id > {{${CHECKPOINT_TEMPLATE_TAG}}} ]]`}</code>
- </Text>
- <Text
- lh="1.25rem"
- c="saturated-red"
- >{t`If you save anyway, this transform will likely not process new data, or might only process partial quantities of new data.`}</Text>
- </Stack>
- );
-};
diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.ts b/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.ts
deleted file mode 100644
index 638c5b5972a..00000000000
--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/index.ts
+++ /dev/null
@@ -1,2 +0,0 @@
-export { QueryComplexityWarning } from "./QueryComplexityWarning";
-export { useQueryComplexityChecks } from "./use-query-complexity-checks";
diff --git a/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx b/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx
deleted file mode 100644
index f7f5142ce5e..00000000000
--- a/frontend/src/metabase/transforms/components/QueryComplexityWarning/use-query-complexity-checks.tsx
+++ /dev/null
@@ -1,78 +0,0 @@
-import { useCallback } from "react";
-import { t } from "ttag";
-
-import { useLazyCheckQueryComplexityQuery } from "metabase/api";
-import { useConfirmation } from "metabase/common/hooks";
-import { useSelector } from "metabase/lib/redux";
-import { getMetadata } from "metabase/selectors/metadata";
-import { CHECKPOINT_TEMPLATE_TAG } from "metabase/transforms/constants";
-import { getLibQuery, isMbqlQuery } from "metabase/transforms/utils";
-import * as Lib from "metabase-lib";
-import type Metadata from "metabase-lib/v1/metadata/Metadata";
-import type { DraftTransformSource } from "metabase-types/api";
-
-import { QueryComplexityWarning } from "./QueryComplexityWarning";
-
-const getQueryForComplexityCheck = (
- source: DraftTransformSource,
- metadata: Metadata,
-): Lib.Query | undefined => {
- const query = getLibQuery(source, metadata);
- if (!query || isMbqlQuery(source, metadata)) {
- return;
- }
-
- const tags = Lib.templateTags(query);
- if (CHECKPOINT_TEMPLATE_TAG in tags) {
- return;
- }
-
- return query;
-};
-
-export const useQueryComplexityChecks = () => {
- const [checkQueryComplexity] = useLazyCheckQueryComplexityQuery();
- const metadata = useSelector(getMetadata);
- const { modalContent: modal, show } = useConfirmation();
-
- const checkComplexity = useCallback(
- async (source: DraftTransformSource) => {
- const query = getQueryForComplexityCheck(source, metadata);
- if (!query) {
- return;
- }
- const rawQuery = Lib.rawNativeQuery(query);
- const complexity = await checkQueryComplexity(rawQuery, true).unwrap();
- if (complexity.is_simple) {
- return;
- }
- return complexity;
- },
- [metadata, checkQueryComplexity],
- );
-
- const confirmIfQueryIsComplex = useCallback(
- async (
- source: DraftTransformSource,
- confirmButtonText: string = t`Save anyway`,
- ) => {
- const complexity = await checkComplexity(source);
- if (!complexity) {
- return true;
- }
- return new Promise<boolean>((resolve) => {
- show({
- title: t`Can't automatically run this transform incrementally`,
- message: <QueryComplexityWarning />,
- onConfirm: () => resolve(true),
- onCancel: () => resolve(false),
- confirmButtonText,
- confirmButtonProps: { color: "saturated-red", variant: "filled" },
- });
- });
- },
- [checkComplexity, show],
- );
-
- return { confirmIfQueryIsComplex, checkComplexity, modal };
-};
diff --git a/frontend/src/metabase/transforms/constants.ts b/frontend/src/metabase/transforms/constants.ts
index 18107133265..d3604777372 100644
--- a/frontend/src/metabase/transforms/constants.ts
+++ b/frontend/src/metabase/transforms/constants.ts
@@ -9,10 +9,9 @@ export const POLLING_INTERVAL = isCypressActive ? 200 : 3000;
export const FILTER_WIDGET_MIN_WIDTH = 300;
export const FILTER_WIDGET_MAX_HEIGHT = 400;
-export const CHECKPOINT_TEMPLATE_TAG = "checkpoint";
export const SOURCE_STRATEGY_OPTIONS = [
{
- value: CHECKPOINT_TEMPLATE_TAG,
+ value: "checkpoint",
get label() {
return t`Checkpoint`;
},
diff --git a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
index 7dac5342e17..e035d7601c4 100644
--- a/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
+++ b/frontend/src/metabase/transforms/hooks/use-transform-with-polling.ts
@@ -1,4 +1,4 @@
-import { useState } from "react";
+import { useEffect, useRef, useState } from "react";
import { skipToken, useGetTransformQuery } from "metabase/api";
import type { Transform } from "metabase-types/api";
@@ -18,11 +18,13 @@ const isPollingNeeded = (transform?: Transform) =>
export const useTransformWithPolling = (transformId: number | undefined) => {
const [isPolling, setIsPolling] = useState(false);
+ const wasPolling = useRef(false);
const {
data: transform,
isLoading,
error,
+ refetch,
} = useGetTransformQuery(transformId ?? skipToken, {
pollingInterval: isPolling ? POLLING_INTERVAL : undefined,
});
@@ -31,5 +33,15 @@ export const useTransformWithPolling = (transformId: number | undefined) => {
setIsPolling(isPollingNeeded(transform));
}
+ // When polling stops (run just finished), do a delayed refetch to pick up
+ // post-execution changes like the updated checkpoint watermark.
+ useEffect(() => {
+ if (wasPolling.current && !isPolling) {
+ const timer = setTimeout(() => refetch(), POLLING_INTERVAL);
+ return () => clearTimeout(timer);
+ }
+ wasPolling.current = isPolling;
+ }, [isPolling, refetch]);
+
return { transform, isLoading, error };
};
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
index 567430118d6..c3cc44322aa 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/CreateTransformModal.tsx
@@ -1,5 +1,5 @@
import { useFormikContext } from "formik";
-import { useMemo, useState } from "react";
+import { useMemo } from "react";
import { t } from "ttag";
import _ from "underscore";
import type * as Yup from "yup";
@@ -20,13 +20,8 @@ import {
FormTextInput,
} from "metabase/forms";
import { IncrementalTransformSettings } from "metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings";
-import {
- QueryComplexityWarning,
- useQueryComplexityChecks,
-} from "metabase/transforms/components/QueryComplexityWarning";
import { Box, Button, Group, Modal, Stack } from "metabase/ui";
import type {
- QueryComplexity,
SchemaName,
Transform,
TransformSource,
@@ -158,17 +153,9 @@ function CreateTransformForm({
showIncrementalSettings = true,
}: CreateTransformFormFieldsProps) {
const { values, setFieldValue } = useFormikContext<NewTransformValues>();
- const { checkComplexity } = useQueryComplexityChecks();
- const [complexity, setComplexity] = useState<QueryComplexity | undefined>();
- const handleIncrementalChange = async (value: boolean) => {
+ const handleIncrementalChange = (value: boolean) => {
setFieldValue("incremental", value);
- if (value) {
- const complexity = await checkComplexity(source);
- setComplexity(complexity);
- } else {
- setComplexity(undefined);
- }
};
return (
@@ -201,17 +188,12 @@ function CreateTransformForm({
onIncrementalChange={handleIncrementalChange}
/>
)}
- {complexity && <QueryComplexityWarning variant="standout" />}
<Group>
<Box flex={1}>
<FormErrorMessage />
</Box>
<Button onClick={onClose}>{t`Back`}</Button>
- <FormSubmitButton
- label={complexity ? t`Save anyway` : t`Save`}
- variant="filled"
- color={complexity ? "saturated-red" : undefined}
- />
+ <FormSubmitButton label={t`Save`} variant="filled" />
</Group>
</Stack>
</Form>
diff --git a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts
index 1bbde70ef0b..8bb4fa16d58 100644
--- a/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts
+++ b/frontend/src/metabase/transforms/pages/NewTransformPage/CreateTransformModal/hooks.ts
@@ -32,7 +32,10 @@ export const useCreateTransform = (
databaseId,
);
const transform = await createTransformMutation(request).unwrap();
- trackTransformCreated({ transformId: transform.id });
+ trackTransformCreated({
+ transformId: transform.id,
+ isIncremental: transform.target.type === "table-incremental",
+ });
return transform;
};
diff --git a/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx b/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx
index 47c09a2e3f9..7d060d032c2 100644
--- a/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx
+++ b/frontend/src/metabase/transforms/pages/RunListPage/RunSidebar/InfoSection/InfoSection.tsx
@@ -1,11 +1,13 @@
-import type { ReactNode } from "react";
+import { type ReactNode, useId } from "react";
import { t } from "ttag";
+import { skipToken, useGetFieldQuery } from "metabase/api";
import { DateTime } from "metabase/common/components/DateTime";
import CS from "metabase/css/core/index.css";
import { Box, Card, Stack } from "metabase/ui";
import type { TransformRun } from "metabase-types/api";
+import { CheckpointValue } from "../../../../components/CheckpointValue";
import {
formatRunMethod,
formatStatus,
@@ -19,6 +21,12 @@ type InfoSectionProps = {
};
export function InfoSection({ run }: InfoSectionProps) {
+ const { data: checkpointField } = useGetFieldQuery(
+ run.checkpoint_filter_field_id
+ ? { id: run.checkpoint_filter_field_id }
+ : skipToken,
+ );
+
return (
<Card p={0} shadow="none" withBorder role="region" aria-label={t`Info`}>
<InfoSectionItem label={t`Started at`}>
@@ -37,6 +45,27 @@ export function InfoSection({ run }: InfoSectionProps) {
<InfoSectionItem label={t`Trigger`}>
{formatRunMethod(run.run_method)}
</InfoSectionItem>
+ {checkpointField != null && (
+ <InfoSectionItem label={t`Checkpoint field`}>
+ {checkpointField.display_name}
+ </InfoSectionItem>
+ )}
+ {run.checkpoint_lo_value != null && (
+ <InfoSectionItem label={t`Checkpoint from`}>
+ <CheckpointValue
+ value={run.checkpoint_lo_value}
+ checkpointField={checkpointField}
+ />
+ </InfoSectionItem>
+ )}
+ {run.checkpoint_hi_value != null && (
+ <InfoSectionItem label={t`Checkpoint to`}>
+ <CheckpointValue
+ value={run.checkpoint_hi_value}
+ checkpointField={checkpointField}
+ />
+ </InfoSectionItem>
+ )}
</Card>
);
}
@@ -47,9 +76,23 @@ type InfoSectionItemProps = {
};
function InfoSectionItem({ label, children }: InfoSectionItemProps) {
+ const labelId = useId();
+
return (
- <Stack className={S.section} p="md" gap="xs">
- <Box className={CS.textWrap} c="text-secondary" fz="sm" lh="h5">
+ <Stack
+ className={S.section}
+ p="md"
+ gap="xs"
+ role="group"
+ aria-labelledby={labelId}
+ >
+ <Box
+ id={labelId}
+ className={CS.textWrap}
+ c="text-secondary"
+ fz="sm"
+ lh="h5"
+ >
{label}
</Box>
<Box lh="h4">{children}</Box>
diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx b/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx
index bec3eba95da..87c92543a59 100644
--- a/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformListPage/TransformListPage.tsx
@@ -1,6 +1,5 @@
import type { Row } from "@tanstack/react-table";
import {
- type ComponentProps,
type PropsWithChildren,
useCallback,
useEffect,
@@ -31,6 +30,7 @@ import { useSelector } from "metabase/lib/redux";
import * as Urls from "metabase/lib/urls";
import { type NamedUser, getUserName } from "metabase/lib/user";
import { PLUGIN_REMOTE_SYNC, PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";
+import { getMetadata } from "metabase/selectors/metadata";
import { CreateTransformMenu } from "metabase/transforms/components/CreateTransformMenu";
import { ListEmptyState } from "metabase/transforms/components/ListEmptyState";
import { useTransformPermissions } from "metabase/transforms/hooks/use-transform-permissions";
@@ -52,7 +52,11 @@ import {
import { CollectionRowMenu } from "./CollectionRowMenu";
import S from "./TransformListPage.module.css";
import { type TreeNode, getCollectionNodeId, isCollectionNode } from "./types";
-import { buildTreeData, getDefaultExpandedIds } from "./utils";
+import {
+ buildTreeData,
+ getDefaultExpandedIds,
+ getIncrementalWarning,
+} from "./utils";
const getNodeId = (node: TreeNode) => node.id;
const getSubRows = (node: TreeNode) => node.children;
@@ -137,10 +141,22 @@ export const TransformListPage = ({
const isLoading =
isLoadingCollections || isLoadingTransforms || isLoadingDatabases;
const error = collectionsError ?? transformsError;
+ const metadata = useSelector(getMetadata);
const shouldShowPythonTransformsUpsell = useSelector(
getShouldShowPythonTransformsUpsell,
);
+ const warningsByTransformId = useMemo(() => {
+ const warnings = new Map<number, string>();
+ for (const transform of transforms ?? []) {
+ const warning = getIncrementalWarning(transform, metadata);
+ if (warning) {
+ warnings.set(transform.id, warning);
+ }
+ }
+ return warnings;
+ }, [transforms, metadata]);
+
const treeData = useMemo(() => {
const data = buildTreeData(collections, transforms);
// Only show Python library item if there's at least one item in the table
@@ -176,16 +192,6 @@ export const TransformListPage = ({
);
const columnDefs = useMemo<TreeTableColumnDef<TreeNode>[]>(() => {
- type EllipsifiedProps = ComponentProps<
- typeof EntityNameCell
- >["ellipsifiedProps"];
- const unreadableTransformEllipsifiedProps: EllipsifiedProps = {
- alwaysShowTooltip: true,
- tooltipProps: {
- openDelay: 300,
- label: t`Sorry, you don’t have permission to see that.`,
- },
- };
return [
{
id: "name",
@@ -194,26 +200,12 @@ export const TransformListPage = ({
minWidth: 280,
maxAutoWidth: 800,
enableSorting: true,
- cell: ({ row }) => {
- const isLibraryWithoutFeature =
- row.original.nodeType === "library" && !hasPythonTransformsFeature;
- return (
- <Group gap="sm" wrap="nowrap" miw={0}>
- <EntityNameCell
- data-testid="tree-node-name"
- icon={row.original.icon}
- iconColor={getNodeIconColor(row.original)}
- name={row.original.name}
- ellipsifiedProps={
- isRowDisabled(row)
- ? unreadableTransformEllipsifiedProps
- : undefined
- }
- />
- {isLibraryWithoutFeature && <UpsellGem.New size={14} />}
- </Group>
- );
- },
+ cell: ({ row }) =>
+ getNameCell({
+ row,
+ hasPythonTransformsFeature,
+ warningsByTransformId,
+ }),
},
{
id: "owner",
@@ -287,7 +279,7 @@ export const TransformListPage = ({
) : null,
},
];
- }, [hasPythonTransformsFeature]);
+ }, [hasPythonTransformsFeature, warningsByTransformId]);
const getRowHref = useCallback((row: Row<TreeNode>) => {
if (isRowDisabled(row)) {
@@ -388,3 +380,56 @@ export const TransformListPage = ({
</PageContainer>
);
};
+
+function getNameCell({
+ row,
+ hasPythonTransformsFeature,
+ warningsByTransformId,
+}: {
+ row: Row<TreeNode>;
+ hasPythonTransformsFeature: boolean;
+ warningsByTransformId: Map<number, string>;
+}) {
+ const getTooltipProps = (message: string | undefined) => {
+ if (!message) {
+ return undefined;
+ }
+
+ return {
+ alwaysShowTooltip: true,
+ tooltipProps: {
+ openDelay: 300,
+ label: message,
+ },
+ };
+ };
+
+ const getWarningMessage = () => {
+ if (isRowDisabled(row)) {
+ return t`Sorry, you don’t have permission to see that.`;
+ }
+
+ if (row.original.transformId) {
+ return warningsByTransformId.get(row.original.transformId);
+ }
+ return undefined;
+ };
+
+ const isLibraryWithoutFeature =
+ row.original.nodeType === "library" && !hasPythonTransformsFeature;
+
+ const hasWarning = !!getWarningMessage();
+
+ return (
+ <Group gap="sm" wrap="nowrap" miw={0}>
+ <EntityNameCell
+ data-testid="tree-node-name"
+ icon={hasWarning ? "warning" : row.original.icon}
+ iconColor={hasWarning ? "warning" : getNodeIconColor(row.original)}
+ name={row.original.name}
+ ellipsifiedProps={{ ...getTooltipProps(getWarningMessage()) }}
+ />
+ {isLibraryWithoutFeature && <UpsellGem.New size={14} />}
+ </Group>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/types.ts b/frontend/src/metabase/transforms/pages/TransformListPage/types.ts
index 1c9851f4496..8748cfe2b08 100644
--- a/frontend/src/metabase/transforms/pages/TransformListPage/types.ts
+++ b/frontend/src/metabase/transforms/pages/TransformListPage/types.ts
@@ -17,6 +17,7 @@ export type TreeNode = {
collectionId?: number;
url?: string;
source_readable?: boolean;
+ warningTooltip?: string;
};
export function isCollectionNode(
diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts b/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts
index 69cd7a40776..92499ff1308 100644
--- a/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts
+++ b/frontend/src/metabase/transforms/pages/TransformListPage/utils.ts
@@ -1,4 +1,9 @@
+import { t } from "ttag";
+
import { getCollectionIcon } from "metabase/entities/collections/utils";
+import { getLibQuery } from "metabase/transforms/utils";
+import * as Lib from "metabase-lib";
+import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type { Collection, CollectionId, Transform } from "metabase-types/api";
import {
@@ -7,6 +12,40 @@ import {
getTransformNodeId,
} from "./types";
+export function getIncrementalWarning(
+ transform: Transform,
+ metadata: Metadata,
+): string | undefined {
+ const isIncremental = transform.target.type === "table-incremental";
+ if (!isIncremental) {
+ return undefined;
+ }
+
+ const isNative = transform.source_type === "native";
+ if (isNative) {
+ const libQuery = getLibQuery(transform.source, metadata);
+ const hasTableTag = libQuery
+ ? Object.values(Lib.templateTags(libQuery)).some(
+ (tag) => tag.type === "table" && tag["table-id"] != null,
+ )
+ : false;
+
+ if (!hasTableTag) {
+ return t`Incremental transform with a native query requires a table variable. Please add a table variable to the query and update the checkpoint field.`;
+ }
+ }
+
+ const checkpointFieldId =
+ transform.source["source-incremental-strategy"]?.[
+ "checkpoint-filter-field-id"
+ ];
+ if (!checkpointFieldId) {
+ return t`Incremental transform is enabled but no checkpoint field is selected. Please select a checkpoint field in the transform settings.`;
+ }
+
+ return undefined;
+}
+
export function buildTreeData(
collections: Collection[] | undefined,
transforms: Transform[] | undefined,
diff --git a/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts b/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts
index ebbbe241f4d..6773ebd78b3 100644
--- a/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts
+++ b/frontend/src/metabase/transforms/pages/TransformListPage/utils.unit.spec.ts
@@ -1,10 +1,32 @@
-import type { TransformOwner } from "metabase-types/api";
+import { createMockMetadata } from "__support__/metadata";
+import { getLibQuery } from "metabase/transforms/utils";
+import * as Lib from "metabase-lib";
+import { DEFAULT_TEST_QUERY, SAMPLE_PROVIDER } from "metabase-lib/test-helpers";
import {
+ createMockStructuredDatasetQuery,
+ createMockTemplateTag,
createMockTransform,
createMockTransformOwner,
+ createMockTransformTarget,
} from "metabase-types/api/mocks";
-import { buildTreeData } from "./utils";
+import { buildTreeData, getIncrementalWarning } from "./utils";
+
+jest.mock("metabase/transforms/utils", () => ({
+ getLibQuery: jest.fn(),
+}));
+
+jest.mock("metabase-lib", () => ({
+ ...jest.requireActual<typeof Lib>("metabase-lib"),
+ templateTags: jest.fn(),
+}));
+
+const query = Lib.createTestQuery(SAMPLE_PROVIDER, DEFAULT_TEST_QUERY);
+
+const mockGetLibQuery = getLibQuery as jest.MockedFunction<typeof getLibQuery>;
+const mockTemplateTags = Lib.templateTags as jest.MockedFunction<
+ typeof Lib.templateTags
+>;
describe("buildTreeData", () => {
it("should return empty array when no collections or transforms", () => {
@@ -38,13 +60,13 @@ describe("buildTreeData", () => {
id: 1,
name: "Test Transform",
owner_email: "[email protected]",
- owner: { email: "[email protected]" } as TransformOwner,
+ owner: createMockTransformOwner({ email: "[email protected]" }),
});
const result = buildTreeData([], [transform]);
expect(result).toHaveLength(1);
- expect(result[0].owner).toEqual({ email: "[email protected]" });
+ expect(result[0].owner).toMatchObject({ email: "[email protected]" });
expect(result[0].owner_email).toBe("[email protected]");
});
@@ -82,7 +104,7 @@ describe("buildTreeData", () => {
id: 2,
name: "Transform with email owner",
owner_email: "[email protected]",
- owner: { email: "[email protected]" } as TransformOwner,
+ owner: createMockTransformOwner({ email: "[email protected]" }),
}),
createMockTransform({
id: 3,
@@ -95,8 +117,129 @@ describe("buildTreeData", () => {
expect(result).toHaveLength(3);
expect(result[0].owner).toEqual(userOwner);
- expect(result[1].owner).toEqual({ email: "[email protected]" });
+ expect(result[1].owner).toMatchObject({ email: "[email protected]" });
expect(result[1].owner_email).toBe("[email protected]");
expect(result[2].owner).toBeUndefined();
});
});
+
+describe("getIncrementalWarning", () => {
+ const metadata = createMockMetadata({});
+
+ it("should return undefined for non-incremental transform", () => {
+ const transform = createMockTransform({
+ target: createMockTransformTarget({ type: "table" }),
+ });
+
+ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();
+ });
+
+ it("should warn when native query has no table variables", () => {
+ mockGetLibQuery.mockReturnValue(query);
+ mockTemplateTags.mockReturnValue({
+ snippet1: createMockTemplateTag({
+ id: "1",
+ name: "snippet1",
+ type: "snippet",
+ }),
+ });
+
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ const result = getIncrementalWarning(transform, metadata);
+ expect(result).toMatch(/table variable/);
+ });
+
+ it("should warn when native query has no lib query (getLibQuery returns null)", () => {
+ mockGetLibQuery.mockReturnValue(null);
+
+ const transform = createMockTransform({
+ source_type: "native",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ const result = getIncrementalWarning(transform, metadata);
+ expect(result).toMatch(/table variable/);
+ });
+
+ it("should warn when incremental transform has no checkpoint field", () => {
+ mockGetLibQuery.mockReturnValue(query);
+ mockTemplateTags.mockReturnValue({
+ my_table: createMockTemplateTag({
+ id: "1",
+ name: "my_table",
+ type: "table",
+ "table-id": 42,
+ }),
+ });
+
+ const transform = createMockTransform({
+ source_type: "native",
+ source: {
+ type: "query",
+ query: createMockStructuredDatasetQuery(),
+ },
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ const result = getIncrementalWarning(transform, metadata);
+ expect(result).toMatch(/checkpoint field/);
+ });
+
+ it("should return undefined for properly configured native incremental transform", () => {
+ mockGetLibQuery.mockReturnValue(query);
+ mockTemplateTags.mockReturnValue({
+ my_table: createMockTemplateTag({
+ id: "1",
+ name: "my_table",
+ type: "table",
+ "table-id": 42,
+ }),
+ });
+
+ const transform = createMockTransform({
+ source_type: "native",
+ source: {
+ type: "query",
+ query: createMockStructuredDatasetQuery(),
+ "source-incremental-strategy": {
+ type: "checkpoint",
+ "checkpoint-filter-field-id": 99,
+ },
+ },
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();
+ });
+
+ it("should warn when mbql incremental transform has no checkpoint field", () => {
+ const transform = createMockTransform({
+ source_type: "mbql",
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ const result = getIncrementalWarning(transform, metadata);
+ expect(result).toMatch(/checkpoint field/);
+ });
+
+ it("should return undefined for properly configured mbql incremental transform", () => {
+ const transform = createMockTransform({
+ source_type: "mbql",
+ source: {
+ type: "query",
+ query: createMockStructuredDatasetQuery(),
+ "source-incremental-strategy": {
+ type: "checkpoint",
+ "checkpoint-filter-field-id": 99,
+ },
+ },
+ target: createMockTransformTarget({ type: "table-incremental" }),
+ });
+
+ expect(getIncrementalWarning(transform, metadata)).toBeUndefined();
+ });
+});
diff --git a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
index eac53b84235..ae8d3b8a4c8 100644
--- a/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformQueryPage/TransformQueryPage.tsx
@@ -32,7 +32,6 @@ import type {
Transform,
} from "metabase-types/api";
-import { useQueryComplexityChecks } from "../../components/QueryComplexityWarning";
import {
TransformEditor,
type TransformEditorProps,
@@ -131,8 +130,6 @@ function TransformQueryPageBody({
useRegisterMetabotTransformContext(transform, source, lastRunError);
- const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();
-
const {
checkData,
isCheckingDependencies,
@@ -186,13 +183,6 @@ function TransformQueryPageBody({
if (!isCompleteSource(source)) {
return;
}
- if ("source-incremental-strategy" in source) {
- const confirmed = await confirmIfQueryIsComplex(source);
- if (!confirmed) {
- return;
- }
- }
-
await handleInitialSave({ id: transform.id, source });
};
@@ -289,7 +279,6 @@ function TransformQueryPageBody({
isEnabled={isDirty && !isSaving && !isCheckingDependencies}
onConfirm={rejectProposed}
/>
- {modal}
</>
);
}
diff --git a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
index 240466e8722..30bae62d951 100644
--- a/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformRunPage/RunSection/RunSection.tsx
@@ -101,7 +101,7 @@ function RunStatusSection({ transform }: RunStatusSectionProps) {
<Stack gap={0}>
<RunStatus
run={last_run ?? null}
- neverRunMessage={t`This transform hasn’t been run before.`}
+ neverRunMessage={t`This transform hasn't been run before.`}
runInfo={
<Anchor
key="link"
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsx
new file mode 100644
index 00000000000..4453a532a05
--- /dev/null
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/ResetCheckpointSection.tsx
@@ -0,0 +1,78 @@
+import { useDisclosure } from "@mantine/hooks";
+import { useId } from "react";
+import { t } from "ttag";
+
+import {
+ skipToken,
+ useGetFieldQuery,
+ useResetCheckpointMutation,
+} from "metabase/api";
+import { ConfirmModal } from "metabase/common/components/ConfirmModal";
+import { useMetadataToasts } from "metabase/metadata/hooks";
+import { CheckpointValue } from "metabase/transforms/components/CheckpointValue";
+import { isTransformRunning } from "metabase/transforms/utils";
+import { Box, Button, Group, Icon, Text } from "metabase/ui";
+import type { Transform } from "metabase-types/api";
+
+export function ResetCheckpointSection({
+ transform,
+}: {
+ transform: Transform;
+}) {
+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();
+ const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+ const [resetCheckpoint, { isLoading }] = useResetCheckpointMutation();
+
+ const checkpointFieldId =
+ transform.source?.["source-incremental-strategy"]?.[
+ "checkpoint-filter-field-id"
+ ];
+ const { data: checkpointField } = useGetFieldQuery(
+ checkpointFieldId ? { id: checkpointFieldId } : skipToken,
+ );
+
+ const labelId = useId();
+
+ const handleConfirm = async () => {
+ const { error } = await resetCheckpoint(transform.id);
+ closeModal();
+ if (error) {
+ sendErrorToast(t`Failed to reset checkpoint`);
+ } else {
+ sendSuccessToast(t`Checkpoint has been reset`);
+ }
+ };
+
+ if (transform.last_checkpoint_value == null) {
+ return null;
+ }
+
+ return (
+ <Group gap="md" align="center">
+ <Box c="text-secondary" role="group" aria-labelledby={labelId}>
+ <span id={labelId}>{t`Current checkpoint`}: </span>
+ <Text component="span" fw="bold" c="text-primary">
+ <CheckpointValue
+ value={transform.last_checkpoint_value}
+ checkpointField={checkpointField}
+ />
+ </Text>
+ </Box>
+ <Button
+ leftSection={<Icon name="revert" aria-hidden />}
+ disabled={isTransformRunning(transform) || isLoading}
+ onClick={openModal}
+ >
+ {t`Reset checkpoint`}
+ </Button>
+ <ConfirmModal
+ title={t`Reset checkpoint?`}
+ message={t`This will cause the next run to reprocess all data from scratch instead of only new rows.`}
+ opened={isModalOpen}
+ onClose={closeModal}
+ onConfirm={handleConfirm}
+ confirmButtonText={t`Reset`}
+ />
+ </Group>
+ );
+}
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx
index 66250768345..d3b729d48ed 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TargetSection.unit.spec.tsx
@@ -28,6 +28,17 @@ import { createMockState } from "metabase-types/store/mocks";
import { TransformSettingsSection } from "./TransformSettingsSection";
+jest.mock(
+ "metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions",
+ () => ({
+ useHasCheckpointOptions: jest.fn().mockReturnValue({
+ hasCheckpointOptions: true,
+ hasNativeCheckpointOptions: true,
+ transformType: "mbql",
+ }),
+ }),
+);
+
type SetupOpts = {
remoteSyncType?: EnterpriseSettings["remote-sync-type"];
transform?: Transform;
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx
index c80ae6db9f7..0ce20d285e5 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/TransformSettingsSection.unit.spec.tsx
@@ -32,6 +32,17 @@ type SetupOpts = {
transform?: Transform;
};
+jest.mock(
+ "metabase/transforms/components/IncrementalTransform/useHasCheckpointOptions",
+ () => ({
+ useHasCheckpointOptions: jest.fn().mockReturnValue({
+ hasCheckpointOptions: true,
+ hasNativeCheckpointOptions: true,
+ transformType: "mbql",
+ }),
+ }),
+);
+
function setup({
transform = createMockTransform(),
remoteSyncType,
diff --git a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
index f74c4c28d17..e8c1a02bc5d 100644
--- a/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
+++ b/frontend/src/metabase/transforms/pages/TransformSettingsPage/TransformSettingsSection/UpdateIncrementalSettings.tsx
@@ -1,7 +1,10 @@
+import { useDisclosure } from "@mantine/hooks";
import { useFormikContext } from "formik";
+import { useCallback, useRef } from "react";
import { t } from "ttag";
import _ from "underscore";
+import { ConfirmModal } from "metabase/common/components/ConfirmModal";
import { Form, FormInlineUpdater, FormProvider } from "metabase/forms";
import { useMetadataToasts } from "metabase/metadata/hooks";
import type { IncrementalSettingsFormValues } from "metabase/transforms/components/IncrementalTransform";
@@ -9,9 +12,10 @@ import {
IncrementalTransformSettings,
useUpdateIncrementalSettings,
} from "metabase/transforms/components/IncrementalTransform";
-import { useQueryComplexityChecks } from "metabase/transforms/components/QueryComplexityWarning";
import type { Transform } from "metabase-types/api";
+import { ResetCheckpointSection } from "./ResetCheckpointSection";
+
type UpdateIncrementalSettingsProps = {
transform: Transform;
readOnly?: boolean;
@@ -23,44 +27,72 @@ const IncrementalTransformSettingsWrapper = ({
}: UpdateIncrementalSettingsProps) => {
const { values, setFieldValue } =
useFormikContext<IncrementalSettingsFormValues>();
- const { confirmIfQueryIsComplex, modal } = useQueryComplexityChecks();
- const handleIncrementalChange = async (value: boolean) => {
- if (value) {
- const confirmed = await confirmIfQueryIsComplex(transform.source);
- if (!confirmed) {
- return;
- }
- }
+ const handleIncrementalChange = (value: boolean) => {
setFieldValue("incremental", value);
+
+ // If incremental is turned off, reset the checkpoint filter field
+ if (!value) {
+ setFieldValue("checkpointFilterFieldId", null);
+ }
};
return (
- <>
- <IncrementalTransformSettings
- source={transform.source}
- incremental={values.incremental}
- onIncrementalChange={handleIncrementalChange}
- variant="standalone"
- readOnly={readOnly}
- />
- {modal}
- </>
+ <IncrementalTransformSettings
+ source={transform.source}
+ incremental={values.incremental}
+ onIncrementalChange={handleIncrementalChange}
+ variant="standalone"
+ readOnly={readOnly}
+ extraActions={
+ !readOnly && transform.last_checkpoint_value != null ? (
+ <ResetCheckpointSection transform={transform} />
+ ) : undefined
+ }
+ />
);
};
-export const UpdateIncrementalSettings = ({
+function UpdateIncrementalSettingsContent({
transform,
readOnly,
-}: UpdateIncrementalSettingsProps) => {
+ updateIncrementalSettings,
+}: UpdateIncrementalSettingsProps & {
+ updateIncrementalSettings: (
+ values: IncrementalSettingsFormValues,
+ ) => Promise<unknown>;
+}) {
const { sendSuccessToast, sendErrorToast } = useMetadataToasts();
+ const { onBeforeUpdate, modalProps } =
+ useCheckpointChangeInterceptor(transform);
- const showSuccessToast = () =>
- sendSuccessToast(t`Incremental transformation settings updated`);
-
- const showErrorToast = () =>
- sendErrorToast(t`Failed to update incremental transformation settings`);
+ return (
+ <>
+ <FormInlineUpdater
+ update={updateIncrementalSettings}
+ onBeforeUpdate={onBeforeUpdate}
+ onSuccess={() =>
+ sendSuccessToast(t`Incremental transformation settings updated`)
+ }
+ onError={() =>
+ sendErrorToast(
+ t`Failed to update incremental transformation settings`,
+ )
+ }
+ />
+ <ConfirmModal {...modalProps} />
+ <IncrementalTransformSettingsWrapper
+ transform={transform}
+ readOnly={readOnly}
+ />
+ </>
+ );
+}
+export function UpdateIncrementalSettings({
+ transform,
+ readOnly,
+}: UpdateIncrementalSettingsProps) {
const { initialValues, validationSchema, updateIncrementalSettings } =
useUpdateIncrementalSettings(transform);
@@ -72,16 +104,82 @@ export const UpdateIncrementalSettings = ({
enableReinitialize
>
<Form>
- <FormInlineUpdater
- update={updateIncrementalSettings}
- onSuccess={showSuccessToast}
- onError={showErrorToast}
- />
- <IncrementalTransformSettingsWrapper
+ <UpdateIncrementalSettingsContent
transform={transform}
readOnly={readOnly}
+ updateIncrementalSettings={updateIncrementalSettings}
/>
</Form>
</FormProvider>
);
-};
+}
+
+/**
+ * Hook that intercepts checkpoint field changes to show a confirmation modal
+ * when a checkpoint value already exists. Must be used inside FormProvider.
+ */
+function useCheckpointChangeInterceptor(transform: Transform) {
+ const { resetForm } = useFormikContext<IncrementalSettingsFormValues>();
+ const [isModalOpen, { open: openModal, close: closeModal }] = useDisclosure();
+ const pendingResolve = useRef<((shouldProceed: boolean) => void) | null>(
+ null,
+ );
+
+ const currentFieldId =
+ transform.source?.["source-incremental-strategy"]?.[
+ "checkpoint-filter-field-id"
+ ];
+
+ const onBeforeUpdate = useCallback(
+ async (values: IncrementalSettingsFormValues): Promise<boolean> => {
+ const waitingForCheckpointSelection =
+ values.incremental &&
+ values.sourceStrategy === "checkpoint" &&
+ values.checkpointFilterFieldId == null;
+
+ if (waitingForCheckpointSelection) {
+ return false;
+ }
+
+ const fieldChanged =
+ currentFieldId != null &&
+ values.checkpointFilterFieldId != null &&
+ String(currentFieldId) !== values.checkpointFilterFieldId;
+
+ if (fieldChanged && transform.last_checkpoint_value != null) {
+ return new Promise<boolean>((resolve) => {
+ pendingResolve.current = resolve;
+ openModal();
+ });
+ }
+
+ return true;
+ },
+ [transform, currentFieldId, openModal],
+ );
+
+ const handleConfirm = useCallback(() => {
+ pendingResolve.current?.(true);
+ pendingResolve.current = null;
+ closeModal();
+ }, [closeModal]);
+
+ const handleCancel = useCallback(() => {
+ resetForm();
+ pendingResolve.current?.(false);
+ pendingResolve.current = null;
+ closeModal();
+ }, [resetForm, closeModal]);
+
+ return {
+ onBeforeUpdate,
+ modalProps: {
+ title: t`Change checkpoint field?`,
+ message: t`Changing the checkpoint field will reset the stored checkpoint value. The next run will reprocess all data from scratch.`,
+ confirmButtonText: t`Change checkpoint field`,
+ opened: isModalOpen,
+ onClose: handleCancel,
+ onConfirm: handleConfirm,
+ },
+ };
+}
diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.ts
index 05a2e5592cf..7ac1df46c8b 100644
--- a/frontend/src/metabase/transforms/utils.ts
+++ b/frontend/src/metabase/transforms/utils.ts
@@ -21,8 +21,6 @@ import type {
TransformSource,
} from "metabase-types/api";
-import { CHECKPOINT_TEMPLATE_TAG } from "./constants";
-
export function parseTimestampWithTimezone(
timestamp: string,
systemTimezone: string | undefined,
@@ -258,11 +256,7 @@ export function getValidationResult(query: Lib.Query): ValidationResult {
return { isValid: Lib.canSave(query, "question") };
}
-const ALLOWED_TEMPLATE_TYPES = new Set([
- "card",
- "snippet",
- CHECKPOINT_TEMPLATE_TAG,
-]);
+const ALLOWED_TEMPLATE_TYPES = new Set(["card", "snippet", "table"]);
function validateTemplateTag(tag: TemplateTag): ValidationResult {
// Allow snippets, cards, and the special transform variables ({checkpoint})
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 26976ff61a2..526cb863c9f 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -3384,6 +3384,98 @@ databaseChangeLog:
);
rollback: # nothing to undo - these rows should not exist in v59
+ - changeSet:
+ id: v59.2026-03-12T12:00:00
+ author: wotbrew
+ comment: Add last_checkpoint_value to transform table
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform
+ columnName: last_checkpoint_value
+ changes:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: last_checkpoint_value
+ type: ${text.type}
+ remarks: Serialized checkpoint watermark value for incremental transforms, temporal as ISO8601
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v59.2026-03-13T00:00:00
+ author: bronsa
+ comment: Remove legacy incremental transform strategies that use checkpoint-filter instead of checkpoint-filter-field-id
+ changes:
+ - customChange:
+ class: "metabase.app_db.custom_migrations.RemoveLegacyIncrementalStrategies"
+
+ - changeSet:
+ id: v59.2026-03-13T00:00:01
+ author: bronsa
+ comment: Add checkpoint_filter_field_id column to transform_run
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_run
+ columnName: checkpoint_filter_field_id
+ changes:
+ - addColumn:
+ tableName: transform_run
+ columns:
+ - column:
+ name: checkpoint_filter_field_id
+ type: int
+ remarks: The field ID used for incremental checkpoint filtering
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v59.2026-03-13T00:00:02
+ author: bronsa
+ comment: Add checkpoint_lo_value column to transform_run
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_run
+ columnName: checkpoint_lo_value
+ changes:
+ - addColumn:
+ tableName: transform_run
+ columns:
+ - column:
+ name: checkpoint_lo_value
+ type: ${text.type}
+ remarks: The low checkpoint value for this run (exclusive lower bound)
+ constraints:
+ nullable: true
+
+ - changeSet:
+ id: v59.2026-03-13T00:00:03
+ author: bronsa
+ comment: Add checkpoint_hi_value column to transform_run
+ preConditions:
+ - onFail: MARK_RAN
+ - not:
+ - columnExists:
+ tableName: transform_run
+ columnName: checkpoint_hi_value
+ changes:
+ - addColumn:
+ tableName: transform_run
+ columns:
+ - column:
+ name: checkpoint_hi_value
+ type: ${text.type}
+ remarks: The high checkpoint value for this run (inclusive upper bound)
+ constraints:
+ nullable: true
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 5d6f1d70da5..7eedb67395e 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1993,3 +1993,61 @@
:schema uploads_schema_name}
:where [:= :id id]}))]
(run! retire-and-revive-upload-table! inactive-upload-tables))))))
+
+(defn- legacy-checkpoint-column-name
+ "Extract the column name from a legacy column-unique-key string.
+ Uses the same format as [[metabase.lib.metadata.column/unpack-unique-key]]:
+ `column-unique-key-v<version>$<column-key>`."
+ [unique-key]
+ (when-let [[_match _version column-key] (re-find #"^column-unique-key-v(\d+)\$(.+$)" unique-key)]
+ column-key))
+
+(defn- legacy-checkpoint-source-table-id
+ "Extract the source table ID from a parsed transform source for checkpoint migration.
+ Returns nil for native transforms (which can't be migrated)."
+ [parsed]
+ (case (keyword (:type parsed))
+ :query (get-in parsed [:query :stages 0 :source-table])
+ :python (let [source-tables (:source-tables parsed)]
+ (cond
+ ;; vec format: [{:alias "t" :table_id 42}]
+ (sequential? source-tables)
+ (when (= 1 (count source-tables))
+ (:table_id (first source-tables)))
+ ;; map format: {"t" 42} or {"t" {:table_id 42}}
+ (map? source-tables)
+ (when (= 1 (count source-tables))
+ (let [v (first (vals source-tables))]
+ (if (map? v) (:table_id v) v)))))
+ nil))
+
+(defn- resolve-checkpoint-field-id
+ "Try to resolve a legacy checkpoint-filter-unique-key to a field ID."
+ [parsed]
+ (when-let [unique-key (get-in parsed [:source-incremental-strategy :checkpoint-filter-unique-key])]
+ (when-let [column-name (legacy-checkpoint-column-name unique-key)]
+ (when-let [table-id (legacy-checkpoint-source-table-id parsed)]
+ (when (int? table-id)
+ (t2/select-one-pk :model/Field
+ :table_id table-id
+ :name column-name
+ :active true))))))
+
+(define-migration RemoveLegacyIncrementalStrategies
+ (doseq [{:keys [id source]} (t2/select [:transform :id :source])]
+ (let [parsed (json-out source true)]
+ (when (:source-incremental-strategy parsed)
+ (if-let [field-id (resolve-checkpoint-field-id parsed)]
+ ;; Migrate: replace legacy keys with checkpoint-filter-field-id
+ (let [migrated (update parsed :source-incremental-strategy
+ (fn [s]
+ (-> s
+ (dissoc :checkpoint-filter-unique-key :checkpoint-filter)
+ (assoc :checkpoint-filter-field-id field-id))))]
+ (t2/query {:update :transform
+ :set {:source (json-in migrated)}
+ :where [:= :id id]}))
+ ;; Can't migrate: strip incremental strategy
+ (t2/query {:update :transform
+ :set {:source (json-in (dissoc parsed :source-incremental-strategy))}
+ :where [:= :id id]}))))))
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 93bdc5cf98b..06c50b9a61b 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -123,14 +123,21 @@
(when-let [new-collection (:collection_id (t2/changes transform))]
(collection/check-collection-namespace :model/Transform new-collection)
(collection/check-allowed-content :model/Transform new-collection))
- (cond-> transform
- source
- (assoc :source_type (transforms-base.u/transform-source-type source)
- :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
-
- (or (:source (t2/changes transform)) (:target (t2/changes transform)))
- ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.
- (assoc :target_db_id (transforms-base.i/target-db-id transform))))
+ (let [changes (t2/changes transform)]
+ (cond-> transform
+ source
+ (assoc :source_type (transforms-base.u/transform-source-type source)
+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
+
+ (or (:source changes) (:target changes))
+ ;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.
+ (assoc :target_db_id (transforms-base.i/target-db-id transform))
+
+ ;; Reset checkpoint when the incremental filter field changes
+ (let [old-field-id (get-in (t2/original transform) [:source :source-incremental-strategy :checkpoint-filter-field-id])
+ new-field-id (get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])]
+ (and old-field-id (not= old-field-id new-field-id)))
+ (assoc :last_checkpoint_value nil))))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
@@ -323,7 +330,7 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
- :skip [:dependency_analysis_version :source_type :target_db_id]
+ :skip [:dependency_analysis_version :source_type :target_db_id :last_checkpoint_value]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 88b43649a68..801ad9b2c78 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -34,8 +34,6 @@
source-tables-vec->map-for-fe
check-database-feature
check-feature-enabled!
- extract-all-columns-from-query
- extract-incremental-filter-columns-from-query
validate-incremental-column-type!
validate-transform-query!
get-transforms
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index 38d21a3e6b5..e612fb2d9a3 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -5,24 +5,17 @@
(:require
[metabase.api.common :as api]
[metabase.database-routing.core :as database-routing]
- [metabase.driver :as driver]
- [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
- [metabase.lib.core :as lib]
[metabase.models.interface :as mi]
[metabase.models.transforms.transform :as transform.model]
- [metabase.query-processor.compile :as qp.compile]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.ordering :as transforms-base.ordering]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
- [metabase.util.log :as log]
- [toucan2.core :as t2])
- (:import
- (java.sql PreparedStatement)))
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -65,82 +58,21 @@
(assoc error
:status-code 400)))))
-(defn extract-all-columns-from-query
- "Extracts column metadata (name and type) from a query.
-
- Returns a sequence of maps with `:name` and `:base_type` keys, or nil if extraction fails.
-
- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
- to inspect the query structure. This works for most modern JDBC drivers but may not
- be supported by all drivers or for all query types."
- [driver database-id query]
- (try
- (let [{:keys [query]} (qp.compile/compile query)]
- (sql-jdbc.execute/do-with-connection-with-options
- driver
- database-id
- {}
- (fn [conn]
- (with-open [^PreparedStatement stmt (sql-jdbc.execute/prepared-statement driver conn query [])]
- (when-let [rsmeta (.getMetaData stmt)]
- (seq (sql-jdbc.execute/column-metadata driver rsmeta)))))))
- (catch Exception e
- (log/debugf e "Failed to extract columns from query: %s" (ex-message e))
- nil)))
-
-(defn extract-incremental-filter-columns-from-query
- "Extracts column names suitable for incremental transform checkpoint filtering.
-
- This function is specifically for incremental transform checkpoint column selection.
- It only returns columns with types supported for checkpoint filtering:
- - Temporal types (timestamp, timestamp with timezone)
- - Numeric types (integer, float, decimal)
-
- Text, boolean, and other types are filtered out as they are not supported for
- incremental checkpointing.
-
- Returns a vector of column names (as strings), or nil if extraction fails.
-
- The query is first compiled to native SQL, then uses PreparedStatement.getMetaData()
- to inspect the query structure. This works for most modern JDBC drivers but may not
- be supported by all drivers or for all query types."
- [driver database-id query]
- (some->> (extract-all-columns-from-query driver database-id query)
- (filter (comp transforms-base.u/supported-incremental-filter-type? :base_type))
- (mapv :name)))
-
(defn validate-incremental-column-type!
"Validates that the checkpoint column for an incremental transform has a supported type.
- For MBQL/Python transforms, resolves the column from the query using the unique key.
- For native queries, extracts columns from the query and checks the checkpoint-filter column.
-
- Throws a 400 error if the column type is not supported or cannot be resolved."
+ Resolves the field by ID and checks that its base-type is numeric or temporal.
+ Throws a 400 error if the field cannot be found or its type is not supported."
[{:keys [source]}]
- (when-let [{:keys [checkpoint-filter checkpoint-filter-unique-key] strategy-type :type}
+ (when-let [{:keys [checkpoint-filter-field-id] strategy-type :type}
(:source-incremental-strategy source)]
- (when (and (= :query (:type source)) (= "checkpoint" strategy-type))
- (let [{:keys [query]} source
- database-id (:database query)
- database (api/check-404 (t2/select-one :model/Database :id database-id))
- driver-name (driver/the-initialized-driver (:engine database))]
- (cond
- ;; For MBQL, resolve column from query metadata
- checkpoint-filter-unique-key
- (let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]
- (api/check-400 column (deferred-tru "Checkpoint column not found in query."))
- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base-type column))
- (deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."
- (pr-str (:base-type column)))))
-
- ;; For native query with checkpoint-filter, validate type if we can extract the column metadata
- checkpoint-filter
- (when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]
- (when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]
- (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type column))
- (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
- checkpoint-filter
- (pr-str (:base_type column)))))))))))
+ (when (= "checkpoint" strategy-type)
+ (let [field (t2/select-one :model/Field checkpoint-filter-field-id)]
+ (api/check-400 field (deferred-tru "Checkpoint field not found."))
+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type field))
+ (deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
+ (:name field)
+ (pr-str (:base_type field))))))))
(defn get-transforms
"Get a list of transforms."
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index de3ed6bb1aa..bce1e9f32be 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -35,25 +35,28 @@
:db/id (:id db)
:db/engine (name driver)}
(let [conn-spec (driver/connection-spec driver db)
- transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}]
- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
- (transforms.u/run-cancelable-transform!
- run-id driver transform-details
- (fn [cancel-chan]
- (let [result (transforms-base.i/execute-base!
- transform
- {:cancelled? #(boolean (a/poll! cancel-chan))
- :run-id run-id
- :with-stage-timing-fn (fn [rid stage thunk]
- (transforms.instrumentation/with-stage-timing [rid stage]
- (thunk)))})]
- ;; Bridge result-map to exception-based flow for run-cancelable-transform!
- (when-not (= :succeeded (:status result))
- (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
- result))))
+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
+ exec-result
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.u/run-cancelable-transform!
+ run-id transform driver transform-details
+ (fn [cancel-chan source-range-params]
+ (let [result (transforms-base.i/execute-base!
+ transform
+ {:cancelled? #(boolean (a/poll! cancel-chan))
+ :run-id run-id
+ :source-range-params source-range-params
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})]
+ ;; Bridge result-map to exception-based flow for run-cancelable-transform!
+ (when-not (= :succeeded (:status result))
+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
+ result))))]
;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
(transforms-base.u/complete-execution! transform
{:run-id run-id
+ :source-range-params (:source-range-params exec-result)
:with-stage-timing-fn (fn [rid stage thunk]
(transforms.instrumentation/with-stage-timing [rid stage]
(thunk)))})))))
diff --git a/src/metabase/transforms/schema.clj b/src/metabase/transforms/schema.clj
index a235243e4b1..46679487d71 100644
--- a/src/metabase/transforms/schema.clj
+++ b/src/metabase/transforms/schema.clj
@@ -1,7 +1,7 @@
(ns metabase.transforms.schema
(:require
- [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.schema :as queries.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util.malli.registry :as mr]
@@ -10,11 +10,7 @@
(mr/def ::checkpoint-strategy
[:map
[:type [:= "checkpoint"]]
- ;; for native
- [:checkpoint-filter {:optional true} :string]
- ;; for mbql and python
- [:checkpoint-filter-unique-key {:optional true}
- ::lib.schema/column-unique-key]])
+ [:checkpoint-filter-field-id {:optional true} ::lib.schema.id/field]])
(mr/def ::source-incremental-strategy
[:multi {:dispatch :type}
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index 7692aa73a69..e302b750786 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -106,20 +106,25 @@
(defn run-cancelable-transform!
"Execute a transform with cancellation support and proper error handling.
+ Computes `source-range-params` once upfront, saves them to the run record, and passes them
+ to `run-transform!` which receives `cancel-chan` and `source-range-params` as arguments.
+
Options:
- `:ex-message-fn` change how caught exceptions are presented to the user in run logs, by default the same as clojure.core/ex-message"
- [run-id driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}]
+ [run-id transform driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}]
;; local run is responsible for status, using canceling lifecycle
(try
(when-not (driver/schema-exists? driver db-id output-schema)
(driver/create-schema-if-needed! driver conn-spec output-schema))
- (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
- (let [cancel-chan (a/promise-chan)
- ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
- (canceling/chan-start-run! run-id cancel-chan)
- (run-transform! cancel-chan))]
- (transform-run/succeed-started-run! run-id)
- ret)
+ (let [source-range-params (transforms-base.u/get-source-range-params transform)]
+ (transforms-base.u/save-run-checkpoint-range! run-id source-range-params)
+ (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
+ (let [cancel-chan (a/promise-chan)
+ ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
+ (canceling/chan-start-run! run-id cancel-chan)
+ (run-transform! cancel-chan source-range-params))]
+ (transform-run/succeed-started-run! run-id)
+ ret))
(catch Throwable t
(if (:timeout (ex-data t))
(transform-run/timeout-run! run-id {:message (ex-message-fn t)})
diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.clj
index df3f8e654d7..84926b47f17 100644
--- a/src/metabase/transforms_base/core.clj
+++ b/src/metabase/transforms_base/core.clj
@@ -42,5 +42,6 @@
([transform opts]
(let [result (transforms-base.i/execute-base! transform opts)]
(when (= :succeeded (:status result))
- (transforms-base.u/complete-execution! transform opts))
+ (transforms-base.u/complete-execution! transform
+ (assoc opts :source-range-params (:source-range-params result))))
result)))
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
index cfb9347745f..aa8ab55e614 100644
--- a/src/metabase/transforms_base/query.clj
+++ b/src/metabase/transforms_base/query.clj
@@ -79,7 +79,7 @@
:result <driver result>
:error <exception if failed>}"
[{:keys [id source target] :as transform} :- ::transforms-base.schema/transform
- {:keys [cancelled?] :as _opts} :- [:maybe ::transforms-base.schema/execute-base-options]]
+ {:keys [cancelled? source-range-params] :as _opts} :- [:maybe ::transforms-base.schema/execute-base-options]]
(try
;; Check cancellation before starting
(when (and cancelled? (cancelled?))
@@ -88,12 +88,18 @@
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
_ (transforms-base.u/throw-if-db-routing-enabled! transform database)
+ ;; First incremental run (no checkpoint) should behave like non-incremental
+ ;; to drop and recreate the table rather than appending to existing data.
+ effective-transform-type (if (and (= :table-incremental (keyword (:type target)))
+ (nil? (:last_checkpoint_value transform)))
+ :table
+ (keyword (:type target)))
transform-details {:db-id db
:database database
:transform-id id
- :transform-type (keyword (:type target))
+ :transform-type effective-transform-type
:conn-spec (driver/connection-spec driver database)
- :query (transforms-base.u/compile-source transform)
+ :query (transforms-base.u/compile-source transform source-range-params)
:output-schema (:schema target)
:output-table (transforms-base.u/qualified-table-name driver target)}
opts (transform-opts transform-details)
@@ -121,7 +127,8 @@
(throw (ex-info "Transform cancelled after query execution" {:status :cancelled})))
{:status :succeeded
- :result result}))
+ :result result
+ :source-range-params source-range-params}))
(catch Exception e
(let [data (ex-data e)]
diff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.clj
index 0604a3137ce..cb0a0aed8cb 100644
--- a/src/metabase/transforms_base/schema.clj
+++ b/src/metabase/transforms_base/schema.clj
@@ -5,6 +5,8 @@
The full transforms module (metabase.transforms.schema) extends these with
additional fields like :id (required for scheduled execution)."
(:require
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms])
(:import
@@ -39,6 +41,22 @@
[:name {:optional true} :string]
[:description {:optional true} [:maybe :string]]])
+;;; ----------------------------------------- Source Range Params -----------------------------------------------
+
+(mr/def ::checkpoint-bound
+ "A bound (lo or hi) for incremental checkpoint filtering."
+ [:map
+ [:value :any]])
+
+(mr/def ::source-range-params
+ "Parameters for incremental range filtering on a source query.
+ Returned by get-source-range-params."
+ [:map
+ [:column ::lib.schema.metadata/column]
+ [:checkpoint-filter-field-id ::lib.schema.id/field]
+ [:lo {:optional true} [:maybe ::checkpoint-bound]]
+ [:hi {:optional true} [:maybe ::checkpoint-bound]]])
+
;;; ------------------------------------------------- Options -------------------------------------------------
(mr/def ::execute-base-options
@@ -49,7 +67,8 @@
[:with-stage-timing-fn {:optional true} ifn?]
[:publish-events? {:optional true} :boolean]
[:message-log {:optional true} [:maybe ::atom]]
- [:cancel-chan {:optional true} [:maybe ::chan]]])
+ [:cancel-chan {:optional true} [:maybe ::chan]]
+ [:source-range-params {:optional true} [:maybe ::source-range-params]]])
;;; ------------------------------------------------- Result -------------------------------------------------
@@ -59,4 +78,5 @@
[:status [:enum :succeeded :failed :cancelled :timeout]]
[:result {:optional true} :any]
[:error {:optional true} [:maybe (ms/InstanceOfClass Throwable)]]
- [:logs {:optional true} [:maybe :string]]])
+ [:logs {:optional true} [:maybe :string]]
+ [:source-range-params {:optional true} [:maybe ::source-range-params]]])
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 94ecc69dafb..a62fa0cff3c 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -4,18 +4,15 @@
This namespace contains functions that are used by both query and python transforms
and do NOT require transform_run database access."
(:require
- [buddy.core.codecs :as codecs]
- [buddy.core.hash :as buddy-hash]
[clojure.string :as str]
[java-time.api :as t]
[metabase.driver :as driver]
- [metabase.driver.sql.query-processor :as sql.qp]
- [metabase.driver.util :as driver.u]
[metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema.common :as lib.schema.common]
+ [metabase.lib.util :as lib.util]
[metabase.query-processor :as qp]
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.middleware.add-remaps :as remap]
@@ -24,9 +21,9 @@
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.sync.core :as sync]
[metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.schema :as transforms-base.schema]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
- [metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -145,11 +142,6 @@
;;; ------------------------------------------------- Incremental/Checkpoint Helpers -------------------------------------------------
-(defn- checkpoint-incremental?
- "Returns true if `source` uses checkpoint-based incremental strategy."
- [source]
- (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
-
(defn supported-incremental-filter-type?
"Returns true if the given base-type is supported for incremental filtering.
@@ -158,118 +150,144 @@
(or (isa? base-type :type/Temporal)
(isa? base-type :type/Number)))
-(defn- source->checkpoint-filter-unique-key
- "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
- [query source-incremental-strategy]
- (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))
-
-(defn- source->checkpoint-filter-column
- "Resolve the checkpoint filter column for an incremental transform.
-
- Tries to resolve the column using the unique key first.
- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.
-
- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).
- Throws an exception if the column type is not supported."
- [query source-incremental-strategy table metadata-provider]
- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]
- (when-some [{column-name :name
- :keys [base-type]
- :as column}
- (cond
- checkpoint-filter-unique-key
- (source->checkpoint-filter-unique-key query source-incremental-strategy)
- checkpoint-filter
- (when-some [field-id (t2/select-one-pk :model/Field
- :table_id (:id table)
- :name checkpoint-filter)]
- (lib.metadata/field metadata-provider field-id)))]
- (when-not (supported-incremental-filter-type? base-type)
- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "
- "Only numeric and temporal columns are supported for incremental filtering.")
- {:column-name column-name
- :base-type base-type})))
- column)))
-
-(declare target-table)
-
-(defn next-checkpoint
- "Build a query to compute the MAX of the checkpoint column from the target table.
-
- Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),
- or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."
- [{:keys [source target] :as transform}]
- (let [db-id (transforms-base.i/target-db-id transform)]
- (when (checkpoint-incremental? source)
- (when-let [table (target-table db-id target)]
- (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
- table-metadata (lib.metadata/table metadata-provider (:id table))
- query (lib/query metadata-provider table-metadata)]
- (when-let [filter-column (source->checkpoint-filter-column query
- (:source-incremental-strategy source)
- table metadata-provider)]
- {:query (-> query (lib/aggregate (lib/max filter-column)))
- :filter-column filter-column}))))))
-
-(defn- next-checkpoint-value
- "Execute the checkpoint query and normalize the result for database insertion.
- Returns `nil` if the target table is empty."
- [{:keys [query filter-column]}]
- (let [{:keys [base-type]} filter-column
- v (some-> query qp/process-query :data :rows first first)]
- ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right
- ;; shape for reinsertion
- (cond
- (nil? v)
- nil
-
- (isa? base-type :type/Integer)
- (bigint v)
-
- ;; any other number that's not an integer, should be a decimal/float
- (number? v)
- (bigdec v)
+(defn- encode-checkpoint-value [v]
+ (if (number? v)
+ (str v)
+ (u.date/format v)))
+
+(defn save-watermark!
+ "Commits the incremental transforms :hi watermark value to the appdb."
+ [transform-id source-range-params]
+ (t2/update! :model/Transform
+ transform-id
+ {:last_checkpoint_value (some-> source-range-params :hi :value encode-checkpoint-value)}))
+
+(defn save-run-checkpoint-range!
+ "Persist the checkpoint range (lo/hi) on a transform run record.
+ This is called early in execution so the range is recorded even if the run fails."
+ [run-id source-range-params]
+ (when (and run-id source-range-params)
+ (let [{:keys [checkpoint-filter-field-id lo hi]} source-range-params]
+ (t2/update! :model/TransformRun run-id
+ (cond-> {:checkpoint_filter_field_id checkpoint-filter-field-id}
+ lo (assoc :checkpoint_lo_value (encode-checkpoint-value (:value lo)))
+ hi (assoc :checkpoint_hi_value (encode-checkpoint-value (:value hi))))))))
+
+(defn- parse-checkpoint-value
+ "Parse a serialized checkpoint value string according to its base-type keyword."
+ [base-type s]
+ (cond
+ (not (string? s)) s
+ (isa? base-type :type/Float) (bigdec s)
+ (isa? base-type :type/Number) (biginteger s)
+ (isa? base-type :type/Temporal) (u.date/parse s)
+ :else (throw (ex-info (str "Unsupported checkpoint type: " (pr-str base-type))
+ {:base-type base-type}))))
+
+(defn- tag-checkpoint-value
+ "Wrap a raw checkpoint value from the QP into a map `{:value parsed}`."
+ [base-type raw-value]
+ {:value (parse-checkpoint-value base-type raw-value)})
+
+(defn- inject-filters-into-table-tag
+ "Inject `:source-filters` into the table template tag matching the checkpoint field's table.
+
+ Instead of manually expanding {{tag}} to a subquery SQL string, this adds filter metadata
+ to the template tag so the QP's existing pipeline handles substitution. The filters specify
+ field-id-based comparisons (e.g. :> lo, :<= hi) that get rendered as a filtered subquery
+ in `->replacement-snippet-info`."
+ [query source-range-params]
+ (let [{:keys [checkpoint-filter-field-id lo hi column]} source-range-params
+ table-id (:table-id column)
+ tags (lib/template-tags query)
+ tag-name (some (fn [[k v]]
+ (when (and (#{:table "table"} (:type v))
+ (= table-id (:table-id v)))
+ k))
+ tags)
+ _ (when-not tag-name
+ (throw (ex-info "No table variable found for checkpoint field's table"
+ {:checkpoint-table-id table-id
+ :template-tags tags})))
+ filters (cond-> []
+ lo (conj {:field-id checkpoint-filter-field-id :op :> :value (:value lo)})
+ hi (conj {:field-id checkpoint-filter-field-id :op :<= :value (:value hi)}))]
+ (lib.util/update-query-stage
+ query 0
+ #(assoc-in % [:template-tags tag-name :source-filters] filters))))
+
+(mu/defn get-source-range-params :- [:maybe ::transforms-base.schema/source-range-params]
+ "Returns information on the incremental range filters that ought to be applied to a source query.
- :else v)))
+ Returns a map:
+ :column (the lib column value of the incremental filter column)
+ :checkpoint-filter-field-id (the field ID of the checkpoint column)
+ Range predicate terms (maps :type, :value), can be nil (in which case the filter clause should be omitted):
+ :lo values in the source table must be > this :value.
+ :hi values in the source table must be <= this :value."
+ [{:keys [source target] :as transform}]
+ (let [{:keys [source-incremental-strategy]} source
+ {:keys [checkpoint-filter-field-id]} source-incremental-strategy]
+ (when (and (= "table-incremental" (:type target))
+ (native-query-transform? transform)
+ (not (some (fn [[_k v]] (#{:table "table"} (:type v)))
+ (lib/template-tags (:query source)))))
+ (let [msg (i18n/tru (str "Incremental transform with a native query requires a table variable. "
+ "Please add a table variable to the query and update the checkpoint field."))]
+ (throw (ex-info msg {:transform-message msg}))))
+ (when (and (= "table-incremental" (:type target))
+ (not checkpoint-filter-field-id))
+ (let [msg (i18n/tru (str "Incremental transform is enabled but no checkpoint field is selected. "
+ "Please select a checkpoint field in the transform settings."))]
+ (throw (ex-info msg {:transform-message msg}))))
+ (when checkpoint-filter-field-id
+ (let [{:keys [last_checkpoint_value]} transform
+ db-id (transforms-base.i/target-db-id transform)
+ metadata-provider (lib-be/application-database-metadata-provider db-id)
+ column (lib.metadata/field metadata-provider checkpoint-filter-field-id)
+ _ (when (or (nil? column) (not (:active column)))
+ (throw (ex-info "Checkpoint field does not exist or is not active"
+ {:checkpoint-filter-field-id checkpoint-filter-field-id})))
+ _ (when-not (supported-incremental-filter-type? (:base-type column))
+ (throw (ex-info (str "Checkpoint column '" (:name column) "' has unsupported type " (pr-str (:base-type column)) ". "
+ "Only numeric and temporal columns are supported for incremental filtering.")
+ {:column column})))
+ base-type (:base-type column)
+ lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))
+
+ max-value
+ (let [table-id (:table-id column)
+ table-metadata (lib.metadata/table metadata-provider table-id)
+ base-query (lib/query metadata-provider table-metadata)
+ filtered-query (if lo (lib/filter base-query (lib/> column lo)) base-query)
+ query (lib/aggregate (lib/append-stage filtered-query) (lib/max column))
+ query-result (qp/process-query query)]
+ (ffirst (get-in query-result [:data :rows])))]
+ {:column column
+ :checkpoint-filter-field-id checkpoint-filter-field-id
+ :lo (when lo {:value lo})
+ :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)
+ lo {:value lo})}))))
(defn preprocess-incremental-query
- "Add checkpoint checkpoint filtering to a query for incremental execution.
-
- For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.
- For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.
- Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."
- [query source-incremental-strategy checkpoint]
- (if-let [checkpoint-value (next-checkpoint-value checkpoint)]
+ "Add checkpoint filtering to a query for incremental execution.
+
+ For native queries, injects `:source-filters` into the table template tag matching the checkpoint
+ field's table. The QP's substitution pipeline renders these as a filtered subquery.
+ For MBQL queries, adds filter clauses `WHERE checkpoint_column > lo AND checkpoint_column <= hi`.
+ Returns the query unchanged when source-range-params is nil."
+ [query source-range-params]
+ (if source-range-params
(if (lib/native? query)
- ;; native query with explicit checkpoint filter
- (if (get-in query [:stages 0 :template-tags "checkpoint"])
- (update query :parameters conj
- {:type (if (number? checkpoint-value) :number :text)
- :target [:variable [:template-tag "checkpoint"]]
- :value checkpoint-value})
- query)
- ;; mbql query
- (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))
+ ;; Native: inject source-filters into the table template tag — QP handles the rest
+ (inject-filters-into-table-tag query source-range-params)
+ ;; MBQL: add filter clauses
+ (cond-> query
+ (:lo source-range-params) (lib/filter 0 (lib/> (:column source-range-params) (:value (:lo source-range-params))))
+ (:hi source-range-params) (lib/filter 0 (lib/<= (:column source-range-params) (:value (:hi source-range-params))))))
+ ;; No range params - return unchanged
query))
-(defn- post-process-incremental-query
- "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.
-
- Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "
- [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]
- (let [{:keys [checkpoint-filter]} source-incremental-strategy]
- (if (and (lib/native? (:query source))
- (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))
- (next-checkpoint-value checkpoint))
- (let [wrap-query (fn [query]
- (let [honeysql-query {:select [:*]
- :from [[[:raw (str "(" query ")")] :subquery]]
- :where [:> (h2x/identifier :field checkpoint-filter)
- [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]
- (first (sql.qp/format-honeysql driver honeysql-query))))]
- (update outer-query :query wrap-query))
- outer-query)))
-
(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]
"Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."
[{:keys [source]}]
@@ -283,21 +301,18 @@
(defn compile-source
"Compile the source query of a transform to SQL, applying incremental filtering if required."
- [{:keys [source] :as transform}]
- (let [{:keys [source-incremental-strategy] query-type :type} source]
- (case (keyword query-type)
- :query
- (let [checkpoint (next-checkpoint transform)
- query (:query source)
- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
- (binding [driver/*compile-with-inline-parameters*
- (or (= :clickhouse driver)
- driver/*compile-with-inline-parameters*)]
- (-> query
- (preprocess-incremental-query source-incremental-strategy checkpoint)
- massage-sql-query
- qp.compile/compile
- (post-process-incremental-query driver source checkpoint)))))))
+ [{:keys [source]} source-range-params]
+ (let [{query-type :type} source]
+ (assert (= :query (keyword query-type)))
+ (let [query (:query source)
+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
+ (binding [driver/*compile-with-inline-parameters*
+ (or (= :clickhouse driver)
+ driver/*compile-with-inline-parameters*)]
+ (-> query
+ (preprocess-incremental-query source-range-params)
+ massage-sql-query
+ qp.compile/compile)))))
;;; ------------------------------------------------- Target Table Management -------------------------------------------------
@@ -420,106 +435,13 @@
(log/infof "Renaming tables: %s" (pr-str rename-map))
(driver/rename-tables! driver database-id rename-map))
-;;; ------------------------------------------------- Secondary Index Management -------------------------------------------------
-
-(def ^:private metabase-index-prefix "mb_transform_idx_")
-
-(defn- incremental-filter-index-name [schema table-name filter-column-name]
- (let [prefix metabase-index-prefix
- suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]
- (str prefix suffix)))
-
-(defn- should-index-incremental-filter-column? [driver database]
- (and (driver.u/supports? driver :describe-indexes database)
- (driver.u/supports? driver :transforms/index-ddl database)))
-
-(defn- metabase-owned-index? [index]
- (str/starts-with? (:index-name index) metabase-index-prefix))
-
-(defn- metabase-incremental-filter-index [filter-column target]
- (let [table-name (:name target)
- schema (:schema target)
- filter-name (:name filter-column)]
- ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)
- {:index-name (incremental-filter-index-name schema table-name filter-name)
- :value filter-name}))
-
-(defn- decide-secondary-index-ddl
- "Decides which indexes should be dropped/created on the target table.
-
- e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.
-
- Indexes are represented as maps with at least the keys :index-name, :value.
- This matches the form used by [[metabase.driver/describe-table-indexes]].
-
- Returns a map:
- :drop - a vector of indexes that are redundant and should be dropped.
- :create - a vector of desirable indexes that do not exist and that should be created.
-
- Notes:
- - If user covering indexes exist they should be reused.
- - Will never drop user indexes.
- - Indexes we previously created and are no longer required are dropped."
- [{:keys [filter-column indexes target database]}]
- (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)
- default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))
- column-name (:name filter-column)
- existing-user-index (first (filter #(= (:value %) column-name) user-indexes))
- existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))
- driver (:engine database)
- intended-index (when (should-index-incremental-filter-column? driver database)
- ;; Prefer reuse to not create redundant indexes
- (or existing-user-index default-mb-index))
- drop (if intended-index
- (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)
- mb-indexes)
- create (when (and intended-index
- (not= (:index-name intended-index) (:index-name existing-user-index))
- (not= (:index-name intended-index) (:index-name existing-mb-index)))
- [intended-index])]
- {:drop (vec drop)
- :create (vec create)}))
-
-(defn execute-secondary-index-ddl-if-required!
- "If target table index modifications are required, executes those CREATE/DROP commands.
- See [[decide-secondary-index-ddl]] for details.
-
- `run-id` is optional - if nil, instrumentation/timing is skipped.
- `with-stage-timing-fn` is optional - if provided, will be called as (with-stage-timing-fn run-id stage thunk)."
- ([transform database target]
- (execute-secondary-index-ddl-if-required! transform nil database target nil))
- ([transform run-id database target with-stage-timing-fn]
- (when (driver.u/supports? (:engine database) :describe-indexes database)
- (let [driver (:engine database)
- indexes (driver/describe-table-indexes driver database target)
- checkpoint (next-checkpoint transform)
- {:keys [drop create]}
- (decide-secondary-index-ddl
- {:filter-column (:filter-column checkpoint)
- :database database
- :target target
- :indexes indexes})
- maybe-timed (fn [stage thunk]
- (if (and run-id with-stage-timing-fn)
- (with-stage-timing-fn run-id stage thunk)
- (thunk)))]
- (doseq [{:keys [index-name value]} drop]
- (maybe-timed [:import :drop-incremental-filter-index]
- (fn []
- (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))
- (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name))))
- (doseq [{:keys [index-name value]} create]
- (maybe-timed [:import :create-incremental-filter-index]
- (fn []
- (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))
- (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))))
-
;;; ------------------------------------------------- Post-Execution Completion -------------------------------------------------
(defn complete-execution!
"Post-processing steps after a transform has been executed successfully.
Performs:
+ - Save watermark (if source-range-params provided)
- Sync target table to AppDB
- Set `transform_id` on the target table
- Publish Metabase events (unless `:publish-events?` is false)
@@ -530,10 +452,13 @@
to preserve the correct order of operations."
[transform opts]
(let [{:keys [target]} transform
- {:keys [run-id with-stage-timing-fn publish-events?]
+ {:keys [publish-events? source-range-params]
:or {publish-events? true}} opts
db-id (transforms-base.i/target-db-id transform)
database (t2/select-one :model/Database db-id)]
+ ;; Save watermark if source-range-params available
+ (when source-range-params
+ (save-watermark! (:id transform) source-range-params))
;; Sync target table
(sync-target! target database)
;; Mark the table as owned by this transform
@@ -549,10 +474,7 @@
:transform-id (:id transform)
:transform-type (keyword (:type target))
:output-schema (:schema target)
- :output-table (qualified-table-name (:engine database) target)}}))
- ;; Create secondary indexes if needed
- (execute-secondary-index-ddl-if-required!
- transform run-id database target with-stage-timing-fn)))
+ :output-table (qualified-table-name (:engine database) target)}}))))
;;; ------------------------------------------------- Source Table Schemas -------------------------------------------------
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 4810afc61a6..159d7b636b4 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -4,7 +4,6 @@
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.api.util.handlers :as handlers]
- [metabase.driver :as driver]
[metabase.events.core :as events]
[metabase.query-processor :as qp]
[metabase.query-processor.middleware.constraints :as qp.constraints]
@@ -12,7 +11,6 @@
[metabase.query-processor.streaming :as qp.streaming]
[metabase.request.core :as request]
[metabase.server.core :as server]
- [metabase.sql-tools.core :as sql-tools]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-inspector.core :as inspector]
[metabase.transforms-inspector.schema :as inspector.schema]
@@ -74,7 +72,10 @@
[:message [:maybe :string]]
[:user_id [:maybe pos-int?]]
[:transform_name {:optional true} [:maybe :string]]
- [:transform_entity_id {:optional true} [:maybe :string]]])
+ [:transform_entity_id {:optional true} [:maybe :string]]
+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
+ [:checkpoint_lo_value {:optional true} [:maybe :string]]
+ [:checkpoint_hi_value {:optional true} [:maybe :string]]])
(def ^:private TransformResponse
[:map {:closed true}
@@ -100,7 +101,8 @@
[:table {:optional true} [:maybe :map]]
[:owner_user_id {:optional true} [:maybe pos-int?]]
[:owner_email {:optional true} [:maybe :string]]
- [:owner {:optional true} [:maybe OwnerResponse]]])
+ [:owner {:optional true} [:maybe OwnerResponse]]
+ [:last_checkpoint_value {:optional true} [:maybe :string]]])
(def ^:private TransformRunResponse
[:map {:closed true}
@@ -115,6 +117,9 @@
[:user_id [:maybe pos-int?]]
[:transform_name {:optional true} [:maybe :string]]
[:transform_entity_id {:optional true} [:maybe :string]]
+ [:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
+ [:checkpoint_lo_value {:optional true} [:maybe :string]]
+ [:checkpoint_hi_value {:optional true} [:maybe :string]]
;; Transform can have id/name when exists, or be nil when deleted
[:transform {:optional true} [:maybe [:map {:closed true}
[:id {:optional true} pos-int?]
@@ -299,6 +304,13 @@
(transforms.core/cancel-run! (:id run))))
nil)
+(api.macros/defendpoint :post "/:id/reset-checkpoint" :- :nil
+ "Reset the stored checkpoint for an incremental transform."
+ [{:keys [id]} :- [:map [:id ms/PositiveInt]]]
+ (api/write-check :model/Transform id)
+ (t2/update! :model/Transform id {:last_checkpoint_value nil})
+ nil)
+
(defn run-transform!
"Run a transform. Returns a 202 response with run_id.
The transform must already be fetched and validated."
@@ -328,49 +340,6 @@
[:id ms/PositiveInt]]]
(run-transform! (api/write-check :model/Transform id)))
-(defn- simple-native-query?
- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion.
- Delegates to sql-tools which dispatches to the configured parser backend (macaw or sqlglot)."
- [sql-string]
- (sql-tools/simple-query? sql-string))
-
-(api.macros/defendpoint :post "/is-simple-query" :- [:map
- [:is_simple :boolean]
- [:reason {:optional true} :string]]
- "Checks if a native SQL query string is simple enough for automatic checkpoint insertion"
- [_route-params
- _query-params
- {:keys [query]} :- [:map [:query string?]]]
- (api/check-superuser)
- (simple-native-query? query))
-
-(api.macros/defendpoint :post "/extract-columns"
- :- [:map [:columns [:maybe [:sequential :string]]]]
- "Extract column names suitable for incremental transform checkpoint filtering.
-
- This endpoint is specifically for populating the checkpoint column dropdown in
- incremental transforms. It only returns columns with types supported for checkpoint
- filtering: temporal (timestamp/tz) and numeric (int/float) types.
-
- Text, boolean, and other unsupported column types are filtered out.
-
- The query is compiled to native SQL using [[qp.compile/compile-with-inline-parameters]],
- which handles parameterized queries with template tags. Then extracts column names
- and types using PreparedStatement metadata.
-
- Returns a map with a :columns key containing a vector of column names (strings).
- If extraction fails, returns nil for :columns."
- [_route-params
- _query-params
- {:keys [query]} :- [:map
- [:query ::qp.schema/any-query]]]
- (api/check-superuser)
- (let [database-id (:database query)
- database (api/check-404 (t2/select-one :model/Database :id database-id))
- driver-name (driver/the-initialized-driver (:engine database))
- columns (transforms.core/extract-incremental-filter-columns-from-query driver-name database-id query)]
- {:columns columns}))
-
;;; -------------------------------------------------- Inspector API --------------------------------------------------
(api.macros/defendpoint :get "/:id/inspect"
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index fe841e17733..3240c6277c7 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2848,6 +2848,93 @@
:active true
:is_upload false} unrelated)))))))
+(deftest remove-legacy-incremental-strategies-test
+ (testing "v59.2026-03-13T00:00:00: migrate legacy checkpoint-filter to checkpoint-filter-field-id"
+ (impl/test-migrations ["v59.2026-03-13T00:00:00"] [migrate!]
+ (let [db-id (:id (new-instance-with-default :metabase_database))
+ table-id (:id (new-instance-with-default :metabase_table {:db_id db-id}))
+ field-id (t2/insert-returning-pk! :metabase_field
+ {:name "updated_at"
+ :table_id table-id
+ :base_type "type/DateTimeWithLocalTZ"
+ :database_type "timestamptz"
+ :active true
+ :created_at :%now
+ :updated_at :%now})
+ target (json/encode {:type "table" :schema "public" :name "out"})
+ insert-transform!
+ (fn [source]
+ (t2/insert-returning-pk!
+ :transform {:name (mt/random-name)
+ :source source
+ :target target
+ :source_type "mbql"
+ :source_database_id db-id
+ :created_at :%now
+ :updated_at :%now}))
+ ;; Transform with resolvable legacy checkpoint strategy
+ resolvable-source
+ (json/encode {:type "query"
+ :query {:stages [{:source-table table-id}]}
+ :source-incremental-strategy
+ {:type "checkpoint"
+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"
+ :checkpoint-filter {:some "filter"}}})
+ resolvable-id (insert-transform! resolvable-source)
+ ;; Transform with unresolvable legacy checkpoint strategy (no matching field)
+ unresolvable-source
+ (json/encode {:type "query"
+ :query {:stages [{:source-table table-id}]}
+ :source-incremental-strategy
+ {:type "checkpoint"
+ :checkpoint-filter-unique-key "column-unique-key-v1$nonexistent_column"
+ :checkpoint-filter {:some "filter"}}})
+ unresolvable-id (insert-transform! unresolvable-source)
+ ;; Transform without any incremental strategy — should be untouched
+ no-strategy-source
+ (json/encode {:type "query"
+ :query {:stages [{:source-table table-id}]}})
+ no-strategy-id (insert-transform! no-strategy-source)
+ ;; Python transform with resolvable checkpoint strategy
+ python-resolvable-source
+ (json/encode {:type "python"
+ :body "x=1"
+ :source-tables [{"alias" "t" "table_id" table-id}]
+ :source-incremental-strategy
+ {:type "checkpoint"
+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"
+ :checkpoint-filter {:some "filter"}}})
+ python-resolvable-id (insert-transform! python-resolvable-source)
+ ;; Native transform — can't resolve source table, strategy stripped
+ native-source
+ (json/encode {:type "native"
+ :source-incremental-strategy
+ {:type "checkpoint"
+ :checkpoint-filter-unique-key "column-unique-key-v1$updated_at"
+ :checkpoint-filter {:some "filter"}}})
+ native-id (insert-transform! native-source)
+ get-source (fn [id]
+ (json/decode
+ (t2/select-one-fn :source :transform :id id)
+ keyword))]
+ (migrate!)
+ (testing "Resolvable legacy strategy is migrated to checkpoint-filter-field-id"
+ (let [strategy (:source-incremental-strategy (get-source resolvable-id))]
+ (is (= field-id (:checkpoint-filter-field-id strategy)))
+ (is (not (contains? strategy :checkpoint-filter-unique-key)))
+ (is (not (contains? strategy :checkpoint-filter)))))
+ (testing "Unresolvable legacy strategy is stripped entirely"
+ (is (not (contains? (get-source unresolvable-id) :source-incremental-strategy))))
+ (testing "Transform without strategy is untouched"
+ (is (not (contains? (get-source no-strategy-id) :source-incremental-strategy))))
+ (testing "Python transform with single source-table resolves correctly"
+ (let [strategy (:source-incremental-strategy (get-source python-resolvable-id))]
+ (is (= field-id (:checkpoint-filter-field-id strategy)))
+ (is (not (contains? strategy :checkpoint-filter-unique-key)))
+ (is (not (contains? strategy :checkpoint-filter)))))
+ (testing "Native transform strategy is stripped (can't resolve source table)"
+ (is (not (contains? (get-source native-id) :source-incremental-strategy))))))))
+
(deftest unify-source-tables-format-test
(testing "v60.2026-03-03T12:00:00: convert source-tables from map to vec format"
(impl/test-migrations ["v60.2026-03-03T12:00:00"] [migrate!]
diff --git a/test/metabase/transforms/models_test.clj b/test/metabase/transforms/models_test.clj
index e0bc81bc168..2312a1a352b 100644
--- a/test/metabase/transforms/models_test.clj
+++ b/test/metabase/transforms/models_test.clj
@@ -325,3 +325,51 @@
(let [run (transform-run/start-run! transform-id {:run_method "manual"})]
(is (= transform-name (:transform_name run)))
(is (= entity-id (:transform_entity_id run))))))))
+
+(deftest checkpoint-reset-on-filter-field-change-test
+ (testing "Changing checkpoint-filter-field-id resets stored checkpoint"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temp [:model/Database {db-id :id} {}
+ :model/Transform {transform-id :id}
+ {:source {:type "query"
+ :query {:database db-id
+ :type "native"
+ :native {:query "SELECT 1"
+ :template-tags {}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id 100}}
+ :target {:type "table-incremental"
+ :schema "public"
+ :name "test_incr"
+ :db_id db-id}
+ :last_checkpoint_value "42"}]
+ (testing "checkpoint is present before update"
+ (let [t (t2/select-one :model/Transform transform-id)]
+ (is (= "42" (:last_checkpoint_value t)))))
+
+ (testing "changing checkpoint-filter-field-id resets checkpoint"
+ (t2/update! :model/Transform transform-id
+ {:source {:type "query"
+ :query {:database db-id
+ :type "native"
+ :native {:query "SELECT 1"
+ :template-tags {}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id 200}}})
+ (let [t (t2/select-one :model/Transform transform-id)]
+ (is (nil? (:last_checkpoint_value t)))))
+
+ (testing "updating without changing checkpoint-filter-field-id preserves checkpoint"
+ ;; Set checkpoint again
+ (t2/update! :model/Transform transform-id
+ {:last_checkpoint_value "99"})
+ (t2/update! :model/Transform transform-id
+ {:source {:type "query"
+ :query {:database db-id
+ :type "native"
+ :native {:query "SELECT 2"
+ :template-tags {}}}
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id 200}}})
+ (let [t (t2/select-one :model/Transform transform-id)]
+ (is (= "99" (:last_checkpoint_value t)))))))))
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index e89f0529809..45ae3057eb2 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -363,9 +363,8 @@
;; so it sets transform_id on the target table.
(with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})
transforms-base.u/sync-target! (constantly nil)
- transforms-base.u/execute-secondary-index-ddl-if-required! (constantly nil)
- transforms.u/run-cancelable-transform! (fn [_run-id _driver _details run-fn & _opts]
- (run-fn (a/promise-chan)))]
+ transforms.u/run-cancelable-transform! (fn [_run-id _transform _driver _details run-fn & _opts]
+ (run-fn (a/promise-chan) nil))]
(transforms.execute/execute! transform {:run-method :manual})
(is (= transform-id
(t2/select-one-fn :transform_id :model/Table :id table-id))))))))))
@@ -399,7 +398,7 @@
(mt/with-premium-features #{:transforms-basic}
(let [transform {:source {:type "query"
:query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
- {:keys [query]} (transforms-base.u/compile-source transform)]
+ {:keys [query]} (transforms-base.u/compile-source transform (transforms-base.u/get-source-range-params transform))]
(is (string? query))
(is (not (re-find #"(?i)\bLIMIT\b" query))
(str "Expected no LIMIT clause in compiled SQL, got: " query)))))))
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 0c1a1949f09..63c6f839090 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -3,7 +3,6 @@
(:require
[clojure.test :refer :all]
[medley.core :as m]
- [metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.models.transforms.transform :as transform.model]
@@ -14,7 +13,6 @@
[metabase.test :as mt]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-rest.api.transform]
- [metabase.transforms.crud]
[metabase.transforms.query-test-util :as query-test-util]
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :refer [get-test-schema
@@ -1189,94 +1187,6 @@
(mt/user-http-request :lucky :get 200 "transform")
(mt/user-http-request :lucky :get 200 (str "transform/" (:id transform)))))))))
-(defmethod driver/database-supports? [::driver/driver ::extract-columns-from-query]
- [_driver _feature _database]
- true)
-
-(doseq [driver [:clickhouse :redshift :bigquery-cloud-sdk :snowflake]]
- (defmethod driver/database-supports? [driver ::extract-columns-from-query]
- [_driver _feature _database]
- false))
-
-(deftest ^:parallel extract-columns-from-query-test
- (mt/with-premium-features #{}
- (testing "POST /api/transform/extract-columns"
- (mt/test-drivers (mt/normal-driver-select {:+features [:transforms/table
- ::extract-columns-from-query]})
- (mt/dataset transforms-dataset/transforms-test
- (letfn [(make-native-query [sql]
- {:lib/type :mbql/query
- :database (mt/id)
- :stages [{:lib/type :mbql.stage/native
- :native sql}]})]
- (testing "Successfully extracts columns from a simple SELECT query"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"
- {:query (make-native-query "SELECT id, name, category, price FROM transforms_products")})]
- (is (= ["id" "price"] (:columns response))
- "Should only return numeric (id, price) columns, filtering out text columns (name, category)")))
- (testing "Returns nil for invalid SQL"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"
- {:query (make-native-query "SELECT * FORM invalid_table")})]
- (is (nil? (:columns response)))))
- (testing "Extracts columns from query with aliases"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"
- {:query (make-native-query "SELECT id AS product_id, name AS product_name FROM transforms_products")})]
- (is (= ["product_id"] (:columns response))
- "Should only return numeric column (id), filtering out text column (name)")))
- (testing "Filters columns by type - only returns numeric and temporal columns"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/extract-columns"
- {:query (make-native-query "SELECT id, name, category, price, created_at FROM transforms_products")})]
- (is (= ["id" "price" "created_at"] (:columns response))
- "Should return numeric (id, price) and temporal (created_at) columns, filtering out text columns (name, category)")))
- (testing "Requires superuser permissions"
- (is (= "You don't have permissions to do that."
- (mt/user-http-request :rasta :post 403 "transform/extract-columns"
- {:query (make-native-query "SELECT * FROM transforms_products")}))))
- (testing "Returns 404 for non-existent database"
- (is (= "Not found."
- (mt/user-http-request :crowberto :post 404 "transform/extract-columns"
- {:query {:lib/type :mbql/query
- :database 999999
- :stages [{:lib/type :mbql.stage/native
- :native "SELECT * FROM transforms_products"}]}}))))))))))
-
-(deftest ^:parallel is-simple-query-test
- (mt/with-premium-features #{}
- (testing "POST /api/transform/is-simple-query"
- (testing "Returns true for simple SELECT queries"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT id, name FROM products"})]
- (is (true? (:is_simple response)))
- (is (nil? (:reason response)))))
- (testing "Returns true for simple SELECT with WHERE clause"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT id, name FROM products WHERE category = 'Electronics'"})]
- (is (true? (:is_simple response)))))
- (testing "Returns true for simple SELECT with JOIN"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT p.id, p.name, c.name FROM products p JOIN categories c ON p.category_id = c.id"})]
- (is (true? (:is_simple response)))))
- (testing "Returns false for query with LIMIT"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT id, name FROM products LIMIT 10"})]
- (is (false? (:is_simple response)))
- (is (= "Contains a LIMIT" (:reason response)))))
- (testing "Returns false for query with OFFSET"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT id, name FROM products OFFSET 5"})]
- (is (false? (:is_simple response)))
- (is (= "Contains an OFFSET" (:reason response)))))
- (testing "Returns false for query with LIMIT and OFFSET"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "SELECT id, name FROM products LIMIT 10 OFFSET 5"})]
- (is (false? (:is_simple response)))
- (is (= "Contains a LIMIT" (:reason response)))))
- (testing "Returns false for query with CTE"
- (let [response (mt/user-http-request :crowberto :post 200 "transform/is-simple-query"
- {:query "WITH category_counts AS (SELECT category, COUNT(*) as cnt FROM products GROUP BY category) SELECT * FROM category_counts"})]
- (is (false? (:is_simple response)))
- (is (= "Contains a CTE" (:reason response))))))))
-
;;; ------------------------------------------------------------
;;; User Attribution Tests
;;; ------------------------------------------------------------
@@ -1640,12 +1550,13 @@
(is (= ["collection"] (:here child-coll)))
(is (= ["transform"] (:below child-coll)))))))))
-(deftest native-incremental-column-type-validated-on-create-test
+(deftest incremental-column-type-validated-on-create-test
(mt/with-premium-features #{}
- (testing "POST /api/transform column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (testing "POST /api/transform column type validation with checkpoint-filter-field-id"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
+ (let [schema (get-test-schema)
+ name-field-id (t2/select-one-pk :model/Field :name "name" :table_id (mt/id :transforms_products))]
(testing "Rejects unsupported checkpoint column type (text)"
(let [response (mt/user-http-request :crowberto :post 400 "transform"
{:name "Invalid Incremental Transform"
@@ -1653,20 +1564,21 @@
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category, price FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "name"}}
+ :checkpoint-filter-field-id name-field-id}}
:target {:type "table-incremental"
:schema schema
:name "invalid_incremental"
:target-incremental-strategy {:type "append"}}})]
(is (string? response))
- (is (re-find #"Only numeric and temporal" response))))))))))
+ (is (re-find #"unsupported type" response))))))))))
-(deftest native-incremental-column-type-validated-on-update-test
+(deftest incremental-column-type-validated-on-update-test
(mt/with-premium-features #{}
- (testing "PUT /api/transform column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
+ (testing "PUT /api/transform column type validation with checkpoint-filter-field-id"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
(mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
+ (let [schema (get-test-schema)
+ category-field-id (t2/select-one-pk :model/Field :name "category" :table_id (mt/id :transforms_products))]
(with-transform-cleanup! [table-name "update_incremental_test"]
(let [;; Create a non-incremental transform first
created (mt/user-http-request :crowberto :post 200 "transform"
@@ -1684,60 +1596,13 @@
:query (lib/native-query (mt/metadata-provider)
"SELECT id, name, category FROM transforms_products")
:source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter "category"}}
+ :checkpoint-filter-field-id category-field-id}}
:target {:type "table-incremental"
:schema schema
:name table-name
:target-incremental-strategy {:type "append"}}})]
(is (string? response))
- (is (re-find #"Only numeric and temporal" response))))))))))))
-
-(deftest mbql-incremental-column-type-validated-on-create-test
- (mt/with-premium-features #{}
- (testing "MBQL query with checkpoint-filter-unique-key - checkpoint column type validation"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (testing "Rejects unsupported checkpoint column type (text)"
- (let [query (mt/mbql-query transforms_products)
- response (mt/user-http-request :crowberto :post 400 "transform"
- {:name "Invalid MBQL Incremental"
- :source {:type "query"
- :query query
- :source-incremental-strategy {:type "checkpoint"
- :checkpoint-filter-unique-key "column-unique-key-v1$name"}}
- :target {:type "table-incremental"
- :schema schema
- :name "invalid_mbql_incremental"
- :target-incremental-strategy {:type "append"}}})]
- (is (string? response))
- (is (re-find #"not supported" response))))))))))
-
-(deftest native-incremental-column-validation-when-not-extractable-test
- (mt/with-premium-features #{}
- (testing "Native query checkpoint column validation with text input fallback"
- (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table ::extract-columns-from-query)
- (mt/dataset transforms-dataset/transforms-test
- (let [schema (get-test-schema)]
- (testing "Accepts any column if they were not extractable"
- (with-transform-cleanup! [table-name "fallback_test_unextracted"]
- (with-redefs [metabase.transforms.crud/extract-all-columns-from-query
- ;; simulate lack of driver support for extraction
- (fn [_driver _database-id _query] nil)]
- (let [response (mt/user-http-request :crowberto :post 200 "transform"
- {:name "Unextracted Column - Text Input Fallback"
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT id, name, created_at FROM transforms_products")
- :source-incremental-strategy {:type "checkpoint"
- ;; created_at is in the query but not in our stubbed extraction
- :checkpoint-filter "created_at"}}
- :target {:type "table-incremental"
- :schema schema
- :name table-name
- :target-incremental-strategy {:type "append"}}})]
- (is (some? (:id response))
- "Should accept column not in extracted metadata, allowing text input fallback")))))))))))
+ (is (re-find #"unsupported type" response))))))))))))
(deftest search-filters-transform-source-types-test
(mt/with-premium-features #{}