Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Gdgt 1774 incremental update field not displaying in transforms (#69740)

Egor Grushinbbf5aebd5aad
diff --git a/e2e/support/helpers/e2e-data-studio-helpers.ts b/e2e/support/helpers/e2e-data-studio-helpers.ts
index 007e8a33c8b..c08b5b588f4 100644
--- a/e2e/support/helpers/e2e-data-studio-helpers.ts
+++ b/e2e/support/helpers/e2e-data-studio-helpers.ts
@@ -1,7 +1,13 @@
-import type { MeasureId, SegmentId, TableId } from "metabase-types/api";
+import type {
+ MeasureId,
+ SegmentId,
+ TableId,
+ TransformId,
+} from "metabase-types/api";
import { codeMirrorHelpers } from "./e2e-codemirror-helpers";
import { popover } from "./e2e-ui-elements-helpers";
+
const { H } = cy;
const libraryPage = () => cy.findByTestId("library-page");
@@ -52,6 +58,8 @@ export const DataStudio = {
cy.visit("/data-studio/transforms");
DataStudio.Transforms.list().should("be.visible");
},
+ visitSettingsTab: (transformId: TransformId) =>
+ cy.visit(`/data-studio/transforms/${transformId}/settings`),
pythonResults: () => cy.findByTestId("python-results"),
enableTransformPage: () => cy.findByTestId("enable-transform-page"),
},
diff --git a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
index 0759a6e299f..0c6d92cc8ab 100644
--- a/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/reproductions.cy.spec.ts
@@ -1,7 +1,12 @@
+import { WRITABLE_DB_ID } from "e2e/support/cypress_data";
import { createMockSearchResult } from "metabase-types/api/mocks";
const { H } = cy;
+const SOURCE_TABLE = "Animals";
+const TARGET_TABLE = "transform_table";
+const TARGET_SCHEMA = "Schema A";
+
describe("issue #68378", () => {
beforeEach(() => {
H.restore("postgres-writable");
@@ -78,6 +83,50 @@ describe("issue GDGT-1776", () => {
});
});
+describe("issue GDGT-1774", () => {
+ beforeEach(() => {
+ H.restore("postgres-writable");
+ H.resetTestTable({ type: "postgres", table: "many_schemas" });
+ cy.signInAsAdmin();
+ H.activateToken("bleeding-edge");
+ H.resyncDatabase({ dbId: WRITABLE_DB_ID, tableName: SOURCE_TABLE });
+ });
+
+ it("should display field options in the incremental update field picker (GDGT-1774)", () => {
+ H.getTableId({ name: SOURCE_TABLE })
+ .then((tableId) =>
+ H.createTransform({
+ name: "Incremental MBQL transform",
+ source: {
+ type: "query",
+ query: {
+ database: WRITABLE_DB_ID,
+ type: "query",
+ query: { "source-table": tableId },
+ },
+ "source-incremental-strategy": { type: "checkpoint" },
+ },
+ target: {
+ type: "table-incremental",
+ database: WRITABLE_DB_ID,
+ name: TARGET_TABLE,
+ schema: TARGET_SCHEMA,
+ "target-incremental-strategy": { type: "append" },
+ },
+ }),
+ )
+ .then((res) => H.DataStudio.Transforms.visitSettingsTab(res.body.id));
+
+ cy.log("Field picker should be visible and have selectable options");
+ cy.findByLabelText("Field to check for new values")
+ .scrollIntoView()
+ .should("be.visible")
+ .click();
+
+ H.popover().findAllByRole("option").should("have.length.greaterThan", 0);
+ });
+});
+
function visitTransformListPage() {
return cy.visit("/data-studio/transforms");
}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
index a0f70664888..6450508e0d1 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/IncrementalTransformSettings.tsx
@@ -27,7 +27,7 @@ import type * as Lib from "metabase-lib";
import type { TransformSource } from "metabase-types/api";
import {
- KeysetColumnSelect,
+ MBQLKeysetColumnSelect,
PythonKeysetColumnSelect,
} from "./KeysetColumnSelect";
import { NativeQueryColumnSelect } from "./NativeQueryColumnSelect";
@@ -230,13 +230,14 @@ function SourceStrategyFields({
{values.sourceStrategy === "checkpoint" && (
<>
{type === "query" && query && (
- <KeysetColumnSelect
+ <MBQLKeysetColumnSelect
name="checkpointFilterUniqueKey"
label={t`Field to check for new values`}
placeholder={t`Pick a field`}
description={t`Pick the field that we should scan to determine which records are new or changed`}
descriptionProps={{ lh: "1rem" }}
query={query}
+ source={source}
disabled={readOnly}
/>
)}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
index 4891a4cc119..5be50529d22 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/KeysetColumnSelect.tsx
@@ -1,10 +1,11 @@
import { useMemo } from "react";
import { FormSelect } from "metabase/forms";
-import type {
- DataAttributes,
- InputDescriptionProps,
- SelectOption,
+import {
+ type DataAttributes,
+ type InputDescriptionProps,
+ Loader,
+ type SelectOption,
} from "metabase/ui";
import * as Lib from "metabase-lib";
@@ -16,6 +17,7 @@ type KeysetColumnSelectProps = {
descriptionProps?: InputDescriptionProps & DataAttributes;
query: Lib.Query | null;
disabled?: boolean;
+ isLoading?: boolean;
};
export function KeysetColumnSelect({
@@ -26,6 +28,7 @@ export function KeysetColumnSelect({
descriptionProps,
query,
disabled,
+ isLoading,
}: KeysetColumnSelectProps) {
const columnOptions = useMemo((): Array<SelectOption> => {
if (!query) {
@@ -90,6 +93,7 @@ export function KeysetColumnSelect({
searchable
disabled={disabled || columnOptions.length === 0}
descriptionProps={descriptionProps}
+ rightSection={isLoading ? <Loader size="xs" /> : undefined}
/>
);
}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/MBQLKeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/MBQLKeysetColumnSelect.tsx
new file mode 100644
index 00000000000..43c84ff935f
--- /dev/null
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/MBQLKeysetColumnSelect.tsx
@@ -0,0 +1,47 @@
+import { skipToken, useGetAdhocQueryMetadataQuery } from "metabase/api";
+import type { DataAttributes, InputDescriptionProps } from "metabase/ui";
+import type * as Lib from "metabase-lib";
+import type { TransformSource } from "metabase-types/api";
+
+import { KeysetColumnSelect } from "./KeysetColumnSelect";
+
+type MBQLKeysetColumnSelectProps = {
+ source: TransformSource;
+ name: string;
+ label: string;
+ placeholder: string;
+ description: React.ReactNode;
+ descriptionProps?: InputDescriptionProps & DataAttributes;
+ query: Lib.Query;
+ disabled?: boolean;
+};
+
+export const MBQLKeysetColumnSelect = ({
+ source,
+ name,
+ label,
+ placeholder,
+ description,
+ descriptionProps,
+ query,
+ disabled,
+}: MBQLKeysetColumnSelectProps) => {
+ /**
+ * we need this metadata in order to get incremental fields to select
+ */
+ const { isLoading } = useGetAdhocQueryMetadataQuery(
+ source.type === "query" ? source.query : skipToken,
+ );
+ return (
+ <KeysetColumnSelect
+ name={name}
+ label={label}
+ placeholder={placeholder}
+ description={description}
+ query={query}
+ descriptionProps={descriptionProps}
+ disabled={disabled}
+ isLoading={isLoading}
+ />
+ );
+};
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
index e6502ad0ddf..066f14cc607 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/PythonKeysetColumnSelect.tsx
@@ -71,7 +71,8 @@ export function PythonKeysetColumnSelect({
description={description}
descriptionProps={descriptionProps}
query={query}
- disabled={disabled || isLoading || !!error}
+ disabled={disabled || !!error}
+ isLoading={isLoading}
/>
);
}
diff --git a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
index fda664b4009..e9c499b9ff8 100644
--- a/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
+++ b/frontend/src/metabase/transforms/components/IncrementalTransform/KeysetColumnSelect/index.ts
@@ -1,2 +1,3 @@
export { KeysetColumnSelect } from "./KeysetColumnSelect";
+export { MBQLKeysetColumnSelect } from "./MBQLKeysetColumnSelect";
export { PythonKeysetColumnSelect } from "./PythonKeysetColumnSelect";