Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(studio): validate pipeline wizard data settings (#51821)

Danny White25ede7243e07
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.test.tsx b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.test.tsx
index 9b0bf1c520d..6efdae01f45 100644
--- a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.test.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.test.tsx
@@ -6,6 +6,7 @@ import { NuqsAdapter } from 'nuqs/adapters/react'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { PipelineCreationFlow } from './PipelineCreationFlow'
+import type { ReplicationCostEstimateData } from '@/data/replication/cost-estimate-query'
import { customRender } from '@/tests/lib/custom-render'
import { addAPIMock } from '@/tests/lib/msw'
@@ -28,6 +29,17 @@ vi.mock('@/components/layouts/AppLayout/ProjectDropdown', () => ({
mockAnimationsApi()
beforeEach(() => {
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/:ref/sources/:source_id/publications/:publication_name/cost-estimate',
+ response: () =>
+ HttpResponse.json<ReplicationCostEstimateData>({
+ currency: 'usd',
+ pipeline: { hourly_cost: 0.05, monthly_cost: 36.5 },
+ streaming: { rate_per_gb: 3 },
+ table_copy: { rate_per_gb: 0.6, total_bytes: 0, total_cost: 0, tables: [] },
+ }),
+ })
addAPIMock({
method: 'post',
path: '/platform/replication/:ref/destinations/validate',
@@ -56,7 +68,22 @@ beforeEach(() => {
path: '/platform/replication/v2/:ref/sources/:source_id/publications',
response: () =>
HttpResponse.json<components['schemas']['ReadPublicationsResponse_Output']>({
- publications: [],
+ publications: [{ name: 'analytics' }],
+ }),
+ })
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/v2/:ref/sources/:source_id/publications/:publication_name',
+ response: () =>
+ HttpResponse.json<components['schemas']['PublicationDetailsResponse_Output']>({
+ name: 'analytics',
+ config: {
+ type: 'tables',
+ tables: [],
+ operations: ['insert'],
+ publish_via_partition_root: false,
+ },
+ tables: [],
}),
})
})
@@ -75,10 +102,39 @@ const continueToData = async () => {
next()
await screen.findByText(/Step 3 of 4/)
}
+const selectPublication = async () => {
+ fireEvent.click(screen.getByText('Select publication'))
+ fireEvent.click(await screen.findByText('analytics'))
+}
const back = () => fireEvent.click(screen.getByRole('button', { name: 'Back' }))
describe('PipelineCreationFlow', () => {
- it('requires valid fields, recovers from request and critical failures, and reviews warnings before continuing', async () => {
+ it('requires connection fields and reveals invalid advanced settings', async () => {
+ customRender(<PipelineCreationFlow availableDestinations={['BigQuery']} onClose={vi.fn()} />, {
+ nuqs: { hasMemory: true },
+ })
+ next()
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ fireEvent.change(screen.getByRole('spinbutton', { name: 'Connection pool size' }), {
+ target: { value: '-1' },
+ })
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ fireEvent.click(screen.getByRole('button', { name: 'Continue' }))
+ expect((await screen.findAllByText(/Project ID is required\./)).length).toBeGreaterThan(0)
+ expect(screen.getByRole('spinbutton', { name: 'Connection pool size' })).toBeVisible()
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ expect(
+ screen.queryByRole('spinbutton', { name: 'Connection pool size' })
+ ).not.toBeInTheDocument()
+ fireEvent.click(screen.getByRole('button', { name: 'Continue' }))
+ expect(screen.getByRole('spinbutton', { name: 'Connection pool size' })).toBeVisible()
+ fireEvent.change(screen.getByRole('spinbutton', { name: 'Connection pool size' }), {
+ target: { value: '5' },
+ })
+ expect(screen.queryByText('Check connection settings')).not.toBeInTheDocument()
+ })
+
+ it('recovers from request and critical failures and reviews warnings before continuing', async () => {
let attempt = 0
const requests: unknown[] = []
addAPIMock({
@@ -108,25 +164,6 @@ describe('PipelineCreationFlow', () => {
}
)
next()
- fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
- fireEvent.change(screen.getByRole('spinbutton', { name: 'Connection pool size' }), {
- target: { value: '-1' },
- })
- fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
- fireEvent.click(screen.getByRole('button', { name: 'Continue' }))
- expect((await screen.findAllByText(/Project ID is required\./)).length).toBeGreaterThan(0)
- expect(requests).toHaveLength(0)
- expect(screen.getByRole('spinbutton', { name: 'Connection pool size' })).toBeVisible()
- fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
- expect(
- screen.queryByRole('spinbutton', { name: 'Connection pool size' })
- ).not.toBeInTheDocument()
- fireEvent.click(screen.getByRole('button', { name: 'Continue' }))
- expect(screen.getByRole('spinbutton', { name: 'Connection pool size' })).toBeVisible()
- fireEvent.change(screen.getByRole('spinbutton', { name: 'Connection pool size' }), {
- target: { value: '5' },
- })
- expect(screen.queryByText('Check connection settings')).not.toBeInTheDocument()
fillConnection()
fireEvent.click(screen.getByRole('button', { name: 'Continue' }))
expect(screen.getByRole('button', { name: 'Back' })).toBeDisabled()
@@ -159,8 +196,28 @@ describe('PipelineCreationFlow', () => {
expect(screen.getByText(/Step 2 of 4/)).toBeInTheDocument()
expect(requests[2]).not.toHaveProperty('pipeline_config')
expect(requests[2]).not.toHaveProperty('source_id')
+ })
+
+ it('invalidates connection verification after edits and ignores stale responses', async () => {
+ addAPIMock({
+ method: 'post',
+ path: '/platform/replication/:ref/destinations/validate',
+ response: async () => {
+ await delay(10)
+ return HttpResponse.json<components['schemas']['ValidateDestinationResponse_Output']>({
+ validation_failures: [
+ { name: 'Credentials', reason: 'Check credentials', failure_type: 'warning' },
+ ],
+ })
+ },
+ })
+ customRender(<PipelineCreationFlow availableDestinations={['BigQuery']} onClose={vi.fn()} />, {
+ nuqs: { hasMemory: true },
+ })
next()
- back()
+ fillConnection()
+ next()
+ await screen.findByText(/Replication can start/)
fireEvent.change(screen.getByRole('textbox', { name: 'Project ID' }), {
target: { value: 'changed' },
})
@@ -273,12 +330,109 @@ describe('PipelineCreationFlow', () => {
await continueToData()
fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
expect(screen.getByRole('spinbutton', { name: 'Batch wait time' })).toHaveValue(750)
+ await selectPublication()
next()
- expect(screen.getByText(/Step [1-4] of 4/)).toHaveTextContent('Step 4 of 4')
+ await waitFor(() =>
+ expect(screen.getByText(/Step [1-4] of 4/)).toHaveTextContent('Step 4 of 4')
+ )
expect(screen.getByRole('button', { name: 'Start pipeline' })).toBeDisabled()
expect(screen.getByDisplayValue('Warehouse')).toBeInTheDocument()
})
+ it('blocks incomplete data, retries metadata failures, and reveals invalid advanced settings', async () => {
+ const onUrlUpdate = vi.fn()
+ customRender(<PipelineCreationFlow availableDestinations={['BigQuery']} onClose={vi.fn()} />, {
+ nuqs: { hasMemory: true, rateLimitFactor: 1, onUrlUpdate },
+ })
+ next()
+ fillConnection()
+ await continueToData()
+ next()
+ expect(await screen.findByText('Select an available publication.')).toBeInTheDocument()
+ await selectPublication()
+ await waitFor(() =>
+ expect(screen.queryByText('Select an available publication.')).not.toBeInTheDocument()
+ )
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ fireEvent.change(screen.getByRole('spinbutton', { name: 'Batch wait time' }), {
+ target: { value: '-1' },
+ })
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ next()
+ await waitFor(() =>
+ expect(screen.getByRole('spinbutton', { name: 'Batch wait time' })).toBeVisible()
+ )
+ expect(screen.getByText('Check data settings')).toBeInTheDocument()
+ fireEvent.change(screen.getByRole('spinbutton', { name: 'Batch wait time' }), {
+ target: { value: '750' },
+ })
+ expect(screen.queryByText('Check data settings')).not.toBeInTheDocument()
+ let shouldFail = true
+ addAPIMock({
+ method: 'get',
+ path: '/platform/replication/v2/:ref/sources/:source_id/publications',
+ response: () =>
+ shouldFail
+ ? HttpResponse.json<{ message: string }>({ message: 'Unavailable' }, { status: 500 })
+ : HttpResponse.json<components['schemas']['ReadPublicationsResponse_Output']>({
+ publications: [{ name: 'analytics' }],
+ }),
+ })
+ next()
+ expect(
+ await screen.findByText(
+ 'Publication tables could not be loaded. Click Continue to try again.'
+ )
+ ).toBeInTheDocument()
+ shouldFail = false
+ next()
+ await waitFor(() => expect(screen.getByText(/Step 4 of 4/)).toBeInTheDocument())
+ // Wait for the URL write, not just the optimistic step render, before going back.
+ await waitFor(() =>
+ expect(onUrlUpdate).toHaveBeenLastCalledWith(
+ expect.objectContaining({ queryString: '?step=review' })
+ )
+ )
+ await act(async () => back())
+ fireEvent.click(await screen.findByRole('button', { name: /Advanced settings/ }))
+ expect(screen.getByRole('spinbutton', { name: 'Batch wait time' })).toHaveValue(750)
+ })
+
+ it('blocks browser Forward to Review after data changes', async () => {
+ window.history.replaceState(null, '', '/project/default/database/pipelines/new')
+ customRender(
+ <NuqsAdapter>
+ <PipelineCreationFlow availableDestinations={['BigQuery']} onClose={vi.fn()} />
+ </NuqsAdapter>
+ )
+ next()
+ await waitFor(() => expect(window.location.search).toBe('?step=connection'))
+ fillConnection()
+ await continueToData()
+ await screen.findByText('Select publication')
+ await selectPublication()
+ next()
+ await waitFor(() => expect(window.location.search).toBe('?step=review'))
+ await act(async () => {
+ await new Promise<void>((resolve) => {
+ window.addEventListener('popstate', () => resolve(), { once: true })
+ window.history.back()
+ })
+ })
+ fireEvent.click(screen.getByRole('button', { name: /Advanced settings/ }))
+ fireEvent.change(screen.getByRole('spinbutton', { name: 'Batch wait time' }), {
+ target: { value: '-1' },
+ })
+ await act(async () => {
+ await new Promise<void>((resolve) => {
+ window.addEventListener('popstate', () => resolve(), { once: true })
+ window.history.forward()
+ })
+ })
+ await waitFor(() => expect(window.location.search).toBe('?step=data'))
+ expect(screen.getByRole('spinbutton', { name: 'Batch wait time' })).toHaveValue(-1)
+ })
+
it('confirms destination-only changes and treats a reverted selection as pristine', async () => {
const onClose = vi.fn()
customRender(
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.tsx b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.tsx
index c4739ca3934..f58c876e7c3 100644
--- a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineCreationFlow.tsx
@@ -1,8 +1,9 @@
import { useParams } from 'common'
import { parseAsStringEnum, useQueryState } from 'nuqs'
import { useEffect, useId, useRef, useState } from 'react'
-import { useForm, useWatch } from 'react-hook-form'
+import { useForm } from 'react-hook-form'
import { Form, Separator } from 'ui'
+import { Admonition } from 'ui-patterns/Admonition'
import { DestinationLogo } from '../DestinationLogo'
import { BigQueryFields } from '../DestinationPanel/DestinationForm/BigQuery/Fields'
@@ -21,9 +22,9 @@ import { PipelineDestinationSelection } from './PipelineDestinationSelection'
import { PipelineReviewCosts } from './PipelineReviewCosts'
import { PipelineValidationAdmonition } from './PipelineValidationAdmonition'
import { usePipelineConnectionValidation } from './usePipelineConnectionValidation'
+import { usePipelineDataValidation } from './usePipelineDataValidation'
import { DiscardChangesConfirmationDialog } from '@/components/ui-patterns/Dialogs/DiscardChangesConfirmationDialog'
import { SteppedFlow, SteppedFlowHeader } from '@/components/ui/SteppedFlow/SteppedFlow'
-import { useReplicationPublicationQuery } from '@/data/replication/publication-query'
import { useReplicationSourceId } from '@/data/replication/sources-query'
import { useConfirmOnClose } from '@/hooks/ui/useConfirmOnClose'
@@ -77,17 +78,17 @@ export const PipelineCreationFlow = ({
const { validateConnection, resetValidation, isValidating, isConnectionVerified, failures } =
usePipelineConnectionValidation({ projectRef, type, form })
const sourceId = useReplicationSourceId({ projectRef })
- const publicationName = useWatch({ control: form.control, name: 'publicationName' })
- const { data: publication } = useReplicationPublicationQuery({
- projectRef,
- sourceId,
- publicationName,
- })
- const accessibleStep =
- (step === 'data' || step === 'review') && !isConnectionVerified ? 'connection' : step
+ const { validateData, isValidatingData, isDataVerified, dataError, publication } =
+ usePipelineDataValidation({ projectRef, sourceId, type, form })
+ const accessibleStep = (() => {
+ if ((step === 'data' || step === 'review') && !isConnectionVerified) return 'connection'
+ if (step === 'review' && !isDataVerified) return 'data'
+ return step
+ })()
const handleNext = async () => {
if (accessibleStep === 'connection' && !isConnectionVerified && !(await validateConnection()))
return
+ if (accessibleStep === 'data' && !(await validateData())) return
const next = steps[steps.findIndex(({ id }) => id === accessibleStep) + 1]
if (next) void setStep(next.id)
}
@@ -131,8 +132,8 @@ export const PipelineCreationFlow = ({
currentStep={accessibleStep}
onStepChange={setStep}
onNext={handleNext}
- nextLoading={isValidating}
- navigationDisabled={isValidating}
+ nextLoading={isValidating || isValidatingData}
+ navigationDisabled={isValidating || isValidatingData}
nextLabel="Continue"
nextDisabled={accessibleStep === 'destination' && !canSelectDestination}
onCancel={handleClose}
@@ -167,11 +168,23 @@ export const PipelineCreationFlow = ({
</>
)}
{accessibleStep === 'data' && (
- <PipelineDataSection
- type={type}
- form={form}
- onSelectNewPublication={() => setIsPublicationPanelVisible(true)}
- />
+ <>
+ <PipelineDataSection
+ type={type}
+ form={form}
+ onSelectNewPublication={() => setIsPublicationPanelVisible(true)}
+ revealInvalidSettings
+ />
+ {dataError && (
+ <Admonition
+ type="destructive"
+ title="Check data settings"
+ className="mb-0 rounded-none border-x-0 border-t border-b-0 border-default"
+ >
+ {dataError}
+ </Admonition>
+ )}
+ </>
)}
{accessibleStep === 'review' && (
<>
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataSection.tsx b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataSection.tsx
index 934ec566f3f..5e991f73382 100644
--- a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataSection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataSection.tsx
@@ -10,10 +10,12 @@ export const PipelineDataSection = ({
type,
form,
onSelectNewPublication,
+ revealInvalidSettings = false,
}: {
type: DestinationType
form: UseFormReturn<DestinationPanelSchemaType>
onSelectNewPublication: () => void
+ revealInvalidSettings?: boolean
}) => (
<section aria-label="Data">
<div className="p-5 flex flex-col gap-y-6">
@@ -23,6 +25,11 @@ export const PipelineDataSection = ({
<TableCopySelection form={form} editMode={false} />
</div>
</div>
- <AdvancedSettings type={type} form={form} group="data" />
+ <AdvancedSettings
+ type={type}
+ form={form}
+ group="data"
+ revealInvalidSettings={revealInvalidSettings}
+ />
</section>
)
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.test.ts
new file mode 100644
index 00000000000..32a8236e696
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.test.ts
@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest'
+
+import { generateDefaultValues } from '../DestinationPanel/DestinationForm/DestinationForm.utils'
+import { getDataValidationIssues } from './PipelineDataValidation.utils'
+import type { ReplicationPublicationData } from '@/data/replication/publication-query'
+
+const data = {
+ ...generateDefaultValues({ catalogToken: '', editMode: false }),
+ publicationName: 'analytics',
+}
+const publication: ReplicationPublicationData = {
+ name: 'analytics',
+ config: { type: 'tables', tables: [], operations: ['insert'], publish_via_partition_root: false },
+ tables: [{ id: 1, name: 'orders', schema: 'public', kind: 'table', partition_parent_id: null }],
+}
+
+describe('getDataValidationIssues', () => {
+ it.each(['include_all_tables', 'skip_all_tables'] as const)(
+ 'accepts %s without a table selection',
+ (tableSyncCopyMode) => {
+ expect(
+ getDataValidationIssues({ ...data, tableSyncCopyMode }, 'BigQuery', publication)
+ ).toEqual([])
+ }
+ )
+ it.each(['include_tables', 'skip_tables'] as const)(
+ 'requires current tables for %s',
+ (tableSyncCopyMode) => {
+ expect(
+ getDataValidationIssues(
+ { ...data, tableSyncCopyMode, tableSyncCopyTableIds: ['1'] },
+ 'BigQuery',
+ publication
+ )
+ ).toEqual([])
+ for (const tableSyncCopyTableIds of [[], ['bad'], ['42']]) {
+ expect(
+ getDataValidationIssues(
+ { ...data, tableSyncCopyMode, tableSyncCopyTableIds },
+ 'BigQuery',
+ publication
+ )
+ ).toEqual([expect.objectContaining({ path: 'tableSyncCopyTableIds' })])
+ }
+ }
+ )
+ it('rejects unavailable or mismatched publications and invalid advanced settings', () => {
+ expect(getDataValidationIssues(data, 'BigQuery')).toEqual([
+ expect.objectContaining({ path: 'publicationName' }),
+ ])
+ expect(
+ getDataValidationIssues(data, 'BigQuery', { ...publication, name: 'other' })
+ ).toHaveLength(1)
+ for (const path of [
+ 'maxFillMs',
+ 'maxTableSyncWorkers',
+ 'maxCopyConnectionsPerTable',
+ ] as const) {
+ expect(getDataValidationIssues({ ...data, [path]: -1 }, 'Snowflake', publication)).toEqual([
+ expect.objectContaining({ path }),
+ ])
+ }
+ })
+ it('validates BigQuery table layouts only for BigQuery', () => {
+ const invalidLayout = {
+ ...data,
+ tableOptions: [{ tableId: 1, partitionBy: { kind: 'time_column' as const, column: '' } }],
+ }
+ expect(getDataValidationIssues(invalidLayout, 'BigQuery', publication)).toEqual([
+ expect.objectContaining({ path: 'tableOptions.0.partitionBy.column' }),
+ ])
+ expect(getDataValidationIssues(invalidLayout, 'Snowflake', publication)).toEqual([])
+ })
+})
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.ts b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.ts
new file mode 100644
index 00000000000..2d58b0736ad
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/PipelineDataValidation.utils.ts
@@ -0,0 +1,68 @@
+import type { FieldPath } from 'react-hook-form'
+
+import {
+ DestinationPanelFormSchema,
+ type DestinationPanelSchemaType,
+} from '../DestinationPanel/DestinationForm/DestinationForm.schema'
+import { buildTableSyncCopyConfig } from '../DestinationPanel/DestinationForm/DestinationForm.utils'
+import type { ReplicationPublicationData } from '@/data/replication/publication-query'
+
+const dataSchema = DestinationPanelFormSchema.pick({
+ publicationName: true,
+ tableSyncCopyMode: true,
+ tableSyncCopyTableIds: true,
+ maxFillMs: true,
+ maxTableSyncWorkers: true,
+ maxCopyConnectionsPerTable: true,
+ invalidatedSlotBehavior: true,
+})
+export const dataFieldNames = [...Object.keys(dataSchema.shape), 'tableOptions'].filter(
+ (name): name is keyof DestinationPanelSchemaType => name in DestinationPanelFormSchema.shape
+)
+
+export const getDataValidationIssues = (
+ data: DestinationPanelSchemaType,
+ type: string,
+ publication?: ReplicationPublicationData
+) => {
+ const schema =
+ type === 'BigQuery'
+ ? dataSchema.extend({ tableOptions: DestinationPanelFormSchema.shape.tableOptions })
+ : dataSchema
+ const result = schema.safeParse(data)
+ const issues = result.success
+ ? []
+ : result.error.issues.map((issue) => ({
+ path: issue.path.join('.'),
+ message: issue.message,
+ }))
+ if (!publication || publication.name !== data.publicationName) {
+ issues.push({ path: 'publicationName', message: 'Select an available publication.' })
+ } else if (
+ data.tableSyncCopyMode === 'include_tables' ||
+ data.tableSyncCopyMode === 'skip_tables'
+ ) {
+ try {
+ buildTableSyncCopyConfig({
+ mode: data.tableSyncCopyMode,
+ selectedTableIds: data.tableSyncCopyTableIds,
+ })
+ const ids = new Set(publication.tables.map(({ id }) => String(id)))
+ if (data.tableSyncCopyTableIds.some((id) => !ids.has(id))) {
+ issues.push({
+ path: 'tableSyncCopyTableIds',
+ message: 'Some selected tables are no longer in the publication. Update your selection.',
+ })
+ }
+ } catch (error) {
+ issues.push({
+ path: 'tableSyncCopyTableIds',
+ message: error instanceof Error ? error.message : 'Select initial sync tables.',
+ })
+ }
+ }
+ return issues.filter(
+ (issue): issue is { path: FieldPath<DestinationPanelSchemaType>; message: string } =>
+ issue.path.split('.')[0] in DestinationPanelFormSchema.shape
+ )
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/CreatePipeline/usePipelineDataValidation.ts b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/usePipelineDataValidation.ts
new file mode 100644
index 00000000000..e40fa4cf6d7
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/CreatePipeline/usePipelineDataValidation.ts
@@ -0,0 +1,86 @@
+import { useRef, useState } from 'react'
+import { useWatch, type UseFormReturn } from 'react-hook-form'
+
+import type { DestinationPanelSchemaType } from '../DestinationPanel/DestinationForm/DestinationForm.schema'
+import { dataFieldNames, getDataValidationIssues } from './PipelineDataValidation.utils'
+import { useReplicationPublicationNamesQuery } from '@/data/replication/publication-names-query'
+import { useReplicationPublicationQuery } from '@/data/replication/publication-query'
+
+export const usePipelineDataValidation = ({
+ projectRef,
+ sourceId,
+ type,
+ form,
+}: {
+ projectRef?: string
+ sourceId?: number
+ type: string
+ form: UseFormReturn<DestinationPanelSchemaType>
+}) => {
+ const values = useWatch({ control: form.control, name: dataFieldNames })
+ const publicationName = useWatch({ control: form.control, name: 'publicationName' })
+ const snapshot = JSON.stringify([type, ...values])
+ const currentSnapshot = useRef(snapshot)
+ currentSnapshot.current = snapshot
+ const [verifiedSnapshot, setVerifiedSnapshot] = useState<string>()
+ const [isValidatingData, setIsValidatingData] = useState(false)
+ const isValidatingRef = useRef(false)
+ const [dataErrorState, setDataErrorState] = useState<{ snapshot: string; message: string }>()
+ const setDataError = (message?: string) =>
+ setDataErrorState(message ? { snapshot, message } : undefined)
+ const dataError = dataErrorState?.snapshot === snapshot ? dataErrorState.message : undefined
+ const names = useReplicationPublicationNamesQuery({ projectRef, sourceId })
+ const publication = useReplicationPublicationQuery({ projectRef, sourceId, publicationName })
+ const isDataVerified =
+ verifiedSnapshot === JSON.stringify([snapshot, publication.data]) &&
+ names.isSuccess &&
+ publication.isSuccess &&
+ names.data.some(({ name }) => name === publicationName) &&
+ publication.data.name === publicationName
+
+ const validateData = async () => {
+ if (isValidatingRef.current) return false
+ form.clearErrors(dataFieldNames)
+ setDataError(undefined)
+ setVerifiedSnapshot(undefined)
+ isValidatingRef.current = true
+ setIsValidatingData(true)
+ try {
+ if (!projectRef || sourceId === undefined || !publicationName) {
+ form.setError('publicationName', { message: 'Select an available publication.' })
+ return false
+ }
+ const [freshNames, freshPublication] = await Promise.all([
+ names.refetch(),
+ publication.refetch(),
+ ])
+ if (currentSnapshot.current !== snapshot) return false
+ if (freshNames.isError || freshPublication.isError) {
+ setDataError('Publication tables could not be loaded. Click Continue to try again.')
+ return false
+ }
+ const selected = freshNames.data?.some(({ name }) => name === publicationName)
+ ? freshPublication.data
+ : undefined
+ const issues = getDataValidationIssues(form.getValues(), type, selected)
+ issues.forEach(({ path, message }) => form.setError(path, { type: 'validate', message }))
+ if (issues.length > 0) {
+ setDataError(issues[0].message)
+ form.setFocus(issues[0].path)
+ return false
+ }
+ setVerifiedSnapshot(JSON.stringify([snapshot, freshPublication.data]))
+ return true
+ } finally {
+ isValidatingRef.current = false
+ setIsValidatingData(false)
+ }
+ }
+ return {
+ validateData,
+ isValidatingData,
+ isDataVerified,
+ dataError,
+ publication: publication.data,
+ }
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
index 299a46a1c92..014de9941d5 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
@@ -23,7 +23,15 @@ export const AdvancedSettings = ({
}) => {
const [expanded, setExpanded] = useState('')
const { errors } = useFormState({ control: form.control })
+ const dataError =
+ group !== 'connection' &&
+ (errors.maxFillMs ||
+ errors.maxTableSyncWorkers ||
+ errors.maxCopyConnectionsPerTable ||
+ errors.invalidatedSlotBehavior ||
+ errors.tableOptions)
const advancedError =
+ dataError ||
(type === 'BigQuery' && (errors.connectionPoolSize || errors.maxStalenessMins)) ||
(type === 'DuckLake' && errors.ducklakePoolSize) ||
(type === 'Snowflake' && errors.snowflakeRole)
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
index d973afbc14b..4d6a2442bd7 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.test.ts
@@ -130,6 +130,13 @@ describe('DestinationForm.utils table copy selection', () => {
).toEqual({ type: mode, table_ids: [202, 101] })
})
+ it.each([
+ ['include_tables', 'Select at least one table for the initial sync.'],
+ ['skip_tables', 'Select at least one table to exclude from the initial sync.'],
+ ] as const)('explains the empty selection for %s', (mode, message) => {
+ expect(() => buildTableSyncCopyConfig({ mode, selectedTableIds: [] })).toThrow(message)
+ })
+
it('rejects an invalid selected table id', () => {
expect(() =>
buildTableSyncCopyConfig({
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
index 7a33f34f8d5..7a9bd37770d 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
@@ -171,7 +171,11 @@ export const buildTableSyncCopyConfig = ({
if (mode === 'include_all_tables' || mode === 'skip_all_tables') return { type: mode }
if (selectedTableIds.length === 0) {
- throw new Error('Select at least one table for the initial sync.')
+ throw new Error(
+ mode === 'skip_tables'
+ ? 'Select at least one table to exclude from the initial sync.'
+ : 'Select at least one table for the initial sync.'
+ )
}
const tableIds = selectedTableIds.map(Number)
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
index 6c9fcdd8df7..e0d653f8283 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
@@ -102,6 +102,7 @@ export const PublicationSelection = ({
})
}
field.onChange(value)
+ form.clearErrors('publicationName')
},
}}
sourceId={sourceId}