The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
import { zodResolver } from '@hookform/resolvers/zod'
import { useParams } from 'common'
import { Loader2Icon } from 'lucide-react'
import { AnimatePresence, motion } from 'motion/react'
import { RefObject, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
import { useForm, useWatch } from 'react-hook-form'
import { toast } from 'sonner'
import { Button, DialogSectionSeparator, Form, SheetFooter, SheetSection } from 'ui'
import { Admonition } from 'ui-patterns/Admonition'
import * as z from 'zod'
import { PipelineConnectionSection } from '../../CreatePipeline/PipelineConnectionSection'
import { PipelineDataReview } from '../../CreatePipeline/PipelineDataReview'
import { PipelineDataSection } from '../../CreatePipeline/PipelineDataSection'
import { PipelineDestinationReview } from '../../CreatePipeline/PipelineDestinationReview'
import { PipelineReviewCosts } from '../../CreatePipeline/PipelineReviewCosts'
import { PipelineValidationAdmonition } from '../../CreatePipeline/PipelineValidationAdmonition'
import {
useIsETLBigQueryPrivateAlpha,
useIsETLClickHousePrivateAlpha,
useIsETLDucklakePrivateAlpha,
useIsETLIcebergPrivateAlpha,
useIsETLSnowflakePrivateAlpha,
} from '../../useIsETLPrivateAlpha'
import { type DestinationType, type ExistingDestination } from '../DestinationPanel.types'
import { AdvancedSettings } from './AdvancedSettings'
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
import { AnalyticsBucketFields } from './AnalyticsBucket/Fields'
import {
BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE,
getBigQueryValidationIssues,
} from './BigQuery/BigQuery.utils'
import { BigQueryFields } from './BigQuery/Fields'
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
import { ClickHouseFields } from './ClickHouse/Fields'
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
import {
areValidationFailuresEqual,
buildTableSyncCopyConfig,
generateDefaultValues,
pruneStaleSelectedTableIds,
pruneStaleTableOptions,
} from './DestinationForm.utils'
import { DestinationNameInput } from './DestinationNameInput'
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
import { DuckLakeFields } from './DuckLake/Fields'
import { NewPublicationPanel } from './NewPublicationPanel'
import { NoDestinationsAvailable } from './NoDestinationsAvailable'
import { PipelineCostDialog } from './PipelineCostDialog'
import { PipelineRegionField } from './PipelineRegionField'
import { PublicationSelection } from './PublicationSelection'
import { SnowflakeFields } from './Snowflake/Fields'
import {
getSnowflakeValidationIssues,
SNOWFLAKE_PRIVATE_KEY_FORMAT_MESSAGE,
} from './Snowflake/Snowflake.utils'
import { TableCopySelection } from './TableCopySelection'
import { useDestinationForm } from './useDestinationForm'
import { ValidationFailuresSection } from './ValidationFailuresSection'
import { ValidationWarningsDialog } from './ValidationWarningsDialog'
import { usePipelineCreationPreview } from '@/components/interfaces/App/FeaturePreview/FeaturePreviewContext'
import { CreateAnalyticsBucketSheet } from '@/components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
import { useAPIKeys } from '@/data/api-keys/api-keys-query'
import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query'
import { useReplicationDestinationByIdQuery } from '@/data/replication/destination-by-id-query'
import { useReplicationPipelineByIdQuery } from '@/data/replication/pipeline-by-id-query'
import { useReplicationPublicationNamesQuery } from '@/data/replication/publication-names-query'
import { useReplicationPublicationQuery } from '@/data/replication/publication-query'
import { useReplicationSourceId } from '@/data/replication/sources-query'
import { FGA_PERMISSIONS, useAsyncCheckPermissionsV2 } from '@/hooks/misc/useCheckPermissionsV2'
const formId = 'destination-editor'
interface DestinationFormProps {
selectedType: DestinationType
visible: boolean
existingDestination?: ExistingDestination
typeSelection?: ReactNode
checkIsDirtyRef?: RefObject<() => boolean>
onClose: () => void
onCancel?: () => void
}
export const DestinationForm = ({
selectedType,
visible,
existingDestination,
typeSelection,
checkIsDirtyRef,
onClose,
onCancel = onClose,
}: DestinationFormProps) => {
const { ref: projectRef } = useParams()
const etlEnableBigQuery = useIsETLBigQueryPrivateAlpha()
const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
const etlEnableDucklake = useIsETLDucklakePrivateAlpha()
const etlEnableSnowflake = useIsETLSnowflakePrivateAlpha()
const etlEnableClickHouse = useIsETLClickHousePrivateAlpha()
const { can: canReadAPIKeys } = useAsyncCheckPermissionsV2([
FGA_PERMISSIONS.PROJECT.API_GATEWAY_KEYS_READ,
// api keys reveal = true will require this perms
FGA_PERMISSIONS.PROJECT.API_GATEWAY_KEYS_SECRET_READ,
])
const [showValidationWarningsDialog, setShowValidationWarningsDialog] = useState(false)
const [showCostDialog, setShowCostDialog] = useState(false)
const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)
const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
null
)
const validationSectionRef = useRef<HTMLDivElement>(null)
const editMode = !!existingDestination
const { isEnabled: isCreationPreviewEnabled } = usePipelineCreationPreview()
const shouldShowConnectionSection = isCreationPreviewEnabled && !editMode
// Compute available destinations based on feature flags
const availableDestinations = useMemo(() => {
const destinations = []
if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })
if (etlEnableIceberg)
destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })
if (etlEnableDucklake) destinations.push({ value: 'DuckLake', label: 'DuckLake' })
if (etlEnableSnowflake) destinations.push({ value: 'Snowflake', label: 'Snowflake' })
if (etlEnableClickHouse) destinations.push({ value: 'ClickHouse', label: 'ClickHouse' })
return destinations
}, [
etlEnableBigQuery,
etlEnableDucklake,
etlEnableIceberg,
etlEnableSnowflake,
etlEnableClickHouse,
])
const hasNoAvailableDestinations = availableDestinations.length === 0
const sourceId = useReplicationSourceId({ projectRef })
const { data: publicationNames = [], isSuccess: isSuccessPublicationNames } =
useReplicationPublicationNamesQuery({ projectRef, sourceId })
const {
data: destinationData,
isError: isErrorDestination,
isSuccess: isSuccessDestination,
} = useReplicationDestinationByIdQuery({
projectRef,
destinationId: existingDestination?.destinationId,
})
const {
data: pipelineData,
isError: isErrorPipeline,
isSuccess: isSuccessPipeline,
} = useReplicationPipelineByIdQuery({
projectRef,
pipelineId: existingDestination?.pipelineId,
})
const isErrorExistingConfig = editMode && (isErrorDestination || isErrorPipeline)
const isExistingConfigReady = !editMode || (isSuccessDestination && isSuccessPipeline)
// Revealed API keys are only ever consumed as the default catalog token for
// Analytics Bucket (Iceberg) destinations, so don't fetch them for other
// destination types.
const { data: apiKeysData } = useAPIKeys(
{ projectRef, reveal: true },
{ enabled: canReadAPIKeys && selectedType === 'Analytics Bucket' && !editMode }
)
const { serviceKey } = apiKeysData ?? {}
const catalogToken = serviceKey?.api_key ?? ''
const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })
const {
isValidating,
validateConfiguration,
isSaving,
submitPipeline,
hasRunValidation,
destinationValidationFailures,
pipelineValidationFailures,
resetValidation,
} = useDestinationForm({ selectedType })
const defaultValues = useMemo(
() =>
generateDefaultValues({
destinationData,
pipelineData,
catalogToken,
region: projectSettings?.region,
editMode,
}),
[destinationData, pipelineData, catalogToken, projectSettings, editMode]
)
const form = useForm<z.infer<typeof FormSchema>>({
mode: 'onSubmit',
reValidateMode: 'onChange',
resolver: zodResolver(
FormSchema.superRefine((data, ctx) => {
const addRequiredFieldError = (path: string, message: string) => {
ctx.addIssue({
code: z.ZodIssueCode.custom,
message,
path: [path],
})
}
if (
(data.tableSyncCopyMode === 'include_tables' ||
data.tableSyncCopyMode === 'skip_tables') &&
data.tableSyncCopyTableIds.length === 0
) {
addRequiredFieldError('tableSyncCopyTableIds', 'Select at least one table')
}
if (selectedType === 'BigQuery') {
getBigQueryValidationIssues(data, {
secretsOptional: editMode,
validateJson: false,
}).forEach(({ path, message }) => {
addRequiredFieldError(path, message)
})
} else if (selectedType === 'Analytics Bucket') {
getAnalyticsBucketValidationIssues(data, {
secretsOptional: editMode,
storedS3AccessKeyId: editMode ? defaultValues.s3AccessKeyId : undefined,
}).forEach(({ path, message }) => {
addRequiredFieldError(path, message)
})
} else if (selectedType === 'DuckLake') {
getDucklakeValidationIssues(data, { secretsOptional: editMode }).forEach(
({ path, message }) => {
addRequiredFieldError(path, message)
}
)
} else if (selectedType === 'Snowflake') {
getSnowflakeValidationIssues(data, {
secretsOptional: editMode,
validatePrivateKeyFormat: false,
}).forEach(({ path, message }) => {
addRequiredFieldError(path, message)
})
} else if (selectedType === 'ClickHouse') {
getClickHouseValidationIssues(data).forEach(({ path, message }) => {
addRequiredFieldError(path, message)
})
}
})
),
defaultValues,
})
// Always destructure formState values otherwise they won't be updated
// See https://react-hook-form.com/docs/useform/formstate
const { isDirty } = form.formState
const publicationName = useWatch({ control: form.control, name: 'publicationName' })
const { data: selectedPublication, isSuccess: isSuccessPublication } =
useReplicationPublicationQuery({ projectRef, sourceId, publicationName })
const isSelectedPublicationMissing =
isSuccessPublicationNames &&
!!publicationName &&
!publicationNames.some(({ name }) => name === publicationName)
const allValidationFailures = [...destinationValidationFailures, ...pipelineValidationFailures]
const hasValidationFailures = allValidationFailures.some((f) => f.failure_type === 'critical')
const validationWarnings = allValidationFailures.filter((f) => f.failure_type === 'warning')
const pendingTableSyncCopy = useMemo(
() =>
pendingFormValues === null
? undefined
: buildTableSyncCopyConfig({
mode: pendingFormValues.tableSyncCopyMode,
selectedTableIds: pendingFormValues.tableSyncCopyTableIds,
}),
[pendingFormValues]
)
const pendingPublicationTables =
selectedPublication && selectedPublication.name === pendingFormValues?.publicationName
? selectedPublication.tables
: []
const isSubmitDisabled =
isSaving ||
!isExistingConfigReady ||
!isSuccessPublicationNames ||
(!!publicationName && !isSuccessPublication) ||
isSelectedPublicationMissing ||
(!editMode && hasNoAvailableDestinations)
const getSubmitButtonText = () => {
if (editMode) {
return existingDestination?.enabled ? 'Apply and restart pipeline' : 'Apply changes'
} else {
if (hasRunValidation && validationWarnings.length > 0 && !hasValidationFailures) {
return 'Start pipeline anyway'
}
return 'Start pipeline'
}
}
const getSavingMessage = () => {
if (isValidating) return 'Validating destination configuration...'
if (!editMode) return 'Creating pipeline...'
if (existingDestination?.enabled) return 'Updating destination and restarting pipeline...'
return 'Updating destination...'
}
// Stages the form values and opens the cost-estimation dialog, which is the final gate before
// a pipeline is created and started.
const openCostDialog = (data: z.infer<typeof FormSchema>) => {
setPendingFormValues(data)
setShowCostDialog(true)
}
const onSubmit = async (rawData: z.infer<typeof FormSchema>) => {
if (!isSuccessPublication || !selectedPublication) {
toast.error('Publication tables are unavailable. Refresh and try again.')
return
}
// Drop any previously selected id that has since fallen out of the
// publication before validating a create or submitting an edit.
const data: z.infer<typeof FormSchema> = {
...rawData,
tableSyncCopyTableIds: pruneStaleSelectedTableIds({
mode: rawData.tableSyncCopyMode,
selectedTableIds: rawData.tableSyncCopyTableIds,
publication: selectedPublication,
publicationName: rawData.publicationName,
}),
tableOptions: pruneStaleTableOptions({
tableOptions: rawData.tableOptions,
publication: selectedPublication,
publicationName: rawData.publicationName,
}),
}
if (
(data.tableSyncCopyMode === 'include_tables' || data.tableSyncCopyMode === 'skip_tables') &&
data.tableSyncCopyTableIds.length === 0
) {
form.setError('tableSyncCopyTableIds', { message: 'Select at least one table' })
return
}
if (selectedType === 'BigQuery') {
const jsonIssue = getBigQueryValidationIssues(data, { secretsOptional: editMode }).find(
(issue) => issue.message === BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE
)
if (jsonIssue) {
form.setError(jsonIssue.path, { message: jsonIssue.message })
return
}
}
if (selectedType === 'Snowflake') {
const privateKeyIssue = getSnowflakeValidationIssues(data, {
secretsOptional: editMode,
}).find((issue) => issue.message === SNOWFLAKE_PRIVATE_KEY_FORMAT_MESSAGE)
if (privateKeyIssue) {
form.setError(privateKeyIssue.path, { message: privateKeyIssue.message })
return
}
}
// Pipeline prerequisite validation models a new pipeline and cannot
// account for resources already owned by an existing pipeline. Edits keep
// the established direct-update flow after pruning stale table ids.
if (editMode) {
await submitPipeline({
data,
existingDestination,
existingBatch: pipelineData?.config.batch,
onSuccess: () => form.reset(defaultValues),
onClose,
})
return
}
const previousValidationFailures = allValidationFailures
const previousWarnings = previousValidationFailures.filter((f) => f.failure_type === 'warning')
const previousFailuresAreOnlyWarnings =
hasRunValidation &&
previousValidationFailures.length > 0 &&
previousValidationFailures.every((f) => f.failure_type === 'warning')
const validationResult = await validateConfiguration({
data,
onValidationFail: () => {
setTimeout(() => {
validationSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' })
}, 100)
},
})
if (!validationResult.canContinue) {
// Critical failures shown inline — stop so user can fix them
return
}
const hasWarnings = validationResult.warnings.length > 0
const warningsUnchanged =
previousFailuresAreOnlyWarnings &&
areValidationFailuresEqual(previousWarnings, validationResult.warnings)
// Open the warnings confirmation when there are warnings (and they're unchanged on resubmit),
// otherwise go straight to the cost dialog. New/changed warnings are shown inline so the user
// can review and submit again.
if (hasWarnings) {
if (warningsUnchanged) {
setPendingFormValues(data)
setShowValidationWarningsDialog(true)
}
return
}
openCostDialog(data)
}
// Confirming create warnings advances to the cost dialog.
const handleValidationWarningsConfirm = () => {
if (!pendingFormValues) return
setShowValidationWarningsDialog(false)
openCostDialog(pendingFormValues)
}
const handleCostConfirm = async () => {
if (!pendingFormValues) return
const values = pendingFormValues
setShowCostDialog(false)
await submitPipeline({
data: values,
existingDestination,
existingBatch: pipelineData?.config.batch,
onSuccess: () => form.reset(defaultValues),
onClose,
})
}
useEffect(() => {
if (!checkIsDirtyRef) return
checkIsDirtyRef.current = () => isDirty
return () => {
checkIsDirtyRef.current = () => false
}
}, [checkIsDirtyRef, isDirty])
useEffect(() => {
// Reset when closed (including after discard) so reopening does not restore
// discarded values, and when open but pristine so async defaults can apply.
if (!visible || !isDirty) {
form.reset(defaultValues)
resetValidation()
}
}, [visible, defaultValues, form, isDirty, resetValidation])
const destinationFields = (
<>
{selectedType === 'BigQuery' && etlEnableBigQuery && (
<BigQueryFields form={form} editMode={editMode} />
)}
{selectedType === 'Analytics Bucket' && etlEnableIceberg && (
<AnalyticsBucketFields
form={form}
editMode={editMode}
onSelectNewBucket={() => setNewBucketSheetVisible(true)}
/>
)}
{selectedType === 'DuckLake' && etlEnableDucklake && (
<DuckLakeFields form={form} editMode={editMode} />
)}
{selectedType === 'Snowflake' && etlEnableSnowflake && (
<SnowflakeFields form={form} editMode={editMode} />
)}
{selectedType === 'ClickHouse' && etlEnableClickHouse && (
<ClickHouseFields form={form} editMode={editMode} />
)}
</>
)
return (
<>
<SheetSection className="grow overflow-auto px-0 py-0">
{typeSelection}
{hasNoAvailableDestinations && !editMode ? (
<NoDestinationsAvailable />
) : (
<Form {...form}>
<form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
<fieldset disabled={!isExistingConfigReady} className="contents">
{shouldShowConnectionSection && (
<>
<PipelineConnectionSection type={selectedType} form={form}>
{destinationFields}
</PipelineConnectionSection>
<DialogSectionSeparator />
</>
)}
{shouldShowConnectionSection ? (
<PipelineDataSection
type={selectedType}
form={form}
onSelectNewPublication={() => setPublicationPanelVisible(true)}
/>
) : (
<>
<div className="p-5 flex flex-col gap-y-6">
{isErrorExistingConfig && (
<Admonition type="warning">
<p className="leading-normal!">
The existing destination or pipeline settings could not be loaded.
Refresh before applying changes.
</p>
</Admonition>
)}
<p className="text-sm font-medium text-foreground">Destination details</p>
<div className="flex flex-col gap-y-4">
<DestinationNameInput form={form} destinationType={selectedType} />
<PublicationSelection
form={form}
onSelectNewPublication={() => setPublicationPanelVisible(true)}
/>
<TableCopySelection form={form} editMode={editMode} />
<PipelineRegionField destinationType={selectedType} />
</div>
</div>
<DialogSectionSeparator />
{destinationFields}
<DialogSectionSeparator />
<AdvancedSettings type={selectedType} form={form} group="all" />
</>
)}
{shouldShowConnectionSection && (
<>
<DialogSectionSeparator />
<PipelineDestinationReview type={selectedType} control={form.control} />
<DialogSectionSeparator />
<PipelineDataReview
type={selectedType}
control={form.control}
publication={selectedPublication}
/>
<DialogSectionSeparator />
<PipelineReviewCosts control={form.control} publication={selectedPublication} />
</>
)}
{!editMode && hasRunValidation && !isValidating && (
<>
<DialogSectionSeparator />
<div ref={validationSectionRef}>
{shouldShowConnectionSection ? (
<PipelineValidationAdmonition
failures={allValidationFailures}
hint="Check credentials and Advanced settings above. Some issues may require changes to the project’s database settings."
/>
) : (
<ValidationFailuresSection
destinationFailures={destinationValidationFailures}
pipelineFailures={pipelineValidationFailures}
/>
)}
</div>
</>
)}
</fieldset>
</form>
</Form>
)}
</SheetSection>
<SheetFooter className="justify-between!">
<AnimatePresence mode="wait">
{isValidating || isSaving ? (
<motion.div
className="flex items-center gap-x-2"
initial={{ opacity: 0, y: 5 }}
animate={{ opacity: 1, y: 0 }}
exit={{ opacity: 0, y: 5 }}
transition={{ duration: 0.2, ease: 'easeOut' }}
>
<Loader2Icon className="animate-spin" size={14} />
<p className="text-foreground-light text-sm">{getSavingMessage()}</p>
</motion.div>
) : (
<div />
)}
</AnimatePresence>
<div className="flex items-center gap-x-2">
<Button disabled={isSaving} onClick={onCancel}>
Cancel
</Button>
<Button
variant="primary"
disabled={isSubmitDisabled}
loading={isSaving}
form={formId}
type="submit"
>
{getSubmitButtonText()}
</Button>
</div>
</SheetFooter>
<NewPublicationPanel
visible={publicationPanelVisible}
onClose={(newPublication?: string) => {
if (newPublication) {
form.setValue('tableSyncCopyMode', 'include_all_tables', {
shouldDirty: true,
shouldValidate: true,
})
form.setValue('tableSyncCopyTableIds', [], {
shouldDirty: true,
shouldValidate: true,
})
form.setValue('tableOptions', [], {
shouldDirty: true,
shouldValidate: true,
})
form.setValue('publicationName', newPublication, {
shouldDirty: true,
shouldValidate: true,
})
}
setPublicationPanelVisible(false)
}}
/>
<CreateAnalyticsBucketSheet
open={newBucketSheetVisible}
onOpenChange={setNewBucketSheetVisible}
/>
<ValidationWarningsDialog
open={showValidationWarningsDialog}
onOpenChange={setShowValidationWarningsDialog}
isLoading={isSaving}
warningCount={validationWarnings.length}
onConfirm={handleValidationWarningsConfirm}
/>
<PipelineCostDialog
open={showCostDialog}
isConfirming={isSaving}
publicationName={pendingFormValues?.publicationName}
publicationTables={pendingPublicationTables}
tableSyncCopy={pendingTableSyncCopy}
onOpenChange={setShowCostDialog}
onConfirm={handleCostConfirm}
/>
</>
)
}