Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

24.7 KB
Raw Blame
import { zodResolver } from '@hookform/resolvers/zod'
import { useParams } from 'common'
import { Loader2Icon } from 'lucide-react'
import { AnimatePresence, motion } from 'motion/react'
import { RefObject, useEffect, useMemo, useRef, useState, type ReactNode } from 'react'
import { useForm, useWatch } from 'react-hook-form'
import { toast } from 'sonner'
import { Button, DialogSectionSeparator, Form, SheetFooter, SheetSection } from 'ui'
import { Admonition } from 'ui-patterns/Admonition'
import * as z from 'zod'

import { PipelineConnectionSection } from '../../CreatePipeline/PipelineConnectionSection'
import { PipelineDataReview } from '../../CreatePipeline/PipelineDataReview'
import { PipelineDataSection } from '../../CreatePipeline/PipelineDataSection'
import { PipelineDestinationReview } from '../../CreatePipeline/PipelineDestinationReview'
import { PipelineReviewCosts } from '../../CreatePipeline/PipelineReviewCosts'
import { PipelineValidationAdmonition } from '../../CreatePipeline/PipelineValidationAdmonition'
import {
  useIsETLBigQueryPrivateAlpha,
  useIsETLClickHousePrivateAlpha,
  useIsETLDucklakePrivateAlpha,
  useIsETLIcebergPrivateAlpha,
  useIsETLSnowflakePrivateAlpha,
} from '../../useIsETLPrivateAlpha'
import { type DestinationType, type ExistingDestination } from '../DestinationPanel.types'
import { AdvancedSettings } from './AdvancedSettings'
import { getAnalyticsBucketValidationIssues } from './AnalyticsBucket/AnalyticsBucket.utils'
import { AnalyticsBucketFields } from './AnalyticsBucket/Fields'
import {
  BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE,
  getBigQueryValidationIssues,
} from './BigQuery/BigQuery.utils'
import { BigQueryFields } from './BigQuery/Fields'
import { getClickHouseValidationIssues } from './ClickHouse/ClickHouse.utils'
import { ClickHouseFields } from './ClickHouse/Fields'
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
import {
  areValidationFailuresEqual,
  buildTableSyncCopyConfig,
  generateDefaultValues,
  pruneStaleSelectedTableIds,
  pruneStaleTableOptions,
} from './DestinationForm.utils'
import { DestinationNameInput } from './DestinationNameInput'
import { getDucklakeValidationIssues } from './DuckLake/DuckLake.utils'
import { DuckLakeFields } from './DuckLake/Fields'
import { NewPublicationPanel } from './NewPublicationPanel'
import { NoDestinationsAvailable } from './NoDestinationsAvailable'
import { PipelineCostDialog } from './PipelineCostDialog'
import { PipelineRegionField } from './PipelineRegionField'
import { PublicationSelection } from './PublicationSelection'
import { SnowflakeFields } from './Snowflake/Fields'
import {
  getSnowflakeValidationIssues,
  SNOWFLAKE_PRIVATE_KEY_FORMAT_MESSAGE,
} from './Snowflake/Snowflake.utils'
import { TableCopySelection } from './TableCopySelection'
import { useDestinationForm } from './useDestinationForm'
import { ValidationFailuresSection } from './ValidationFailuresSection'
import { ValidationWarningsDialog } from './ValidationWarningsDialog'
import { usePipelineCreationPreview } from '@/components/interfaces/App/FeaturePreview/FeaturePreviewContext'
import { CreateAnalyticsBucketSheet } from '@/components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
import { useAPIKeys } from '@/data/api-keys/api-keys-query'
import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query'
import { useReplicationDestinationByIdQuery } from '@/data/replication/destination-by-id-query'
import { useReplicationPipelineByIdQuery } from '@/data/replication/pipeline-by-id-query'
import { useReplicationPublicationNamesQuery } from '@/data/replication/publication-names-query'
import { useReplicationPublicationQuery } from '@/data/replication/publication-query'
import { useReplicationSourceId } from '@/data/replication/sources-query'
import { FGA_PERMISSIONS, useAsyncCheckPermissionsV2 } from '@/hooks/misc/useCheckPermissionsV2'

const formId = 'destination-editor'

interface DestinationFormProps {
  selectedType: DestinationType
  visible: boolean
  existingDestination?: ExistingDestination
  typeSelection?: ReactNode
  checkIsDirtyRef?: RefObject<() => boolean>
  onClose: () => void
  onCancel?: () => void
}

export const DestinationForm = ({
  selectedType,
  visible,
  existingDestination,
  typeSelection,
  checkIsDirtyRef,
  onClose,
  onCancel = onClose,
}: DestinationFormProps) => {
  const { ref: projectRef } = useParams()

  const etlEnableBigQuery = useIsETLBigQueryPrivateAlpha()
  const etlEnableIceberg = useIsETLIcebergPrivateAlpha()
  const etlEnableDucklake = useIsETLDucklakePrivateAlpha()
  const etlEnableSnowflake = useIsETLSnowflakePrivateAlpha()
  const etlEnableClickHouse = useIsETLClickHousePrivateAlpha()
  const { can: canReadAPIKeys } = useAsyncCheckPermissionsV2([
    FGA_PERMISSIONS.PROJECT.API_GATEWAY_KEYS_READ,
    // api keys reveal = true will require this perms
    FGA_PERMISSIONS.PROJECT.API_GATEWAY_KEYS_SECRET_READ,
  ])

  const [showValidationWarningsDialog, setShowValidationWarningsDialog] = useState(false)
  const [showCostDialog, setShowCostDialog] = useState(false)
  const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
  const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)
  const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
    null
  )

  const validationSectionRef = useRef<HTMLDivElement>(null)

  const editMode = !!existingDestination
  const { isEnabled: isCreationPreviewEnabled } = usePipelineCreationPreview()
  const shouldShowConnectionSection = isCreationPreviewEnabled && !editMode

  // Compute available destinations based on feature flags
  const availableDestinations = useMemo(() => {
    const destinations = []
    if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })
    if (etlEnableIceberg)
      destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })
    if (etlEnableDucklake) destinations.push({ value: 'DuckLake', label: 'DuckLake' })
    if (etlEnableSnowflake) destinations.push({ value: 'Snowflake', label: 'Snowflake' })
    if (etlEnableClickHouse) destinations.push({ value: 'ClickHouse', label: 'ClickHouse' })
    return destinations
  }, [
    etlEnableBigQuery,
    etlEnableDucklake,
    etlEnableIceberg,
    etlEnableSnowflake,
    etlEnableClickHouse,
  ])
  const hasNoAvailableDestinations = availableDestinations.length === 0

  const sourceId = useReplicationSourceId({ projectRef })

  const { data: publicationNames = [], isSuccess: isSuccessPublicationNames } =
    useReplicationPublicationNamesQuery({ projectRef, sourceId })

  const {
    data: destinationData,
    isError: isErrorDestination,
    isSuccess: isSuccessDestination,
  } = useReplicationDestinationByIdQuery({
    projectRef,
    destinationId: existingDestination?.destinationId,
  })

  const {
    data: pipelineData,
    isError: isErrorPipeline,
    isSuccess: isSuccessPipeline,
  } = useReplicationPipelineByIdQuery({
    projectRef,
    pipelineId: existingDestination?.pipelineId,
  })
  const isErrorExistingConfig = editMode && (isErrorDestination || isErrorPipeline)
  const isExistingConfigReady = !editMode || (isSuccessDestination && isSuccessPipeline)

  // Revealed API keys are only ever consumed as the default catalog token for
  // Analytics Bucket (Iceberg) destinations, so don't fetch them for other
  // destination types.
  const { data: apiKeysData } = useAPIKeys(
    { projectRef, reveal: true },
    { enabled: canReadAPIKeys && selectedType === 'Analytics Bucket' && !editMode }
  )
  const { serviceKey } = apiKeysData ?? {}
  const catalogToken = serviceKey?.api_key ?? ''

  const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })

  const {
    isValidating,
    validateConfiguration,
    isSaving,
    submitPipeline,
    hasRunValidation,
    destinationValidationFailures,
    pipelineValidationFailures,
    resetValidation,
  } = useDestinationForm({ selectedType })

  const defaultValues = useMemo(
    () =>
      generateDefaultValues({
        destinationData,
        pipelineData,
        catalogToken,
        region: projectSettings?.region,
        editMode,
      }),
    [destinationData, pipelineData, catalogToken, projectSettings, editMode]
  )
  const form = useForm<z.infer<typeof FormSchema>>({
    mode: 'onSubmit',
    reValidateMode: 'onChange',
    resolver: zodResolver(
      FormSchema.superRefine((data, ctx) => {
        const addRequiredFieldError = (path: string, message: string) => {
          ctx.addIssue({
            code: z.ZodIssueCode.custom,
            message,
            path: [path],
          })
        }

        if (
          (data.tableSyncCopyMode === 'include_tables' ||
            data.tableSyncCopyMode === 'skip_tables') &&
          data.tableSyncCopyTableIds.length === 0
        ) {
          addRequiredFieldError('tableSyncCopyTableIds', 'Select at least one table')
        }

        if (selectedType === 'BigQuery') {
          getBigQueryValidationIssues(data, {
            secretsOptional: editMode,
            validateJson: false,
          }).forEach(({ path, message }) => {
            addRequiredFieldError(path, message)
          })
        } else if (selectedType === 'Analytics Bucket') {
          getAnalyticsBucketValidationIssues(data, {
            secretsOptional: editMode,
            storedS3AccessKeyId: editMode ? defaultValues.s3AccessKeyId : undefined,
          }).forEach(({ path, message }) => {
            addRequiredFieldError(path, message)
          })
        } else if (selectedType === 'DuckLake') {
          getDucklakeValidationIssues(data, { secretsOptional: editMode }).forEach(
            ({ path, message }) => {
              addRequiredFieldError(path, message)
            }
          )
        } else if (selectedType === 'Snowflake') {
          getSnowflakeValidationIssues(data, {
            secretsOptional: editMode,
            validatePrivateKeyFormat: false,
          }).forEach(({ path, message }) => {
            addRequiredFieldError(path, message)
          })
        } else if (selectedType === 'ClickHouse') {
          getClickHouseValidationIssues(data).forEach(({ path, message }) => {
            addRequiredFieldError(path, message)
          })
        }
      })
    ),
    defaultValues,
  })

  // Always destructure formState values otherwise they won't be updated
  // See https://react-hook-form.com/docs/useform/formstate
  const { isDirty } = form.formState

  const publicationName = useWatch({ control: form.control, name: 'publicationName' })
  const { data: selectedPublication, isSuccess: isSuccessPublication } =
    useReplicationPublicationQuery({ projectRef, sourceId, publicationName })

  const isSelectedPublicationMissing =
    isSuccessPublicationNames &&
    !!publicationName &&
    !publicationNames.some(({ name }) => name === publicationName)

  const allValidationFailures = [...destinationValidationFailures, ...pipelineValidationFailures]
  const hasValidationFailures = allValidationFailures.some((f) => f.failure_type === 'critical')
  const validationWarnings = allValidationFailures.filter((f) => f.failure_type === 'warning')

  const pendingTableSyncCopy = useMemo(
    () =>
      pendingFormValues === null
        ? undefined
        : buildTableSyncCopyConfig({
            mode: pendingFormValues.tableSyncCopyMode,
            selectedTableIds: pendingFormValues.tableSyncCopyTableIds,
          }),
    [pendingFormValues]
  )
  const pendingPublicationTables =
    selectedPublication && selectedPublication.name === pendingFormValues?.publicationName
      ? selectedPublication.tables
      : []

  const isSubmitDisabled =
    isSaving ||
    !isExistingConfigReady ||
    !isSuccessPublicationNames ||
    (!!publicationName && !isSuccessPublication) ||
    isSelectedPublicationMissing ||
    (!editMode && hasNoAvailableDestinations)

  const getSubmitButtonText = () => {
    if (editMode) {
      return existingDestination?.enabled ? 'Apply and restart pipeline' : 'Apply changes'
    } else {
      if (hasRunValidation && validationWarnings.length > 0 && !hasValidationFailures) {
        return 'Start pipeline anyway'
      }

      return 'Start pipeline'
    }
  }

  const getSavingMessage = () => {
    if (isValidating) return 'Validating destination configuration...'
    if (!editMode) return 'Creating pipeline...'
    if (existingDestination?.enabled) return 'Updating destination and restarting pipeline...'
    return 'Updating destination...'
  }

  // Stages the form values and opens the cost-estimation dialog, which is the final gate before
  // a pipeline is created and started.
  const openCostDialog = (data: z.infer<typeof FormSchema>) => {
    setPendingFormValues(data)
    setShowCostDialog(true)
  }

  const onSubmit = async (rawData: z.infer<typeof FormSchema>) => {
    if (!isSuccessPublication || !selectedPublication) {
      toast.error('Publication tables are unavailable. Refresh and try again.')
      return
    }

    // Drop any previously selected id that has since fallen out of the
    // publication before validating a create or submitting an edit.
    const data: z.infer<typeof FormSchema> = {
      ...rawData,
      tableSyncCopyTableIds: pruneStaleSelectedTableIds({
        mode: rawData.tableSyncCopyMode,
        selectedTableIds: rawData.tableSyncCopyTableIds,
        publication: selectedPublication,
        publicationName: rawData.publicationName,
      }),
      tableOptions: pruneStaleTableOptions({
        tableOptions: rawData.tableOptions,
        publication: selectedPublication,
        publicationName: rawData.publicationName,
      }),
    }

    if (
      (data.tableSyncCopyMode === 'include_tables' || data.tableSyncCopyMode === 'skip_tables') &&
      data.tableSyncCopyTableIds.length === 0
    ) {
      form.setError('tableSyncCopyTableIds', { message: 'Select at least one table' })
      return
    }

    if (selectedType === 'BigQuery') {
      const jsonIssue = getBigQueryValidationIssues(data, { secretsOptional: editMode }).find(
        (issue) => issue.message === BIGQUERY_SERVICE_ACCOUNT_JSON_MESSAGE
      )
      if (jsonIssue) {
        form.setError(jsonIssue.path, { message: jsonIssue.message })
        return
      }
    }

    if (selectedType === 'Snowflake') {
      const privateKeyIssue = getSnowflakeValidationIssues(data, {
        secretsOptional: editMode,
      }).find((issue) => issue.message === SNOWFLAKE_PRIVATE_KEY_FORMAT_MESSAGE)
      if (privateKeyIssue) {
        form.setError(privateKeyIssue.path, { message: privateKeyIssue.message })
        return
      }
    }

    // Pipeline prerequisite validation models a new pipeline and cannot
    // account for resources already owned by an existing pipeline. Edits keep
    // the established direct-update flow after pruning stale table ids.
    if (editMode) {
      await submitPipeline({
        data,
        existingDestination,
        existingBatch: pipelineData?.config.batch,
        onSuccess: () => form.reset(defaultValues),
        onClose,
      })
      return
    }

    const previousValidationFailures = allValidationFailures
    const previousWarnings = previousValidationFailures.filter((f) => f.failure_type === 'warning')
    const previousFailuresAreOnlyWarnings =
      hasRunValidation &&
      previousValidationFailures.length > 0 &&
      previousValidationFailures.every((f) => f.failure_type === 'warning')

    const validationResult = await validateConfiguration({
      data,
      onValidationFail: () => {
        setTimeout(() => {
          validationSectionRef.current?.scrollIntoView({ behavior: 'smooth', block: 'start' })
        }, 100)
      },
    })
    if (!validationResult.canContinue) {
      // Critical failures shown inline — stop so user can fix them
      return
    }

    const hasWarnings = validationResult.warnings.length > 0
    const warningsUnchanged =
      previousFailuresAreOnlyWarnings &&
      areValidationFailuresEqual(previousWarnings, validationResult.warnings)

    // Open the warnings confirmation when there are warnings (and they're unchanged on resubmit),
    // otherwise go straight to the cost dialog. New/changed warnings are shown inline so the user
    // can review and submit again.
    if (hasWarnings) {
      if (warningsUnchanged) {
        setPendingFormValues(data)
        setShowValidationWarningsDialog(true)
      }
      return
    }

    openCostDialog(data)
  }

  // Confirming create warnings advances to the cost dialog.
  const handleValidationWarningsConfirm = () => {
    if (!pendingFormValues) return
    setShowValidationWarningsDialog(false)
    openCostDialog(pendingFormValues)
  }

  const handleCostConfirm = async () => {
    if (!pendingFormValues) return

    const values = pendingFormValues
    setShowCostDialog(false)

    await submitPipeline({
      data: values,
      existingDestination,
      existingBatch: pipelineData?.config.batch,
      onSuccess: () => form.reset(defaultValues),
      onClose,
    })
  }

  useEffect(() => {
    if (!checkIsDirtyRef) return

    checkIsDirtyRef.current = () => isDirty
    return () => {
      checkIsDirtyRef.current = () => false
    }
  }, [checkIsDirtyRef, isDirty])

  useEffect(() => {
    // Reset when closed (including after discard) so reopening does not restore
    // discarded values, and when open but pristine so async defaults can apply.
    if (!visible || !isDirty) {
      form.reset(defaultValues)
      resetValidation()
    }
  }, [visible, defaultValues, form, isDirty, resetValidation])

  const destinationFields = (
    <>
      {selectedType === 'BigQuery' && etlEnableBigQuery && (
        <BigQueryFields form={form} editMode={editMode} />
      )}
      {selectedType === 'Analytics Bucket' && etlEnableIceberg && (
        <AnalyticsBucketFields
          form={form}
          editMode={editMode}
          onSelectNewBucket={() => setNewBucketSheetVisible(true)}
        />
      )}
      {selectedType === 'DuckLake' && etlEnableDucklake && (
        <DuckLakeFields form={form} editMode={editMode} />
      )}
      {selectedType === 'Snowflake' && etlEnableSnowflake && (
        <SnowflakeFields form={form} editMode={editMode} />
      )}
      {selectedType === 'ClickHouse' && etlEnableClickHouse && (
        <ClickHouseFields form={form} editMode={editMode} />
      )}
    </>
  )

  return (
    <>
      <SheetSection className="grow overflow-auto px-0 py-0">
        {typeSelection}
        {hasNoAvailableDestinations && !editMode ? (
          <NoDestinationsAvailable />
        ) : (
          <Form {...form}>
            <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
              <fieldset disabled={!isExistingConfigReady} className="contents">
                {shouldShowConnectionSection && (
                  <>
                    <PipelineConnectionSection type={selectedType} form={form}>
                      {destinationFields}
                    </PipelineConnectionSection>
                    <DialogSectionSeparator />
                  </>
                )}
                {shouldShowConnectionSection ? (
                  <PipelineDataSection
                    type={selectedType}
                    form={form}
                    onSelectNewPublication={() => setPublicationPanelVisible(true)}
                  />
                ) : (
                  <>
                    <div className="p-5 flex flex-col gap-y-6">
                      {isErrorExistingConfig && (
                        <Admonition type="warning">
                          <p className="leading-normal!">
                            The existing destination or pipeline settings could not be loaded.
                            Refresh before applying changes.
                          </p>
                        </Admonition>
                      )}
                      <p className="text-sm font-medium text-foreground">Destination details</p>

                      <div className="flex flex-col gap-y-4">
                        <DestinationNameInput form={form} destinationType={selectedType} />
                        <PublicationSelection
                          form={form}
                          onSelectNewPublication={() => setPublicationPanelVisible(true)}
                        />
                        <TableCopySelection form={form} editMode={editMode} />
                        <PipelineRegionField destinationType={selectedType} />
                      </div>
                    </div>

                    <DialogSectionSeparator />
                    {destinationFields}

                    <DialogSectionSeparator />

                    <AdvancedSettings type={selectedType} form={form} group="all" />
                  </>
                )}

                {shouldShowConnectionSection && (
                  <>
                    <DialogSectionSeparator />
                    <PipelineDestinationReview type={selectedType} control={form.control} />
                    <DialogSectionSeparator />
                    <PipelineDataReview
                      type={selectedType}
                      control={form.control}
                      publication={selectedPublication}
                    />
                    <DialogSectionSeparator />
                    <PipelineReviewCosts control={form.control} publication={selectedPublication} />
                  </>
                )}

                {!editMode && hasRunValidation && !isValidating && (
                  <>
                    <DialogSectionSeparator />

                    <div ref={validationSectionRef}>
                      {shouldShowConnectionSection ? (
                        <PipelineValidationAdmonition
                          failures={allValidationFailures}
                          hint="Check credentials and Advanced settings above. Some issues may require changes to the project’s database settings."
                        />
                      ) : (
                        <ValidationFailuresSection
                          destinationFailures={destinationValidationFailures}
                          pipelineFailures={pipelineValidationFailures}
                        />
                      )}
                    </div>
                  </>
                )}
              </fieldset>
            </form>
          </Form>
        )}
      </SheetSection>

      <SheetFooter className="justify-between!">
        <AnimatePresence mode="wait">
          {isValidating || isSaving ? (
            <motion.div
              className="flex items-center gap-x-2"
              initial={{ opacity: 0, y: 5 }}
              animate={{ opacity: 1, y: 0 }}
              exit={{ opacity: 0, y: 5 }}
              transition={{ duration: 0.2, ease: 'easeOut' }}
            >
              <Loader2Icon className="animate-spin" size={14} />
              <p className="text-foreground-light text-sm">{getSavingMessage()}</p>
            </motion.div>
          ) : (
            <div />
          )}
        </AnimatePresence>
        <div className="flex items-center gap-x-2">
          <Button disabled={isSaving} onClick={onCancel}>
            Cancel
          </Button>
          <Button
            variant="primary"
            disabled={isSubmitDisabled}
            loading={isSaving}
            form={formId}
            type="submit"
          >
            {getSubmitButtonText()}
          </Button>
        </div>
      </SheetFooter>

      <NewPublicationPanel
        visible={publicationPanelVisible}
        onClose={(newPublication?: string) => {
          if (newPublication) {
            form.setValue('tableSyncCopyMode', 'include_all_tables', {
              shouldDirty: true,
              shouldValidate: true,
            })
            form.setValue('tableSyncCopyTableIds', [], {
              shouldDirty: true,
              shouldValidate: true,
            })
            form.setValue('tableOptions', [], {
              shouldDirty: true,
              shouldValidate: true,
            })
            form.setValue('publicationName', newPublication, {
              shouldDirty: true,
              shouldValidate: true,
            })
          }
          setPublicationPanelVisible(false)
        }}
      />

      <CreateAnalyticsBucketSheet
        open={newBucketSheetVisible}
        onOpenChange={setNewBucketSheetVisible}
      />

      <ValidationWarningsDialog
        open={showValidationWarningsDialog}
        onOpenChange={setShowValidationWarningsDialog}
        isLoading={isSaving}
        warningCount={validationWarnings.length}
        onConfirm={handleValidationWarningsConfirm}
      />

      <PipelineCostDialog
        open={showCostDialog}
        isConfirming={isSaving}
        publicationName={pendingFormValues?.publicationName}
        publicationTables={pendingPublicationTables}
        tableSyncCopy={pendingTableSyncCopy}
        onOpenChange={setShowCostDialog}
        onConfirm={handleCostConfirm}
      />
    </>
  )
}