The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
Shift ETL destination components into a separate folder (#41457)
Joshen Lim31d2e8ac5b6b
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxsimilarity index 93%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxindex 11dc657b876..f3260a82d46 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx@@ -12,11 +12,16 @@ import {Input_Shadcn_,} from 'ui'import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'--export const AdvancedSettings = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {- const { type } = form.watch()+import { DestinationType } from '../DestinationPanel.types'+import { type DestinationPanelSchemaType } from './DestinationForm.schema'+export const AdvancedSettings = ({+ type,+ form,+}: {+ type: DestinationType+ form: UseFormReturn<DestinationPanelSchemaType>+}) => {const handleNumberChange =(field: { onChange: (value?: number) => void }) => (e: ChangeEvent<HTMLInputElement>) => {const val = e.target.valuediff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.constants.tssimilarity index 100%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.tsrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.constants.tsdiff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.tsnew file mode 100644index 00000000000..97e09b56a95--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts@@ -0,0 +1,25 @@+import * as z from 'zod'++// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components+// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other+export const DestinationPanelFormSchema = z.object({+ // Common fields+ name: z.string().min(1, 'Name is required'),+ publicationName: z.string().min(1, 'Publication is required'),+ maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),+ // BigQuery fields+ projectId: z.string().optional(),+ datasetId: z.string().optional(),+ serviceAccountKey: z.string().optional(),+ maxStalenessMins: z.number().nonnegative().optional(),+ // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields+ warehouseName: z.string().optional(),+ namespace: z.string().optional(),+ newNamespaceName: z.string().optional(),+ catalogToken: z.string().optional(),+ s3AccessKeyId: z.string().optional(),+ s3SecretAccessKey: z.string().optional(),+ s3Region: z.string().optional(),+})++export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsxsimilarity index 90%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsxindex d444f9ae624..2a39aaf8f6d 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsx@@ -2,7 +2,7 @@ import type { UseFormReturn } from 'react-hook-form'import { FormControl_Shadcn_, FormField_Shadcn_, Input_Shadcn_ } from 'ui'import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'+import type { DestinationPanelSchemaType } from './DestinationForm.schema'type DestinationNameInputProps = {form: UseFormReturn<DestinationPanelSchemaType>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsxsimilarity index 98%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsxindex 920a19f7614..9b1003c52cf 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsx@@ -27,8 +27,8 @@ import {import { Admonition } from 'ui-patterns'import { Input } from 'ui-patterns/DataInputs/Input'import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationPanel.constants'-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'+import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'+import type { DestinationPanelSchemaType } from './DestinationForm.schema'export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {return (@@ -114,7 +114,7 @@ export const AnalyticsBucketFields = ({setIsFormInteracting: (value: boolean) => voidonSelectNewBucket: () => void}) => {- const { warehouseName, type, s3AccessKeyId, namespace } = form.watch()+ const { warehouseName, s3AccessKeyId, namespace } = form.watch()const [showCatalogToken, setShowCatalogToken] = useState(false)const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)@@ -153,13 +153,8 @@ export const AnalyticsBucketFields = ({isPending: isLoadingNamespaces,isError: isErrorNamespaces,} = useIcebergNamespacesQuery(- {- projectRef,- warehouse: warehouseName,- },- {- enabled: type === 'Analytics Bucket' && !!serviceApiKey,- }+ { projectRef, warehouse: warehouseName },+ { enabled: !!serviceApiKey })return (diff --git a/apps/studio/components/interfaces/Database/Replication/NewPublicationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsxsimilarity index 100%rename from apps/studio/components/interfaces/Database/Replication/NewPublicationPanel.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsxdiff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/NoDestinationsAvailable.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NoDestinationsAvailable.tsxsimilarity index 100%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/NoDestinationsAvailable.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NoDestinationsAvailable.tsxdiff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsxsimilarity index 96%rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsxindex 0c3da86f327..2df4f0e2de9 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx@@ -9,8 +9,8 @@ import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'import { FormControl_Shadcn_, FormField_Shadcn_ } from 'ui'import { Admonition } from 'ui-patterns'import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'-import { PublicationsComboBox } from '../PublicationsComboBox'-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'+import type { DestinationPanelSchemaType } from './DestinationForm.schema'+import { PublicationsComboBox } from './PublicationsComboBox'type PublicationSelectionProps = {form: UseFormReturn<DestinationPanelSchemaType>diff --git a/apps/studio/components/interfaces/Database/Replication/PublicationsComboBox.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationsComboBox.tsxsimilarity index 100%rename from apps/studio/components/interfaces/Database/Replication/PublicationsComboBox.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationsComboBox.tsxdiff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationDisclaimerDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ReplicationDisclaimerDialog.tsxsimilarity index 100%rename from apps/studio/components/interfaces/Database/Replication/ReplicationDisclaimerDialog.tsxrename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ReplicationDisclaimerDialog.tsxdiff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsxnew file mode 100644index 00000000000..5783b7e8af1--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx@@ -0,0 +1,546 @@+import { zodResolver } from '@hookform/resolvers/zod'+import { PermissionAction } from '@supabase/shared-types/out/constants'+import { snakeCase } from 'lodash'+import { useEffect, useMemo, useState } from 'react'+import { useForm } from 'react-hook-form'+import { toast } from 'sonner'+import * as z from 'zod'++import { useFlag, useParams } from 'common'+import { CreateAnalyticsBucketSheet } from 'components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'+import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'+import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'+import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query'+import {+ BatchConfig,+ BigQueryDestinationConfig,+ DestinationConfig,+ IcebergDestinationConfig,+ useCreateDestinationPipelineMutation,+} from 'data/replication/create-destination-pipeline-mutation'+import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'+import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id-query'+import { useReplicationPublicationsQuery } from 'data/replication/publications-query'+import { useRestartPipelineHelper } from 'data/replication/restart-pipeline-helper'+import { useReplicationSourcesQuery } from 'data/replication/sources-query'+import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'+import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'+import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation'+import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'+import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'+import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'+import {+ PipelineStatusRequestStatus,+ usePipelineRequestStatus,+} from 'state/replication-pipeline-request-status'+import { Button, DialogSectionSeparator, Form_Shadcn_, SheetFooter, SheetSection } from 'ui'+import { DestinationType } from '../DestinationPanel.types'+import { AdvancedSettings } from './AdvancedSettings'+import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'+import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'+import { DestinationNameInput } from './DestinationNameInput'+import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'+import { NewPublicationPanel } from './NewPublicationPanel'+import { NoDestinationsAvailable } from './NoDestinationsAvailable'+import { PublicationSelection } from './PublicationSelection'+import { ReplicationDisclaimerDialog } from './ReplicationDisclaimerDialog'++const formId = 'destination-editor'++interface DestinationFormProps {+ selectedType: DestinationType+ visible: boolean+ existingDestination?: {+ sourceId?: number+ destinationId: number+ pipelineId?: number+ enabled: boolean+ statusName?: string+ }+ onClose: () => void+}++export const DestinationForm = ({+ selectedType,+ visible,+ existingDestination,+ onClose,+}: DestinationFormProps) => {+ const { ref: projectRef } = useParams()+ const { data: project } = useSelectedProjectQuery()+ const { setRequestStatus } = usePipelineRequestStatus()++ const etlEnableBigQuery = useFlag('etlEnableBigQuery')+ const etlEnableIceberg = useFlag('etlEnableIceberg')+ const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')++ const [isFormInteracting, setIsFormInteracting] = useState(false)+ const [showDisclaimerDialog, setShowDisclaimerDialog] = useState(false)+ const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)+ const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)+ const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(+ null+ )++ const editMode = !!existingDestination++ // Compute available destinations based on feature flags+ const availableDestinations = useMemo(() => {+ const destinations = []+ if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })+ if (etlEnableIceberg)+ destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })+ return destinations+ }, [etlEnableBigQuery, etlEnableIceberg])+ const hasNoAvailableDestinations = availableDestinations.length === 0++ const { data: sourcesData } = useReplicationSourcesQuery({ projectRef })+ const sourceId = sourcesData?.sources.find((s) => s.name === projectRef)?.id++ const {+ data: publications = [],+ isSuccess: isSuccessPublications,+ refetch: refetchPublications,+ } = useReplicationPublicationsQuery({ projectRef, sourceId })++ const { data: destinationData } = useReplicationDestinationByIdQuery({+ projectRef,+ destinationId: existingDestination?.destinationId,+ })++ const { data: pipelineData } = useReplicationPipelineByIdQuery({+ projectRef,+ pipelineId: existingDestination?.pipelineId,+ })++ const { data: apiKeys } = useAPIKeysQuery(+ { projectRef, reveal: true },+ { enabled: canReadAPIKeys }+ )+ const { serviceKey } = getKeys(apiKeys)+ const catalogToken = serviceKey?.api_key ?? ''++ const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })++ const { mutateAsync: createDestinationPipeline, isPending: creatingDestinationPipeline } =+ useCreateDestinationPipelineMutation({+ onSuccess: () => form.reset(defaultValues),+ })++ const { mutateAsync: updateDestinationPipeline, isPending: updatingDestinationPipeline } =+ useUpdateDestinationPipelineMutation({+ onSuccess: () => form.reset(defaultValues),+ })++ const { mutateAsync: startPipeline, isPending: startingPipeline } = useStartPipelineMutation()+ const { restartPipeline } = useRestartPipelineHelper()++ const { mutateAsync: createS3AccessKey, isPending: isCreatingS3AccessKey } =+ useS3AccessKeyCreateMutation()++ const { mutateAsync: createNamespace, isPending: isCreatingNamespace } =+ useIcebergNamespaceCreateMutation()++ const defaultValues = useMemo(() => {+ const config = destinationData?.config+ const isBigQueryConfig = config && 'big_query' in config+ const isIcebergConfig = config && 'iceberg' in config++ return {+ // Common fields+ name: destinationData?.name ?? '',+ publicationName: pipelineData?.config.publication_name ?? '',+ maxFillMs: pipelineData?.config?.batch?.max_fill_ms,+ // BigQuery fields+ projectId: isBigQueryConfig ? config.big_query.project_id : '',+ datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',+ serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',+ maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined,+ // Analytics Bucket fields+ warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',+ namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',+ newNamespaceName: '',+ catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,+ s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',+ s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',+ s3Region:+ projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),+ }+ }, [destinationData, pipelineData, catalogToken, projectSettings])++ const form = useForm<z.infer<typeof FormSchema>>({+ mode: 'onChange',+ reValidateMode: 'onChange',+ resolver: zodResolver(+ FormSchema.superRefine((data, ctx) => {+ const addRequiredFieldError = (path: string, message: string) => {+ ctx.addIssue({+ code: z.ZodIssueCode.custom,+ message,+ path: [path],+ })+ }++ if (selectedType === 'BigQuery') {+ if (!data.projectId?.length) addRequiredFieldError('projectId', 'Project ID is required')+ if (!data.datasetId?.length) addRequiredFieldError('datasetId', 'Dataset ID is required')+ if (!data.serviceAccountKey?.length)+ addRequiredFieldError('serviceAccountKey', 'Service Account Key is required')+ } else if (selectedType === 'Analytics Bucket') {+ if (!data.warehouseName?.length)+ addRequiredFieldError('warehouseName', 'Bucket is required')++ const hasValidNamespace =+ (data.namespace?.length && data.namespace !== 'create-new-namespace') ||+ (data.namespace === 'create-new-namespace' && data.newNamespaceName?.length)++ if (!hasValidNamespace) {+ const isCreatingNew = data.namespace === 'create-new-namespace'+ addRequiredFieldError(+ isCreatingNew ? 'newNamespaceName' : 'namespace',+ isCreatingNew ? 'Namespace name is required' : 'Namespace is required'+ )+ }++ if (!data.s3Region?.length) addRequiredFieldError('s3Region', 'S3 Region is required')++ if (!data.s3AccessKeyId?.length)+ addRequiredFieldError('s3AccessKeyId', 'S3 Access Key ID is required')++ if (data.s3AccessKeyId !== 'create-new' && !data.s3SecretAccessKey?.length) {+ addRequiredFieldError('s3SecretAccessKey', 'S3 Secret Access Key is required')+ }+ }+ })+ ),+ defaultValues,+ })++ const { publicationName, warehouseName } = form.watch()++ const publicationNames = useMemo(() => publications?.map((pub) => pub.name) ?? [], [publications])+ const isSelectedPublicationMissing =+ isSuccessPublications && !!publicationName && !publicationNames.includes(publicationName)+ const selectedPublication = publications.find((pub) => pub.name === publicationName)++ const { data: checkPrimaryKeysExistsData, isPending: isLoadingCheck } = useCheckPrimaryKeysExists(+ {+ projectRef: project?.ref,+ connectionString: project?.connectionString,+ tables: selectedPublication?.tables ?? [],+ },+ { enabled: visible && !!selectedPublication }+ )+ const hasTablesWithNoPrimaryKeys = (checkPrimaryKeysExistsData?.offendingTables ?? []).length > 0++ // Helper function to handle namespace creation if needed+ const resolveNamespace = async (data: z.infer<typeof FormSchema>) => {+ if (data.namespace === CREATE_NEW_NAMESPACE) {+ if (!data.newNamespaceName) throw new Error('New namespace name is required')++ await createNamespace({+ projectRef,+ warehouse: data.warehouseName!,+ namespace: data.newNamespaceName,+ })++ return data.newNamespaceName+ }+ return data.namespace+ }++ // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty+ // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics+ // At first glance we could try grouping as edit / create bigquery, edit / create analytics+ // since the destination config seems rather similar between edit and create for the same type+ const submitPipeline = async (data: z.infer<typeof FormSchema>) => {+ if (!projectRef) return console.error('Project ref is required')+ if (!sourceId) return console.error('Source id is required')+ if (isSelectedPublicationMissing) {+ return toast.error('Please select another publication before continuing')+ }++ try {+ if (editMode && existingDestination) {+ if (!existingDestination.pipelineId) return console.error('Pipeline id is required')++ let destinationConfig: DestinationConfig | undefined = undefined++ if (selectedType === 'BigQuery') {+ const bigQueryConfig: BigQueryDestinationConfig = {+ projectId: data.projectId ?? '',+ datasetId: data.datasetId ?? '',+ serviceAccountKey: data.serviceAccountKey ?? '',+ }+ if (!!data.maxStalenessMins) {+ bigQueryConfig.maxStalenessMins = data.maxStalenessMins+ }+ destinationConfig = { bigQuery: bigQueryConfig }+ } else if (selectedType === 'Analytics Bucket') {+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }++ if (data.s3AccessKeyId === CREATE_NEW_KEY) {+ const newKeys = await createS3AccessKey({+ projectRef,+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,+ })+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }+ }++ // Resolve namespace (create if needed)+ const finalNamespace = await resolveNamespace(data)++ const icebergConfig: IcebergDestinationConfig = {+ projectRef: projectRef,+ warehouseName: data.warehouseName ?? '',+ namespace: finalNamespace,+ catalogToken: data.catalogToken ?? '',+ s3AccessKeyId: s3Keys.accessKey ?? '',+ s3SecretAccessKey: s3Keys.secretKey ?? '',+ s3Region: data.s3Region ?? '',+ }+ destinationConfig = { iceberg: icebergConfig }+ }++ const batchConfig: BatchConfig | undefined = !!data.maxFillMs+ ? { maxFillMs: data.maxFillMs }+ : undefined+ const hasBatchFields = batchConfig !== undefined++ if (!destinationConfig) throw new Error('Destination configuration is missing')++ await updateDestinationPipeline({+ destinationId: existingDestination.destinationId,+ pipelineId: existingDestination.pipelineId,+ projectRef,+ destinationName: data.name,+ destinationConfig,+ pipelineConfig: {+ publicationName: data.publicationName,+ ...(hasBatchFields ? { batch: batchConfig } : {}),+ },+ sourceId,+ })+ // Set request status only right before starting, then fire and close+ const snapshot =+ existingDestination.statusName ?? (existingDestination.enabled ? 'started' : 'stopped')+ if (existingDestination.enabled) {+ setRequestStatus(+ existingDestination.pipelineId,+ PipelineStatusRequestStatus.RestartRequested,+ snapshot+ )+ toast.success('Settings applied. Restarting the pipeline...')+ restartPipeline({ projectRef, pipelineId: existingDestination.pipelineId })+ } else {+ setRequestStatus(+ existingDestination.pipelineId,+ PipelineStatusRequestStatus.StartRequested,+ snapshot+ )+ toast.success('Settings applied. Starting the pipeline...')+ startPipeline({ projectRef, pipelineId: existingDestination.pipelineId })+ }+ onClose()+ } else {+ let destinationConfig: DestinationConfig | undefined = undefined++ if (selectedType === 'BigQuery') {+ const bigQueryConfig: BigQueryDestinationConfig = {+ projectId: data.projectId ?? '',+ datasetId: data.datasetId ?? '',+ serviceAccountKey: data.serviceAccountKey ?? '',+ }+ if (!!data.maxStalenessMins) {+ bigQueryConfig.maxStalenessMins = data.maxStalenessMins+ }+ destinationConfig = { bigQuery: bigQueryConfig }+ } else if (selectedType === 'Analytics Bucket') {+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }++ if (data.s3AccessKeyId === CREATE_NEW_KEY) {+ const newKeys = await createS3AccessKey({+ projectRef,+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,+ })+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }+ }++ // Resolve namespace (create if needed)+ const finalNamespace = await resolveNamespace(data)++ const icebergConfig: IcebergDestinationConfig = {+ projectRef: projectRef,+ warehouseName: data.warehouseName ?? '',+ namespace: finalNamespace,+ catalogToken: data.catalogToken ?? '',+ s3AccessKeyId: s3Keys.accessKey ?? '',+ s3SecretAccessKey: s3Keys.secretKey ?? '',+ s3Region: data.s3Region ?? '',+ }+ destinationConfig = { iceberg: icebergConfig }+ }+ const batchConfig: BatchConfig | undefined = !!data.maxFillMs+ ? { maxFillMs: data.maxFillMs }+ : undefined+ const hasBatchFields = batchConfig !== undefined++ if (!destinationConfig) throw new Error('Destination configuration is missing')++ const { pipeline_id: pipelineId } = await createDestinationPipeline({+ projectRef,+ destinationName: data.name,+ destinationConfig,+ sourceId,+ pipelineConfig: {+ publicationName: data.publicationName,+ ...(hasBatchFields ? { batch: batchConfig } : {}),+ },+ })+ // Set request status only right before starting, then fire and close+ setRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)+ toast.success('Destination created. Starting the pipeline...')+ startPipeline({ projectRef, pipelineId })+ onClose()+ }+ } catch (error) {+ const action = editMode ? 'apply and run' : 'create and start'+ toast.error(`Failed to ${action} destination`)+ }+ }++ const isSaving =+ creatingDestinationPipeline ||+ updatingDestinationPipeline ||+ startingPipeline ||+ isCreatingS3AccessKey ||+ isCreatingNamespace++ const isSubmitDisabled =+ isSaving ||+ isSelectedPublicationMissing ||+ (!!selectedPublication && isLoadingCheck) ||+ hasTablesWithNoPrimaryKeys ||+ (!editMode && hasNoAvailableDestinations)++ const onSubmit = async (data: z.infer<typeof FormSchema>) => {+ if (!editMode) {+ setPendingFormValues(data)+ setShowDisclaimerDialog(true)+ return+ }++ await submitPipeline(data)+ }++ const handleDisclaimerDialogChange = (open: boolean) => {+ setShowDisclaimerDialog(open)+ if (!open) {+ setPendingFormValues(null)+ }+ }++ const handleDisclaimerConfirm = async () => {+ if (!pendingFormValues) return++ const values = pendingFormValues+ setPendingFormValues(null)+ setShowDisclaimerDialog(false)+ await submitPipeline(values)+ }++ useEffect(() => {+ if (editMode && destinationData && pipelineData && !isFormInteracting) {+ form.reset(defaultValues)+ }+ }, [destinationData, pipelineData, editMode, defaultValues, form, isFormInteracting])++ // Ensure the form always reflects the freshest data whenever the panel opens+ useEffect(() => {+ if (visible) {+ form.reset(defaultValues)+ setIsFormInteracting(false)+ }+ }, [visible, defaultValues, form])++ useEffect(() => {+ if (visible && projectRef && sourceId) {+ refetchPublications()+ }+ }, [visible, projectRef, sourceId, refetchPublications])++ return (+ <>+ <SheetSection className="flex-grow overflow-auto px-0 py-0">+ {hasNoAvailableDestinations && !editMode ? (+ <NoDestinationsAvailable />+ ) : (+ <Form_Shadcn_ {...form}>+ <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>+ <div className="p-5 flex flex-col gap-y-6">+ <p className="text-sm font-medium text-foreground">Destination details</p>++ <div className="space-y-4">+ <DestinationNameInput form={form} />+ <PublicationSelection+ form={form}+ sourceId={sourceId}+ visible={visible}+ onSelectNewPublication={() => setPublicationPanelVisible(true)}+ />+ </div>+ </div>++ <DialogSectionSeparator />++ {selectedType === 'BigQuery' && etlEnableBigQuery ? (+ <BigQueryFields form={form} />+ ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (+ <AnalyticsBucketFields+ form={form}+ setIsFormInteracting={setIsFormInteracting}+ onSelectNewBucket={() => setNewBucketSheetVisible(true)}+ />+ ) : null}++ <DialogSectionSeparator />++ <AdvancedSettings type={selectedType} form={form} />+ </form>+ </Form_Shadcn_>+ )}+ </SheetSection>++ <SheetFooter>+ <Button disabled={isSaving} type="default" onClick={onClose}>+ Cancel+ </Button>+ <Button disabled={isSubmitDisabled} loading={isSaving} form={formId} htmlType="submit">+ {editMode+ ? existingDestination?.enabled+ ? 'Apply and restart'+ : 'Apply and start'+ : 'Create and start'}+ </Button>+ </SheetFooter>++ <NewPublicationPanel+ sourceId={sourceId}+ visible={publicationPanelVisible}+ onClose={() => setPublicationPanelVisible(false)}+ />++ <CreateAnalyticsBucketSheet+ open={newBucketSheetVisible}+ onOpenChange={setNewBucketSheetVisible}+ />++ <ReplicationDisclaimerDialog+ open={showDisclaimerDialog}+ onOpenChange={handleDisclaimerDialogChange}+ isLoading={isSaving}+ onConfirm={handleDisclaimerConfirm}+ />+ </>+ )+}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.tsdeleted file mode 100644index 6050c7bc4c8..00000000000--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts+++ /dev/null@@ -1,69 +0,0 @@-import * as z from 'zod'--const types = ['BigQuery', 'Analytics Bucket'] as const-export const TypeEnum = z.enum(types)--// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components-// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other-export const DestinationPanelFormSchema = z- .object({- // Common fields- type: TypeEnum,- name: z.string().min(1, 'Name is required'),- publicationName: z.string().min(1, 'Publication is required'),- maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),- // BigQuery fields- projectId: z.string().optional(),- datasetId: z.string().optional(),- serviceAccountKey: z.string().optional(),- maxStalenessMins: z.number().nonnegative().optional(),- // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields- warehouseName: z.string().optional(),- namespace: z.string().optional(),- newNamespaceName: z.string().optional(),- catalogToken: z.string().optional(),- s3AccessKeyId: z.string().optional(),- s3SecretAccessKey: z.string().optional(),- s3Region: z.string().optional(),- })- .superRefine((data, ctx) => {- const addRequiredFieldError = (path: string, message: string) => {- ctx.addIssue({- code: z.ZodIssueCode.custom,- message,- path: [path],- })- }-- if (data.type === 'BigQuery') {- if (!data.projectId?.length) addRequiredFieldError('projectId', 'Project ID is required')- if (!data.datasetId?.length) addRequiredFieldError('datasetId', 'Dataset ID is required')- if (!data.serviceAccountKey?.length)- addRequiredFieldError('serviceAccountKey', 'Service Account Key is required')- } else if (data.type === 'Analytics Bucket') {- if (!data.warehouseName?.length) addRequiredFieldError('warehouseName', 'Bucket is required')-- const hasValidNamespace =- (data.namespace?.length && data.namespace !== 'create-new-namespace') ||- (data.namespace === 'create-new-namespace' && data.newNamespaceName?.length)-- if (!hasValidNamespace) {- const isCreatingNew = data.namespace === 'create-new-namespace'- addRequiredFieldError(- isCreatingNew ? 'newNamespaceName' : 'namespace',- isCreatingNew ? 'Namespace name is required' : 'Namespace is required'- )- }-- if (!data.s3Region?.length) addRequiredFieldError('s3Region', 'S3 Region is required')-- if (!data.s3AccessKeyId?.length)- addRequiredFieldError('s3AccessKeyId', 'S3 Access Key ID is required')-- if (data.s3AccessKeyId !== 'create-new' && !data.s3SecretAccessKey?.length) {- addRequiredFieldError('s3SecretAccessKey', 'S3 Secret Access Key is required')- }- }- })--export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsxindex 69ad59bb8c8..6747e252f11 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx@@ -1,61 +1,21 @@-import { zodResolver } from '@hookform/resolvers/zod'-import { PermissionAction } from '@supabase/shared-types/out/constants'-import { snakeCase } from 'lodash'-import { useEffect, useMemo, useState } from 'react'-import { useForm } from 'react-hook-form'-import { toast } from 'sonner'-import * as z from 'zod'+import { useState } from 'react'-import { useFlag, useParams } from 'common'-import { CreateAnalyticsBucketSheet } from 'components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'-import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'-import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'-import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query'-import { useCreateDestinationPipelineMutation } from 'data/replication/create-destination-pipeline-mutation'-import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'-import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id-query'-import { useReplicationPublicationsQuery } from 'data/replication/publications-query'-import { useRestartPipelineHelper } from 'data/replication/restart-pipeline-helper'-import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'-import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'-import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation'-import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'-import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'-import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'+import { useFlag } from 'common'import {- PipelineStatusRequestStatus,- usePipelineRequestStatus,-} from 'state/replication-pipeline-request-status'-import {- Button,cn,DialogSectionSeparator,- Form_Shadcn_,Sheet,SheetContent,SheetDescription,- SheetFooter,SheetHeader,- SheetSection,SheetTitle,} from 'ui'-import { NewPublicationPanel } from '../NewPublicationPanel'-import { ReplicationDisclaimerDialog } from '../ReplicationDisclaimerDialog'-import { AdvancedSettings } from './AdvancedSettings'-import { DestinationNameInput } from './DestinationNameInput'-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationPanel.constants'-import { DestinationPanelFormSchema as FormSchema, TypeEnum } from './DestinationPanel.schema'-import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'+import { DestinationForm } from './DestinationForm'+import { DestinationType } from './DestinationPanel.types'import { DestinationTypeSelection } from './DestinationTypeSelection'-import { NoDestinationsAvailable } from './NoDestinationsAvailable'-import { PublicationSelection } from './PublicationSelection'--const formId = 'destination-editor'interface DestinationPanelProps {visible: boolean- sourceId: number | undefined- onClose: () => voidexistingDestination?: {sourceId?: numberdestinationId: number@@ -63,379 +23,21 @@ interface DestinationPanelProps {enabled: booleanstatusName?: string}+ onClose: () => void}export const DestinationPanel = ({visible,- sourceId,- onClose,existingDestination,+ onClose,}: DestinationPanelProps) => {- const { ref: projectRef } = useParams()- const { data: project } = useSelectedProjectQuery()- const { setRequestStatus } = usePipelineRequestStatus()-- // Feature flags for ETL destinations- const etlEnableBigQuery = useFlag('etlEnableBigQuery')- const etlEnableIceberg = useFlag('etlEnableIceberg')-- // Compute available destinations based on feature flags- const availableDestinations = useMemo(() => {- const destinations = []- if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })- if (etlEnableIceberg)- destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })- return destinations- }, [etlEnableBigQuery, etlEnableIceberg])-- const hasNoAvailableDestinations = availableDestinations.length === 0-- const editMode = !!existingDestination- const [showDisclaimerDialog, setShowDisclaimerDialog] = useState(false)- const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)- const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)- const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(- null- )- const [isFormInteracting, setIsFormInteracting] = useState(false)-- const { mutateAsync: createDestinationPipeline, isPending: creatingDestinationPipeline } =- useCreateDestinationPipelineMutation({- onSuccess: () => form.reset(defaultValues),- })-- const { mutateAsync: updateDestinationPipeline, isPending: updatingDestinationPipeline } =- useUpdateDestinationPipelineMutation({- onSuccess: () => form.reset(defaultValues),- })-- const { mutateAsync: startPipeline, isPending: startingPipeline } = useStartPipelineMutation()- const { restartPipeline } = useRestartPipelineHelper()-- const { mutateAsync: createS3AccessKey, isPending: isCreatingS3AccessKey } =- useS3AccessKeyCreateMutation()-- const { mutateAsync: createNamespace, isPending: isCreatingNamespace } =- useIcebergNamespaceCreateMutation()-- const {- data: publications = [],- isSuccess: isSuccessPublications,- refetch: refetchPublications,- } = useReplicationPublicationsQuery({ projectRef, sourceId })+ const unifiedReplication = useFlag('unifiedReplication')- const { data: destinationData } = useReplicationDestinationByIdQuery({- projectRef,- destinationId: existingDestination?.destinationId,- })-- const { data: pipelineData } = useReplicationPipelineByIdQuery({- projectRef,- pipelineId: existingDestination?.pipelineId,- })-- const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')- const { data: apiKeys } = useAPIKeysQuery(- { projectRef, reveal: true },- { enabled: canReadAPIKeys }- )- const { serviceKey } = getKeys(apiKeys)- const catalogToken = serviceKey?.api_key ?? ''-- const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })-- const defaultValues = useMemo(() => {- const config = destinationData?.config- const isBigQueryConfig = config && 'big_query' in config- const isIcebergConfig = config && 'iceberg' in config-- const defaultType = editMode- ? isBigQueryConfig- ? TypeEnum.enum.BigQuery- : TypeEnum.enum['Analytics Bucket']- : (availableDestinations[0]?.value as z.infer<typeof TypeEnum>) || TypeEnum.enum.BigQuery-- return {- // Common fields- type: defaultType,- name: destinationData?.name ?? '',- publicationName: pipelineData?.config.publication_name ?? '',- maxFillMs: pipelineData?.config?.batch?.max_fill_ms,- // BigQuery fields- projectId: isBigQueryConfig ? config.big_query.project_id : '',- datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',- serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',- maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined,- // Analytics Bucket fields- warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',- namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',- newNamespaceName: '',- catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,- s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',- s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',- s3Region:- projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),- }- }, [- destinationData,- pipelineData,- catalogToken,- projectSettings,- editMode,- availableDestinations,- ])-- const form = useForm<z.infer<typeof FormSchema>>({- mode: 'onChange',- reValidateMode: 'onChange',- resolver: zodResolver(FormSchema),- defaultValues,- })- const { publicationName, type: selectedType, warehouseName } = form.watch()- const isSaving =- creatingDestinationPipeline ||- updatingDestinationPipeline ||- startingPipeline ||- isCreatingS3AccessKey ||- isCreatingNamespace-- const publicationNames = useMemo(() => publications?.map((pub) => pub.name) ?? [], [publications])- const isSelectedPublicationMissing =- isSuccessPublications && !!publicationName && !publicationNames.includes(publicationName)-- const selectedPublication = publications.find((pub) => pub.name === publicationName)- const { data: checkPrimaryKeysExistsData, isPending: isLoadingCheck } = useCheckPrimaryKeysExists(- {- projectRef: project?.ref,- connectionString: project?.connectionString,- tables: selectedPublication?.tables ?? [],- },- { enabled: visible && !!selectedPublication }+ const [selectedType, setSelectedType] = useState<DestinationType>(+ unifiedReplication ? 'Read Replica' : 'BigQuery')- const hasTablesWithNoPrimaryKeys = (checkPrimaryKeysExistsData?.offendingTables ?? []).length > 0-- const isSubmitDisabled =- isSaving ||- isSelectedPublicationMissing ||- (!!selectedPublication && isLoadingCheck) ||- hasTablesWithNoPrimaryKeys ||- (!editMode && hasNoAvailableDestinations)-- // Helper function to handle namespace creation if needed- const resolveNamespace = async (data: z.infer<typeof FormSchema>) => {- if (data.namespace === CREATE_NEW_NAMESPACE) {- if (!data.newNamespaceName) throw new Error('New namespace name is required')-- await createNamespace({- projectRef,- warehouse: data.warehouseName!,- namespace: data.newNamespaceName,- })-- return data.newNamespaceName- }- return data.namespace- }-- // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty- // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics- // At first glance we could try grouping as edit / create bigquery, edit / create analytics- // since the destination config seems rather similar between edit and create for the same type- const submitPipeline = async (data: z.infer<typeof FormSchema>) => {- if (!projectRef) return console.error('Project ref is required')- if (!sourceId) return console.error('Source id is required')- if (isSelectedPublicationMissing) {- return toast.error('Please select another publication before continuing')- }-- try {- if (editMode && existingDestination) {- if (!existingDestination.pipelineId) return console.error('Pipeline id is required')-- let destinationConfig: any = {}- if (data.type === 'BigQuery') {- const bigQueryConfig: any = {- projectId: data.projectId,- datasetId: data.datasetId,- serviceAccountKey: data.serviceAccountKey,- }- if (!!data.maxStalenessMins) {- bigQueryConfig.maxStalenessMins = data.maxStalenessMins- }- destinationConfig = { bigQuery: bigQueryConfig }- } else if (data.type === 'Analytics Bucket') {- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }-- if (data.s3AccessKeyId === CREATE_NEW_KEY) {- const newKeys = await createS3AccessKey({- projectRef,- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,- })- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }- }-- // Resolve namespace (create if needed)- const finalNamespace = await resolveNamespace(data)-- const icebergConfig: any = {- projectRef: projectRef,- warehouseName: data.warehouseName,- namespace: finalNamespace,- catalogToken: data.catalogToken,- s3AccessKeyId: s3Keys.accessKey,- s3SecretAccessKey: s3Keys.secretKey,- s3Region: data.s3Region,- }- destinationConfig = { iceberg: icebergConfig }- }-- const batchConfig: any = {}- if (!!data.maxFillMs) batchConfig.maxFillMs = data.maxFillMs- const hasBatchFields = Object.keys(batchConfig).length > 0-- await updateDestinationPipeline({- destinationId: existingDestination.destinationId,- pipelineId: existingDestination.pipelineId,- projectRef,- destinationName: data.name,- destinationConfig,- pipelineConfig: {- publicationName: data.publicationName,- ...(hasBatchFields ? { batch: batchConfig } : {}),- },- sourceId,- })- // Set request status only right before starting, then fire and close- const snapshot =- existingDestination.statusName ?? (existingDestination.enabled ? 'started' : 'stopped')- if (existingDestination.enabled) {- setRequestStatus(- existingDestination.pipelineId,- PipelineStatusRequestStatus.RestartRequested,- snapshot- )- toast.success('Settings applied. Restarting the pipeline...')- restartPipeline({ projectRef, pipelineId: existingDestination.pipelineId })- } else {- setRequestStatus(- existingDestination.pipelineId,- PipelineStatusRequestStatus.StartRequested,- snapshot- )- toast.success('Settings applied. Starting the pipeline...')- startPipeline({ projectRef, pipelineId: existingDestination.pipelineId })- }- onClose()- } else {- let destinationConfig: any = {}-- if (data.type === 'BigQuery') {- const bigQueryConfig: any = {- projectId: data.projectId,- datasetId: data.datasetId,- serviceAccountKey: data.serviceAccountKey,- }- if (!!data.maxStalenessMins) {- bigQueryConfig.maxStalenessMins = data.maxStalenessMins- }- destinationConfig = { bigQuery: bigQueryConfig }- } else if (data.type === 'Analytics Bucket') {- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }-- if (data.s3AccessKeyId === CREATE_NEW_KEY) {- const newKeys = await createS3AccessKey({- projectRef,- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,- })- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }- }-- // Resolve namespace (create if needed)- const finalNamespace = await resolveNamespace(data)-- const icebergConfig: any = {- projectRef: projectRef,- warehouseName: data.warehouseName,- namespace: finalNamespace,- catalogToken: data.catalogToken,- s3AccessKeyId: s3Keys.accessKey,- s3SecretAccessKey: s3Keys.secretKey,- s3Region: data.s3Region,- }- destinationConfig = { iceberg: icebergConfig }- }- const batchConfig: any = {}- if (!!data.maxFillMs) batchConfig.maxFillMs = data.maxFillMs- const hasBatchFields = Object.keys(batchConfig).length > 0-- const { pipeline_id: pipelineId } = await createDestinationPipeline({- projectRef,- destinationName: data.name,- destinationConfig,- sourceId,- pipelineConfig: {- publicationName: data.publicationName,- ...(hasBatchFields ? { batch: batchConfig } : {}),- },- })- // Set request status only right before starting, then fire and close- setRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)- toast.success('Destination created. Starting the pipeline...')- startPipeline({ projectRef, pipelineId })- onClose()- }- } catch (error) {- const action = editMode ? 'apply and run' : 'create and start'- toast.error(`Failed to ${action} destination`)- }- }-- const onSubmit = async (data: z.infer<typeof FormSchema>) => {- if (!editMode) {- setPendingFormValues(data)- setShowDisclaimerDialog(true)- return- }-- await submitPipeline(data)- }-- const handleDisclaimerDialogChange = (open: boolean) => {- setShowDisclaimerDialog(open)- if (!open) {- setPendingFormValues(null)- }- }-- const handleDisclaimerConfirm = async () => {- if (!pendingFormValues) return-- const values = pendingFormValues- setPendingFormValues(null)- setShowDisclaimerDialog(false)- await submitPipeline(values)- }-- useEffect(() => {- if (editMode && destinationData && pipelineData && !isFormInteracting) {- form.reset(defaultValues)- }- }, [destinationData, pipelineData, editMode, defaultValues, form, isFormInteracting])-- // Ensure the form always reflects the freshest data whenever the panel opens- useEffect(() => {- if (visible) {- form.reset(defaultValues)- setIsFormInteracting(false)- }- }, [visible, defaultValues, form])-- useEffect(() => {- if (visible && projectRef && sourceId) {- refetchPublications()- }- }, [visible, projectRef, sourceId, refetchPublications])+ const editMode = !!existingDestinationreturn (<>@@ -443,10 +45,7 @@ export const DestinationPanel = ({<SheetContentsize="default"showClose={false}- className={cn(- 'md:!w-[700px]',- publicationPanelVisible || newBucketSheetVisible ? 'right-20' : 'right-0'- )}+ className={cn(unifiedReplication ? 'md:!w-[850px]' : 'md:!w-[700px]')}><div className="flex flex-col h-full" tabIndex={-1}><SheetHeader>@@ -458,88 +57,29 @@ export const DestinationPanel = ({</SheetDescription></SheetHeader>- <SheetSection className="flex-grow overflow-auto px-0 py-0">- {hasNoAvailableDestinations && !editMode ? (- <NoDestinationsAvailable />- ) : (- <Form_Shadcn_ {...form}>- <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>- <DestinationTypeSelection form={form} editMode={editMode} />-- <DialogSectionSeparator />-- <div className="p-5 flex flex-col gap-y-6">- <p className="text-sm font-medium text-foreground">Destination details</p>-- <div className="space-y-4">- <DestinationNameInput form={form} />- <PublicationSelection- form={form}- sourceId={sourceId}- visible={visible}- onSelectNewPublication={() => setPublicationPanelVisible(true)}- />- </div>- </div>-- <DialogSectionSeparator />-- {selectedType === 'BigQuery' && etlEnableBigQuery ? (- <BigQueryFields form={form} />- ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (- <AnalyticsBucketFields- form={form}- setIsFormInteracting={setIsFormInteracting}- onSelectNewBucket={() => setNewBucketSheetVisible(true)}- />- ) : null}-- <DialogSectionSeparator />-- <AdvancedSettings form={form} />- </form>- </Form_Shadcn_>- )}- </SheetSection>-- <SheetFooter>- <Button disabled={isSaving} type="default" onClick={onClose}>- Cancel- </Button>- <Button- disabled={isSubmitDisabled}- loading={isSaving}- form={formId}- htmlType="submit"- >- {editMode- ? existingDestination?.enabled- ? 'Apply and restart'- : 'Apply and start'- : 'Create and start'}- </Button>- </SheetFooter>+ <DestinationTypeSelection+ editMode={editMode}+ selectedType={selectedType}+ setSelectedType={setSelectedType}+ />++ <DialogSectionSeparator />++ {selectedType === 'Read Replica' ? (+ <div className="w-full h-full flex items-center justify-center">+ <p className="text-foreground-lighter text-sm">Coming soon</p>+ </div>+ ) : (+ <DestinationForm+ visible={visible}+ selectedType={selectedType}+ existingDestination={existingDestination}+ onClose={onClose}+ />+ )}</div></SheetContent></Sheet>-- <NewPublicationPanel- sourceId={sourceId}- visible={publicationPanelVisible}- onClose={() => setPublicationPanelVisible(false)}- />-- <CreateAnalyticsBucketSheet- open={newBucketSheetVisible}- onOpenChange={setNewBucketSheetVisible}- />-- <ReplicationDisclaimerDialog- open={showDisclaimerDialog}- onOpenChange={handleDisclaimerDialogChange}- isLoading={isSaving}- onConfirm={handleDisclaimerConfirm}- /></>)}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.tsnew file mode 100644index 00000000000..657395b8b66--- /dev/null+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.ts@@ -0,0 +1 @@+export type DestinationType = 'Read Replica' | 'BigQuery' | 'Analytics Bucket'diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsxindex c7b9362a717..6f6e30b2a6e 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx@@ -1,26 +1,28 @@-import type { UseFormReturn } from 'react-hook-form'-import { useFlag } from 'common'-import { AnalyticsBucket, BigQuery } from 'icons'-import {- cn,- FormControl_Shadcn_,- FormField_Shadcn_,- RadioGroupStacked,- RadioGroupStackedItem,-} from 'ui'+import { AnalyticsBucket, BigQuery, Database } from 'icons'+import { cn, RadioGroupStacked, RadioGroupStackedItem } from 'ui'import { Admonition } from 'ui-patterns'-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'+import { DestinationType } from './DestinationPanel.types'type DestinationTypeSelectionProps = {- form: UseFormReturn<DestinationPanelSchemaType>editMode: boolean+ selectedType: DestinationType+ setSelectedType: (value: DestinationType) => void}-export const DestinationTypeSelection = ({ form, editMode }: DestinationTypeSelectionProps) => {+export const DestinationTypeSelection = ({+ editMode,+ selectedType,+ setSelectedType,+}: DestinationTypeSelectionProps) => {+ const unifiedReplication = useFlag('unifiedReplication')const etlEnableBigQuery = useFlag('etlEnableBigQuery')const etlEnableIceberg = useFlag('etlEnableIceberg')+ const numberOfTypes = [unifiedReplication, etlEnableBigQuery, etlEnableIceberg].filter(+ Boolean+ ).length+return (<div className="px-5 py-5"><div className="flex flex-col gap-y-2 mb-4">@@ -32,64 +34,71 @@ export const DestinationTypeSelection = ({ form, editMode }: DestinationTypeSele/>)}</div>- <FormField_Shadcn_- name="type"- control={form.control}- render={({ field }) => (- <FormControl_Shadcn_>- <RadioGroupStacked- disabled={editMode}- value={field.value}- onValueChange={(value) => field.onChange(value)}- className={cn(- 'grid grid-cols-2 [&>button>div]:py-4',- '[&>button:first-of-type]:rounded-none [&>button:last-of-type]:rounded-none',- '[&>button:first-of-type]:!rounded-l-lg [&>button:last-of-type]:!rounded-r-lg'- )}- >- {((!editMode && etlEnableBigQuery) || (editMode && field.value === 'BigQuery')) && (- <RadioGroupStackedItem- label=""- showIndicator={false}- id="BigQuery"- value="BigQuery"- >- <div className="flex flex-col gap-y-2">- <BigQuery size={20} />- <div className="flex flex-col gap-y-0.5 text-sm text-left">- <p>BigQuery</p>- <p className="text-foreground-lighter">- Send data to Google Cloud's data warehouse for analytics and business- intelligence- </p>- </div>- </div>- </RadioGroupStackedItem>- )}- {((!editMode && etlEnableIceberg) ||- (editMode && field.value === 'Analytics Bucket')) && (- <RadioGroupStackedItem- label=""- showIndicator={false}- id="Analytics Bucket"- value="Analytics Bucket"- >- <div className="flex flex-col gap-y-2">- <AnalyticsBucket size={20} />- <div className="flex flex-col gap-y-0.5 text-sm text-left">- <p>Analytics Bucket</p>- <p className="text-foreground-lighter">- Send data to Apache Iceberg tables in your Supabase Storage for flexible- analytics workflows- </p>- </div>- </div>- </RadioGroupStackedItem>- )}- </RadioGroupStacked>- </FormControl_Shadcn_>+ <RadioGroupStacked+ disabled={editMode}+ value={selectedType}+ onValueChange={(value) => setSelectedType(value as DestinationType)}+ className={cn(+ 'grid [&>button>div]:py-4',+ numberOfTypes === 3 ? 'grid-cols-3' : numberOfTypes === 2 ? 'grid-cols-2' : 'grid-cols-1',+ '[&>button:first-of-type]:rounded-none [&>button:last-of-type]:rounded-none',+ '[&>button:first-of-type]:!rounded-l-lg [&>button:last-of-type]:!rounded-r-lg'+ )}+ >+ {((!editMode && unifiedReplication) || (editMode && selectedType === 'Read Replica')) && (+ <RadioGroupStackedItem+ label=""+ showIndicator={false}+ id="Read Replica"+ value="Read Replica"+ >+ <div className="flex flex-col gap-y-2">+ <Database size={20} />+ <div className="flex flex-col gap-y-0.5 text-sm text-left">+ <p>Read Replica</p>+ <p className="text-foreground-lighter">+ Deploy read-only databases across multiple regions for lower latency and better+ resource management+ </p>+ </div>+ </div>+ </RadioGroupStackedItem>+ )}++ {((!editMode && etlEnableBigQuery) || (editMode && selectedType === 'BigQuery')) && (+ <RadioGroupStackedItem label="" showIndicator={false} id="BigQuery" value="BigQuery">+ <div className="flex flex-col gap-y-2">+ <BigQuery size={20} />+ <div className="flex flex-col gap-y-0.5 text-sm text-left">+ <p>BigQuery</p>+ <p className="text-foreground-lighter">+ Send data to Google Cloud's data warehouse for analytics and business intelligence+ </p>+ </div>+ </div>+ </RadioGroupStackedItem>+ )}++ {((!editMode && etlEnableIceberg) || (editMode && selectedType === 'Analytics Bucket')) && (+ <RadioGroupStackedItem+ label=""+ showIndicator={false}+ id="Analytics Bucket"+ value="Analytics Bucket"+ >+ <div className="flex flex-col gap-y-2">+ <AnalyticsBucket size={20} />+ <div className="flex flex-col gap-y-0.5 text-sm text-left">+ <p>Analytics Bucket</p>+ <p className="text-foreground-lighter">+ Send data to Apache Iceberg tables in your Supabase Storage for flexible analytics+ workflows+ </p>+ </div>+ </div>+ </RadioGroupStackedItem>)}- />+ </RadioGroupStacked></div>)}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsxindex 5e818630956..b3dbb5da599 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx@@ -21,7 +21,7 @@ import { Button, Tooltip, TooltipContent, TooltipTrigger } from 'ui'import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader'import { DeleteDestination } from './DeleteDestination'import { DestinationPanel } from './DestinationPanel/DestinationPanel'-import { DestinationPanelSchemaType } from './DestinationPanel/DestinationPanel.schema'+import { DestinationType } from './DestinationPanel/DestinationPanel.types'import { getStatusName, PIPELINE_ERROR_MESSAGES } from './Pipeline.utils'import { PipelineStatus } from './PipelineStatus'import { PipelineStatusName, STATUS_REFRESH_FREQUENCY_MS } from './Replication.constants'@@ -32,7 +32,7 @@ interface DestinationRowProps {sourceId?: numberdestinationId: numberdestinationName: string- type: DestinationPanelSchemaType['type'] | 'Other'+ type: DestinationType | 'Other'pipeline?: Pipelineerror: ResponseError | nullisLoading: boolean@@ -223,7 +223,6 @@ export const DestinationRow = ({<DestinationPanelvisible={showEditDestinationPanel}onClose={() => setShowEditDestinationPanel(false)}- sourceId={sourceId}existingDestination={{sourceId,destinationId: destinationId,diff --git a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx b/apps/studio/components/interfaces/Database/Replication/Destinations.tsxindex 06cd96e310a..512b905fdc9 100644--- a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx+++ b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx@@ -230,7 +230,6 @@ export const Destinations = () => {<DestinationPanelvisible={showNewDestinationPanel}- sourceId={sourceId}onClose={() => setShowNewDestinationPanel(false)}/></>diff --git a/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx b/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsxindex 1210d9e6dea..65f18ec09ad 100644--- a/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx+++ b/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx@@ -1,4 +1,5 @@-import { useParams } from 'common'+import { useFlag, useParams } from 'common'+import { NoticeBar } from 'components/interfaces/DiskManagement/ui/NoticeBar'import {ScaffoldContainer,ScaffoldDivider,@@ -12,11 +13,13 @@ import { useProjectServiceVersionsQuery } from 'data/projects/project-service-veimport { useReadReplicasQuery } from 'data/read-replicas/replicas-query'import { useIsFeatureEnabled } from 'hooks/misc/useIsFeatureEnabled'import { useIsOrioleDb, useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'+import Link from 'next/link'import {AlertDescription_Shadcn_,AlertTitle_Shadcn_,Alert_Shadcn_,Badge,+ Button,Input,Tooltip,TooltipContent,@@ -36,6 +39,8 @@ const InfrastructureInfo = () => {const { ref } = useParams()const { data: project } = useSelectedProjectQuery()+ const unifiedReplication = useFlag('unifiedReplication')+const { projectAuthAll: authEnabled, projectSettingsDatabaseUpgrades: showDatabaseUpgrades } =useIsFeatureEnabled(['project_auth:all', 'project_settings:database_upgrades'])@@ -83,12 +88,31 @@ const InfrastructureInfo = () => {return (<><ScaffoldDivider />- {project?.cloud_provider !== 'FLY' && (- <>- <InstanceConfiguration />- <ScaffoldDivider />- </>- )}+ {project?.cloud_provider !== 'FLY' &&+ (unifiedReplication ? (+ <ScaffoldContainer>+ <ScaffoldSection isFullWidth>+ <NoticeBar+ visible={true}+ type="default"+ title="Management of read replicas has moved"+ description="Read replicas is now managed under Replication in the Database section."+ actions={+ <Button type="default" asChild>+ <Link href={`/project/${ref}/database/replication`} className="!no-underline">+ Go to Replication+ </Link>+ </Button>+ }+ />+ </ScaffoldSection>+ </ScaffoldContainer>+ ) : (+ <>+ <InstanceConfiguration />+ <ScaffoldDivider />+ </>+ ))}<ScaffoldContainer><ScaffoldSection>diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex 78476e137d7..8141dac26e6 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -6,6 +6,14 @@ import { handleError, post } from 'data/fetchers'import type { ResponseError, UseCustomMutationOptions } from 'types'import { replicationKeys } from './keys'+export type DestinationConfig =+ | {+ bigQuery: BigQueryDestinationConfig+ }+ | {+ iceberg: IcebergDestinationConfig+ }+export type BigQueryDestinationConfig = {projectId: stringdatasetId: string@@ -23,22 +31,18 @@ export type IcebergDestinationConfig = {s3Region: string}+export type BatchConfig = {+ maxFillMs: number+}+export type CreateDestinationPipelineParams = {projectRef: stringdestinationName: string- destinationConfig:- | {- bigQuery: BigQueryDestinationConfig- }- | {- iceberg: IcebergDestinationConfig- }+ destinationConfig: DestinationConfigsourceId: numberpipelineConfig: {publicationName: string- batch?: {- maxFillMs: number- }+ batch?: BatchConfig}}diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex ae7def269b4..12da1fcd1db 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -4,43 +4,19 @@ import { toast } from 'sonner'import type { components } from 'api-types'import { handleError, post } from 'data/fetchers'import type { ResponseError, UseCustomMutationOptions } from 'types'+import { BatchConfig, DestinationConfig } from './create-destination-pipeline-mutation'import { replicationKeys } from './keys'-export type BigQueryDestinationConfig = {- projectId: string- datasetId: string- serviceAccountKey: string- maxStalenessMins?: number-}--export type IcebergDestinationConfig = {- projectRef: string- warehouseName: string- namespace: string- catalogToken: string- s3AccessKeyId: string- s3SecretAccessKey: string- s3Region: string-}-export type UpdateDestinationPipelineParams = {destinationId: numberpipelineId: numberprojectRef: stringdestinationName: string- destinationConfig:- | {- bigQuery: BigQueryDestinationConfig- }- | {- iceberg: IcebergDestinationConfig- }+ destinationConfig: DestinationConfigsourceId: numberpipelineConfig: {publicationName: string- batch?: {- maxFillMs: number- }+ batch?: BatchConfig}}