Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

Shift ETL destination components into a separate folder (#41457)

Joshen Lim31d2e8ac5b6b
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
similarity index 93%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
index 11dc657b876..f3260a82d46 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/AdvancedSettings.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
@@ -12,11 +12,16 @@ import {
Input_Shadcn_,
} from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'
-
-export const AdvancedSettings = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
- const { type } = form.watch()
+import { DestinationType } from '../DestinationPanel.types'
+import { type DestinationPanelSchemaType } from './DestinationForm.schema'
+export const AdvancedSettings = ({
+ type,
+ form,
+}: {
+ type: DestinationType
+ form: UseFormReturn<DestinationPanelSchemaType>
+}) => {
const handleNumberChange =
(field: { onChange: (value?: number) => void }) => (e: ChangeEvent<HTMLInputElement>) => {
const val = e.target.value
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.constants.ts
similarity index 100%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.constants.ts
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.constants.ts
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
new file mode 100644
index 00000000000..97e09b56a95
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
@@ -0,0 +1,25 @@
+import * as z from 'zod'
+
+// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components
+// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other
+export const DestinationPanelFormSchema = z.object({
+ // Common fields
+ name: z.string().min(1, 'Name is required'),
+ publicationName: z.string().min(1, 'Publication is required'),
+ maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),
+ // BigQuery fields
+ projectId: z.string().optional(),
+ datasetId: z.string().optional(),
+ serviceAccountKey: z.string().optional(),
+ maxStalenessMins: z.number().nonnegative().optional(),
+ // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
+ warehouseName: z.string().optional(),
+ namespace: z.string().optional(),
+ newNamespaceName: z.string().optional(),
+ catalogToken: z.string().optional(),
+ s3AccessKeyId: z.string().optional(),
+ s3SecretAccessKey: z.string().optional(),
+ s3Region: z.string().optional(),
+})
+
+export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsx
similarity index 90%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsx
index d444f9ae624..2a39aaf8f6d 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationNameInput.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationNameInput.tsx
@@ -2,7 +2,7 @@ import type { UseFormReturn } from 'react-hook-form'
import { FormControl_Shadcn_, FormField_Shadcn_, Input_Shadcn_ } from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
type DestinationNameInputProps = {
form: UseFormReturn<DestinationPanelSchemaType>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsx
similarity index 98%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsx
index 920a19f7614..9b1003c52cf 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanelFields.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationPanelFields.tsx
@@ -27,8 +27,8 @@ import {
import { Admonition } from 'ui-patterns'
import { Input } from 'ui-patterns/DataInputs/Input'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationPanel.constants'
-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'
+import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
return (
@@ -114,7 +114,7 @@ export const AnalyticsBucketFields = ({
setIsFormInteracting: (value: boolean) => void
onSelectNewBucket: () => void
}) => {
- const { warehouseName, type, s3AccessKeyId, namespace } = form.watch()
+ const { warehouseName, s3AccessKeyId, namespace } = form.watch()
const [showCatalogToken, setShowCatalogToken] = useState(false)
const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
@@ -153,13 +153,8 @@ export const AnalyticsBucketFields = ({
isPending: isLoadingNamespaces,
isError: isErrorNamespaces,
} = useIcebergNamespacesQuery(
- {
- projectRef,
- warehouse: warehouseName,
- },
- {
- enabled: type === 'Analytics Bucket' && !!serviceApiKey,
- }
+ { projectRef, warehouse: warehouseName },
+ { enabled: !!serviceApiKey }
)
return (
diff --git a/apps/studio/components/interfaces/Database/Replication/NewPublicationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx
similarity index 100%
rename from apps/studio/components/interfaces/Database/Replication/NewPublicationPanel.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NewPublicationPanel.tsx
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/NoDestinationsAvailable.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NoDestinationsAvailable.tsx
similarity index 100%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/NoDestinationsAvailable.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/NoDestinationsAvailable.tsx
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
similarity index 96%
rename from apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
index 0c3da86f327..2df4f0e2de9 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/PublicationSelection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationSelection.tsx
@@ -9,8 +9,8 @@ import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
import { FormControl_Shadcn_, FormField_Shadcn_ } from 'ui'
import { Admonition } from 'ui-patterns'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
-import { PublicationsComboBox } from '../PublicationsComboBox'
-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'
+import type { DestinationPanelSchemaType } from './DestinationForm.schema'
+import { PublicationsComboBox } from './PublicationsComboBox'
type PublicationSelectionProps = {
form: UseFormReturn<DestinationPanelSchemaType>
diff --git a/apps/studio/components/interfaces/Database/Replication/PublicationsComboBox.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationsComboBox.tsx
similarity index 100%
rename from apps/studio/components/interfaces/Database/Replication/PublicationsComboBox.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PublicationsComboBox.tsx
diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationDisclaimerDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ReplicationDisclaimerDialog.tsx
similarity index 100%
rename from apps/studio/components/interfaces/Database/Replication/ReplicationDisclaimerDialog.tsx
rename to apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/ReplicationDisclaimerDialog.tsx
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
new file mode 100644
index 00000000000..5783b7e8af1
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
@@ -0,0 +1,546 @@
+import { zodResolver } from '@hookform/resolvers/zod'
+import { PermissionAction } from '@supabase/shared-types/out/constants'
+import { snakeCase } from 'lodash'
+import { useEffect, useMemo, useState } from 'react'
+import { useForm } from 'react-hook-form'
+import { toast } from 'sonner'
+import * as z from 'zod'
+
+import { useFlag, useParams } from 'common'
+import { CreateAnalyticsBucketSheet } from 'components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
+import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'
+import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'
+import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query'
+import {
+ BatchConfig,
+ BigQueryDestinationConfig,
+ DestinationConfig,
+ IcebergDestinationConfig,
+ useCreateDestinationPipelineMutation,
+} from 'data/replication/create-destination-pipeline-mutation'
+import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'
+import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id-query'
+import { useReplicationPublicationsQuery } from 'data/replication/publications-query'
+import { useRestartPipelineHelper } from 'data/replication/restart-pipeline-helper'
+import { useReplicationSourcesQuery } from 'data/replication/sources-query'
+import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'
+import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'
+import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation'
+import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'
+import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'
+import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
+import {
+ PipelineStatusRequestStatus,
+ usePipelineRequestStatus,
+} from 'state/replication-pipeline-request-status'
+import { Button, DialogSectionSeparator, Form_Shadcn_, SheetFooter, SheetSection } from 'ui'
+import { DestinationType } from '../DestinationPanel.types'
+import { AdvancedSettings } from './AdvancedSettings'
+import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
+import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
+import { DestinationNameInput } from './DestinationNameInput'
+import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'
+import { NewPublicationPanel } from './NewPublicationPanel'
+import { NoDestinationsAvailable } from './NoDestinationsAvailable'
+import { PublicationSelection } from './PublicationSelection'
+import { ReplicationDisclaimerDialog } from './ReplicationDisclaimerDialog'
+
+const formId = 'destination-editor'
+
+interface DestinationFormProps {
+ selectedType: DestinationType
+ visible: boolean
+ existingDestination?: {
+ sourceId?: number
+ destinationId: number
+ pipelineId?: number
+ enabled: boolean
+ statusName?: string
+ }
+ onClose: () => void
+}
+
+export const DestinationForm = ({
+ selectedType,
+ visible,
+ existingDestination,
+ onClose,
+}: DestinationFormProps) => {
+ const { ref: projectRef } = useParams()
+ const { data: project } = useSelectedProjectQuery()
+ const { setRequestStatus } = usePipelineRequestStatus()
+
+ const etlEnableBigQuery = useFlag('etlEnableBigQuery')
+ const etlEnableIceberg = useFlag('etlEnableIceberg')
+ const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
+
+ const [isFormInteracting, setIsFormInteracting] = useState(false)
+ const [showDisclaimerDialog, setShowDisclaimerDialog] = useState(false)
+ const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
+ const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)
+ const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
+ null
+ )
+
+ const editMode = !!existingDestination
+
+ // Compute available destinations based on feature flags
+ const availableDestinations = useMemo(() => {
+ const destinations = []
+ if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })
+ if (etlEnableIceberg)
+ destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })
+ return destinations
+ }, [etlEnableBigQuery, etlEnableIceberg])
+ const hasNoAvailableDestinations = availableDestinations.length === 0
+
+ const { data: sourcesData } = useReplicationSourcesQuery({ projectRef })
+ const sourceId = sourcesData?.sources.find((s) => s.name === projectRef)?.id
+
+ const {
+ data: publications = [],
+ isSuccess: isSuccessPublications,
+ refetch: refetchPublications,
+ } = useReplicationPublicationsQuery({ projectRef, sourceId })
+
+ const { data: destinationData } = useReplicationDestinationByIdQuery({
+ projectRef,
+ destinationId: existingDestination?.destinationId,
+ })
+
+ const { data: pipelineData } = useReplicationPipelineByIdQuery({
+ projectRef,
+ pipelineId: existingDestination?.pipelineId,
+ })
+
+ const { data: apiKeys } = useAPIKeysQuery(
+ { projectRef, reveal: true },
+ { enabled: canReadAPIKeys }
+ )
+ const { serviceKey } = getKeys(apiKeys)
+ const catalogToken = serviceKey?.api_key ?? ''
+
+ const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })
+
+ const { mutateAsync: createDestinationPipeline, isPending: creatingDestinationPipeline } =
+ useCreateDestinationPipelineMutation({
+ onSuccess: () => form.reset(defaultValues),
+ })
+
+ const { mutateAsync: updateDestinationPipeline, isPending: updatingDestinationPipeline } =
+ useUpdateDestinationPipelineMutation({
+ onSuccess: () => form.reset(defaultValues),
+ })
+
+ const { mutateAsync: startPipeline, isPending: startingPipeline } = useStartPipelineMutation()
+ const { restartPipeline } = useRestartPipelineHelper()
+
+ const { mutateAsync: createS3AccessKey, isPending: isCreatingS3AccessKey } =
+ useS3AccessKeyCreateMutation()
+
+ const { mutateAsync: createNamespace, isPending: isCreatingNamespace } =
+ useIcebergNamespaceCreateMutation()
+
+ const defaultValues = useMemo(() => {
+ const config = destinationData?.config
+ const isBigQueryConfig = config && 'big_query' in config
+ const isIcebergConfig = config && 'iceberg' in config
+
+ return {
+ // Common fields
+ name: destinationData?.name ?? '',
+ publicationName: pipelineData?.config.publication_name ?? '',
+ maxFillMs: pipelineData?.config?.batch?.max_fill_ms,
+ // BigQuery fields
+ projectId: isBigQueryConfig ? config.big_query.project_id : '',
+ datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
+ serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
+ maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined,
+ // Analytics Bucket fields
+ warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
+ namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',
+ newNamespaceName: '',
+ catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,
+ s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',
+ s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',
+ s3Region:
+ projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),
+ }
+ }, [destinationData, pipelineData, catalogToken, projectSettings])
+
+ const form = useForm<z.infer<typeof FormSchema>>({
+ mode: 'onChange',
+ reValidateMode: 'onChange',
+ resolver: zodResolver(
+ FormSchema.superRefine((data, ctx) => {
+ const addRequiredFieldError = (path: string, message: string) => {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ message,
+ path: [path],
+ })
+ }
+
+ if (selectedType === 'BigQuery') {
+ if (!data.projectId?.length) addRequiredFieldError('projectId', 'Project ID is required')
+ if (!data.datasetId?.length) addRequiredFieldError('datasetId', 'Dataset ID is required')
+ if (!data.serviceAccountKey?.length)
+ addRequiredFieldError('serviceAccountKey', 'Service Account Key is required')
+ } else if (selectedType === 'Analytics Bucket') {
+ if (!data.warehouseName?.length)
+ addRequiredFieldError('warehouseName', 'Bucket is required')
+
+ const hasValidNamespace =
+ (data.namespace?.length && data.namespace !== 'create-new-namespace') ||
+ (data.namespace === 'create-new-namespace' && data.newNamespaceName?.length)
+
+ if (!hasValidNamespace) {
+ const isCreatingNew = data.namespace === 'create-new-namespace'
+ addRequiredFieldError(
+ isCreatingNew ? 'newNamespaceName' : 'namespace',
+ isCreatingNew ? 'Namespace name is required' : 'Namespace is required'
+ )
+ }
+
+ if (!data.s3Region?.length) addRequiredFieldError('s3Region', 'S3 Region is required')
+
+ if (!data.s3AccessKeyId?.length)
+ addRequiredFieldError('s3AccessKeyId', 'S3 Access Key ID is required')
+
+ if (data.s3AccessKeyId !== 'create-new' && !data.s3SecretAccessKey?.length) {
+ addRequiredFieldError('s3SecretAccessKey', 'S3 Secret Access Key is required')
+ }
+ }
+ })
+ ),
+ defaultValues,
+ })
+
+ const { publicationName, warehouseName } = form.watch()
+
+ const publicationNames = useMemo(() => publications?.map((pub) => pub.name) ?? [], [publications])
+ const isSelectedPublicationMissing =
+ isSuccessPublications && !!publicationName && !publicationNames.includes(publicationName)
+ const selectedPublication = publications.find((pub) => pub.name === publicationName)
+
+ const { data: checkPrimaryKeysExistsData, isPending: isLoadingCheck } = useCheckPrimaryKeysExists(
+ {
+ projectRef: project?.ref,
+ connectionString: project?.connectionString,
+ tables: selectedPublication?.tables ?? [],
+ },
+ { enabled: visible && !!selectedPublication }
+ )
+ const hasTablesWithNoPrimaryKeys = (checkPrimaryKeysExistsData?.offendingTables ?? []).length > 0
+
+ // Helper function to handle namespace creation if needed
+ const resolveNamespace = async (data: z.infer<typeof FormSchema>) => {
+ if (data.namespace === CREATE_NEW_NAMESPACE) {
+ if (!data.newNamespaceName) throw new Error('New namespace name is required')
+
+ await createNamespace({
+ projectRef,
+ warehouse: data.warehouseName!,
+ namespace: data.newNamespaceName,
+ })
+
+ return data.newNamespaceName
+ }
+ return data.namespace
+ }
+
+ // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty
+ // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics
+ // At first glance we could try grouping as edit / create bigquery, edit / create analytics
+ // since the destination config seems rather similar between edit and create for the same type
+ const submitPipeline = async (data: z.infer<typeof FormSchema>) => {
+ if (!projectRef) return console.error('Project ref is required')
+ if (!sourceId) return console.error('Source id is required')
+ if (isSelectedPublicationMissing) {
+ return toast.error('Please select another publication before continuing')
+ }
+
+ try {
+ if (editMode && existingDestination) {
+ if (!existingDestination.pipelineId) return console.error('Pipeline id is required')
+
+ let destinationConfig: DestinationConfig | undefined = undefined
+
+ if (selectedType === 'BigQuery') {
+ const bigQueryConfig: BigQueryDestinationConfig = {
+ projectId: data.projectId ?? '',
+ datasetId: data.datasetId ?? '',
+ serviceAccountKey: data.serviceAccountKey ?? '',
+ }
+ if (!!data.maxStalenessMins) {
+ bigQueryConfig.maxStalenessMins = data.maxStalenessMins
+ }
+ destinationConfig = { bigQuery: bigQueryConfig }
+ } else if (selectedType === 'Analytics Bucket') {
+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
+
+ if (data.s3AccessKeyId === CREATE_NEW_KEY) {
+ const newKeys = await createS3AccessKey({
+ projectRef,
+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
+ })
+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
+ }
+
+ // Resolve namespace (create if needed)
+ const finalNamespace = await resolveNamespace(data)
+
+ const icebergConfig: IcebergDestinationConfig = {
+ projectRef: projectRef,
+ warehouseName: data.warehouseName ?? '',
+ namespace: finalNamespace,
+ catalogToken: data.catalogToken ?? '',
+ s3AccessKeyId: s3Keys.accessKey ?? '',
+ s3SecretAccessKey: s3Keys.secretKey ?? '',
+ s3Region: data.s3Region ?? '',
+ }
+ destinationConfig = { iceberg: icebergConfig }
+ }
+
+ const batchConfig: BatchConfig | undefined = !!data.maxFillMs
+ ? { maxFillMs: data.maxFillMs }
+ : undefined
+ const hasBatchFields = batchConfig !== undefined
+
+ if (!destinationConfig) throw new Error('Destination configuration is missing')
+
+ await updateDestinationPipeline({
+ destinationId: existingDestination.destinationId,
+ pipelineId: existingDestination.pipelineId,
+ projectRef,
+ destinationName: data.name,
+ destinationConfig,
+ pipelineConfig: {
+ publicationName: data.publicationName,
+ ...(hasBatchFields ? { batch: batchConfig } : {}),
+ },
+ sourceId,
+ })
+ // Set request status only right before starting, then fire and close
+ const snapshot =
+ existingDestination.statusName ?? (existingDestination.enabled ? 'started' : 'stopped')
+ if (existingDestination.enabled) {
+ setRequestStatus(
+ existingDestination.pipelineId,
+ PipelineStatusRequestStatus.RestartRequested,
+ snapshot
+ )
+ toast.success('Settings applied. Restarting the pipeline...')
+ restartPipeline({ projectRef, pipelineId: existingDestination.pipelineId })
+ } else {
+ setRequestStatus(
+ existingDestination.pipelineId,
+ PipelineStatusRequestStatus.StartRequested,
+ snapshot
+ )
+ toast.success('Settings applied. Starting the pipeline...')
+ startPipeline({ projectRef, pipelineId: existingDestination.pipelineId })
+ }
+ onClose()
+ } else {
+ let destinationConfig: DestinationConfig | undefined = undefined
+
+ if (selectedType === 'BigQuery') {
+ const bigQueryConfig: BigQueryDestinationConfig = {
+ projectId: data.projectId ?? '',
+ datasetId: data.datasetId ?? '',
+ serviceAccountKey: data.serviceAccountKey ?? '',
+ }
+ if (!!data.maxStalenessMins) {
+ bigQueryConfig.maxStalenessMins = data.maxStalenessMins
+ }
+ destinationConfig = { bigQuery: bigQueryConfig }
+ } else if (selectedType === 'Analytics Bucket') {
+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
+
+ if (data.s3AccessKeyId === CREATE_NEW_KEY) {
+ const newKeys = await createS3AccessKey({
+ projectRef,
+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
+ })
+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
+ }
+
+ // Resolve namespace (create if needed)
+ const finalNamespace = await resolveNamespace(data)
+
+ const icebergConfig: IcebergDestinationConfig = {
+ projectRef: projectRef,
+ warehouseName: data.warehouseName ?? '',
+ namespace: finalNamespace,
+ catalogToken: data.catalogToken ?? '',
+ s3AccessKeyId: s3Keys.accessKey ?? '',
+ s3SecretAccessKey: s3Keys.secretKey ?? '',
+ s3Region: data.s3Region ?? '',
+ }
+ destinationConfig = { iceberg: icebergConfig }
+ }
+ const batchConfig: BatchConfig | undefined = !!data.maxFillMs
+ ? { maxFillMs: data.maxFillMs }
+ : undefined
+ const hasBatchFields = batchConfig !== undefined
+
+ if (!destinationConfig) throw new Error('Destination configuration is missing')
+
+ const { pipeline_id: pipelineId } = await createDestinationPipeline({
+ projectRef,
+ destinationName: data.name,
+ destinationConfig,
+ sourceId,
+ pipelineConfig: {
+ publicationName: data.publicationName,
+ ...(hasBatchFields ? { batch: batchConfig } : {}),
+ },
+ })
+ // Set request status only right before starting, then fire and close
+ setRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)
+ toast.success('Destination created. Starting the pipeline...')
+ startPipeline({ projectRef, pipelineId })
+ onClose()
+ }
+ } catch (error) {
+ const action = editMode ? 'apply and run' : 'create and start'
+ toast.error(`Failed to ${action} destination`)
+ }
+ }
+
+ const isSaving =
+ creatingDestinationPipeline ||
+ updatingDestinationPipeline ||
+ startingPipeline ||
+ isCreatingS3AccessKey ||
+ isCreatingNamespace
+
+ const isSubmitDisabled =
+ isSaving ||
+ isSelectedPublicationMissing ||
+ (!!selectedPublication && isLoadingCheck) ||
+ hasTablesWithNoPrimaryKeys ||
+ (!editMode && hasNoAvailableDestinations)
+
+ const onSubmit = async (data: z.infer<typeof FormSchema>) => {
+ if (!editMode) {
+ setPendingFormValues(data)
+ setShowDisclaimerDialog(true)
+ return
+ }
+
+ await submitPipeline(data)
+ }
+
+ const handleDisclaimerDialogChange = (open: boolean) => {
+ setShowDisclaimerDialog(open)
+ if (!open) {
+ setPendingFormValues(null)
+ }
+ }
+
+ const handleDisclaimerConfirm = async () => {
+ if (!pendingFormValues) return
+
+ const values = pendingFormValues
+ setPendingFormValues(null)
+ setShowDisclaimerDialog(false)
+ await submitPipeline(values)
+ }
+
+ useEffect(() => {
+ if (editMode && destinationData && pipelineData && !isFormInteracting) {
+ form.reset(defaultValues)
+ }
+ }, [destinationData, pipelineData, editMode, defaultValues, form, isFormInteracting])
+
+ // Ensure the form always reflects the freshest data whenever the panel opens
+ useEffect(() => {
+ if (visible) {
+ form.reset(defaultValues)
+ setIsFormInteracting(false)
+ }
+ }, [visible, defaultValues, form])
+
+ useEffect(() => {
+ if (visible && projectRef && sourceId) {
+ refetchPublications()
+ }
+ }, [visible, projectRef, sourceId, refetchPublications])
+
+ return (
+ <>
+ <SheetSection className="flex-grow overflow-auto px-0 py-0">
+ {hasNoAvailableDestinations && !editMode ? (
+ <NoDestinationsAvailable />
+ ) : (
+ <Form_Shadcn_ {...form}>
+ <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
+ <div className="p-5 flex flex-col gap-y-6">
+ <p className="text-sm font-medium text-foreground">Destination details</p>
+
+ <div className="space-y-4">
+ <DestinationNameInput form={form} />
+ <PublicationSelection
+ form={form}
+ sourceId={sourceId}
+ visible={visible}
+ onSelectNewPublication={() => setPublicationPanelVisible(true)}
+ />
+ </div>
+ </div>
+
+ <DialogSectionSeparator />
+
+ {selectedType === 'BigQuery' && etlEnableBigQuery ? (
+ <BigQueryFields form={form} />
+ ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (
+ <AnalyticsBucketFields
+ form={form}
+ setIsFormInteracting={setIsFormInteracting}
+ onSelectNewBucket={() => setNewBucketSheetVisible(true)}
+ />
+ ) : null}
+
+ <DialogSectionSeparator />
+
+ <AdvancedSettings type={selectedType} form={form} />
+ </form>
+ </Form_Shadcn_>
+ )}
+ </SheetSection>
+
+ <SheetFooter>
+ <Button disabled={isSaving} type="default" onClick={onClose}>
+ Cancel
+ </Button>
+ <Button disabled={isSubmitDisabled} loading={isSaving} form={formId} htmlType="submit">
+ {editMode
+ ? existingDestination?.enabled
+ ? 'Apply and restart'
+ : 'Apply and start'
+ : 'Create and start'}
+ </Button>
+ </SheetFooter>
+
+ <NewPublicationPanel
+ sourceId={sourceId}
+ visible={publicationPanelVisible}
+ onClose={() => setPublicationPanelVisible(false)}
+ />
+
+ <CreateAnalyticsBucketSheet
+ open={newBucketSheetVisible}
+ onOpenChange={setNewBucketSheetVisible}
+ />
+
+ <ReplicationDisclaimerDialog
+ open={showDisclaimerDialog}
+ onOpenChange={handleDisclaimerDialogChange}
+ isLoading={isSaving}
+ onConfirm={handleDisclaimerConfirm}
+ />
+ </>
+ )
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts
deleted file mode 100644
index 6050c7bc4c8..00000000000
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.schema.ts
+++ /dev/null
@@ -1,69 +0,0 @@
-import * as z from 'zod'
-
-const types = ['BigQuery', 'Analytics Bucket'] as const
-export const TypeEnum = z.enum(types)
-
-// [Joshen] JFYI if we plan to add another type here, I reckon we split this out into smaller components
-// then as FormSchema is getting quite complex with some fields that aren't necessary if the type is one or the other
-export const DestinationPanelFormSchema = z
- .object({
- // Common fields
- type: TypeEnum,
- name: z.string().min(1, 'Name is required'),
- publicationName: z.string().min(1, 'Publication is required'),
- maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),
- // BigQuery fields
- projectId: z.string().optional(),
- datasetId: z.string().optional(),
- serviceAccountKey: z.string().optional(),
- maxStalenessMins: z.number().nonnegative().optional(),
- // Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
- warehouseName: z.string().optional(),
- namespace: z.string().optional(),
- newNamespaceName: z.string().optional(),
- catalogToken: z.string().optional(),
- s3AccessKeyId: z.string().optional(),
- s3SecretAccessKey: z.string().optional(),
- s3Region: z.string().optional(),
- })
- .superRefine((data, ctx) => {
- const addRequiredFieldError = (path: string, message: string) => {
- ctx.addIssue({
- code: z.ZodIssueCode.custom,
- message,
- path: [path],
- })
- }
-
- if (data.type === 'BigQuery') {
- if (!data.projectId?.length) addRequiredFieldError('projectId', 'Project ID is required')
- if (!data.datasetId?.length) addRequiredFieldError('datasetId', 'Dataset ID is required')
- if (!data.serviceAccountKey?.length)
- addRequiredFieldError('serviceAccountKey', 'Service Account Key is required')
- } else if (data.type === 'Analytics Bucket') {
- if (!data.warehouseName?.length) addRequiredFieldError('warehouseName', 'Bucket is required')
-
- const hasValidNamespace =
- (data.namespace?.length && data.namespace !== 'create-new-namespace') ||
- (data.namespace === 'create-new-namespace' && data.newNamespaceName?.length)
-
- if (!hasValidNamespace) {
- const isCreatingNew = data.namespace === 'create-new-namespace'
- addRequiredFieldError(
- isCreatingNew ? 'newNamespaceName' : 'namespace',
- isCreatingNew ? 'Namespace name is required' : 'Namespace is required'
- )
- }
-
- if (!data.s3Region?.length) addRequiredFieldError('s3Region', 'S3 Region is required')
-
- if (!data.s3AccessKeyId?.length)
- addRequiredFieldError('s3AccessKeyId', 'S3 Access Key ID is required')
-
- if (data.s3AccessKeyId !== 'create-new' && !data.s3SecretAccessKey?.length) {
- addRequiredFieldError('s3SecretAccessKey', 'S3 Secret Access Key is required')
- }
- }
- })
-
-export type DestinationPanelSchemaType = z.infer<typeof DestinationPanelFormSchema>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
index 69ad59bb8c8..6747e252f11 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
@@ -1,61 +1,21 @@
-import { zodResolver } from '@hookform/resolvers/zod'
-import { PermissionAction } from '@supabase/shared-types/out/constants'
-import { snakeCase } from 'lodash'
-import { useEffect, useMemo, useState } from 'react'
-import { useForm } from 'react-hook-form'
-import { toast } from 'sonner'
-import * as z from 'zod'
+import { useState } from 'react'
-import { useFlag, useParams } from 'common'
-import { CreateAnalyticsBucketSheet } from 'components/interfaces/Storage/AnalyticsBuckets/CreateAnalyticsBucketSheet'
-import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'
-import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'
-import { useCheckPrimaryKeysExists } from 'data/database/primary-keys-exists-query'
-import { useCreateDestinationPipelineMutation } from 'data/replication/create-destination-pipeline-mutation'
-import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'
-import { useReplicationPipelineByIdQuery } from 'data/replication/pipeline-by-id-query'
-import { useReplicationPublicationsQuery } from 'data/replication/publications-query'
-import { useRestartPipelineHelper } from 'data/replication/restart-pipeline-helper'
-import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'
-import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'
-import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation'
-import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'
-import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'
-import { useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
+import { useFlag } from 'common'
import {
- PipelineStatusRequestStatus,
- usePipelineRequestStatus,
-} from 'state/replication-pipeline-request-status'
-import {
- Button,
cn,
DialogSectionSeparator,
- Form_Shadcn_,
Sheet,
SheetContent,
SheetDescription,
- SheetFooter,
SheetHeader,
- SheetSection,
SheetTitle,
} from 'ui'
-import { NewPublicationPanel } from '../NewPublicationPanel'
-import { ReplicationDisclaimerDialog } from '../ReplicationDisclaimerDialog'
-import { AdvancedSettings } from './AdvancedSettings'
-import { DestinationNameInput } from './DestinationNameInput'
-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationPanel.constants'
-import { DestinationPanelFormSchema as FormSchema, TypeEnum } from './DestinationPanel.schema'
-import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'
+import { DestinationForm } from './DestinationForm'
+import { DestinationType } from './DestinationPanel.types'
import { DestinationTypeSelection } from './DestinationTypeSelection'
-import { NoDestinationsAvailable } from './NoDestinationsAvailable'
-import { PublicationSelection } from './PublicationSelection'
-
-const formId = 'destination-editor'
interface DestinationPanelProps {
visible: boolean
- sourceId: number | undefined
- onClose: () => void
existingDestination?: {
sourceId?: number
destinationId: number
@@ -63,379 +23,21 @@ interface DestinationPanelProps {
enabled: boolean
statusName?: string
}
+ onClose: () => void
}
export const DestinationPanel = ({
visible,
- sourceId,
- onClose,
existingDestination,
+ onClose,
}: DestinationPanelProps) => {
- const { ref: projectRef } = useParams()
- const { data: project } = useSelectedProjectQuery()
- const { setRequestStatus } = usePipelineRequestStatus()
-
- // Feature flags for ETL destinations
- const etlEnableBigQuery = useFlag('etlEnableBigQuery')
- const etlEnableIceberg = useFlag('etlEnableIceberg')
-
- // Compute available destinations based on feature flags
- const availableDestinations = useMemo(() => {
- const destinations = []
- if (etlEnableBigQuery) destinations.push({ value: 'BigQuery', label: 'BigQuery' })
- if (etlEnableIceberg)
- destinations.push({ value: 'Analytics Bucket', label: 'Analytics Bucket' })
- return destinations
- }, [etlEnableBigQuery, etlEnableIceberg])
-
- const hasNoAvailableDestinations = availableDestinations.length === 0
-
- const editMode = !!existingDestination
- const [showDisclaimerDialog, setShowDisclaimerDialog] = useState(false)
- const [publicationPanelVisible, setPublicationPanelVisible] = useState(false)
- const [newBucketSheetVisible, setNewBucketSheetVisible] = useState(false)
- const [pendingFormValues, setPendingFormValues] = useState<z.infer<typeof FormSchema> | null>(
- null
- )
- const [isFormInteracting, setIsFormInteracting] = useState(false)
-
- const { mutateAsync: createDestinationPipeline, isPending: creatingDestinationPipeline } =
- useCreateDestinationPipelineMutation({
- onSuccess: () => form.reset(defaultValues),
- })
-
- const { mutateAsync: updateDestinationPipeline, isPending: updatingDestinationPipeline } =
- useUpdateDestinationPipelineMutation({
- onSuccess: () => form.reset(defaultValues),
- })
-
- const { mutateAsync: startPipeline, isPending: startingPipeline } = useStartPipelineMutation()
- const { restartPipeline } = useRestartPipelineHelper()
-
- const { mutateAsync: createS3AccessKey, isPending: isCreatingS3AccessKey } =
- useS3AccessKeyCreateMutation()
-
- const { mutateAsync: createNamespace, isPending: isCreatingNamespace } =
- useIcebergNamespaceCreateMutation()
-
- const {
- data: publications = [],
- isSuccess: isSuccessPublications,
- refetch: refetchPublications,
- } = useReplicationPublicationsQuery({ projectRef, sourceId })
+ const unifiedReplication = useFlag('unifiedReplication')
- const { data: destinationData } = useReplicationDestinationByIdQuery({
- projectRef,
- destinationId: existingDestination?.destinationId,
- })
-
- const { data: pipelineData } = useReplicationPipelineByIdQuery({
- projectRef,
- pipelineId: existingDestination?.pipelineId,
- })
-
- const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
- const { data: apiKeys } = useAPIKeysQuery(
- { projectRef, reveal: true },
- { enabled: canReadAPIKeys }
- )
- const { serviceKey } = getKeys(apiKeys)
- const catalogToken = serviceKey?.api_key ?? ''
-
- const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })
-
- const defaultValues = useMemo(() => {
- const config = destinationData?.config
- const isBigQueryConfig = config && 'big_query' in config
- const isIcebergConfig = config && 'iceberg' in config
-
- const defaultType = editMode
- ? isBigQueryConfig
- ? TypeEnum.enum.BigQuery
- : TypeEnum.enum['Analytics Bucket']
- : (availableDestinations[0]?.value as z.infer<typeof TypeEnum>) || TypeEnum.enum.BigQuery
-
- return {
- // Common fields
- type: defaultType,
- name: destinationData?.name ?? '',
- publicationName: pipelineData?.config.publication_name ?? '',
- maxFillMs: pipelineData?.config?.batch?.max_fill_ms,
- // BigQuery fields
- projectId: isBigQueryConfig ? config.big_query.project_id : '',
- datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
- serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
- maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined,
- // Analytics Bucket fields
- warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
- namespace: isIcebergConfig ? config.iceberg.supabase.namespace : '',
- newNamespaceName: '',
- catalogToken: isIcebergConfig ? config.iceberg.supabase.catalog_token : catalogToken,
- s3AccessKeyId: isIcebergConfig ? config.iceberg.supabase.s3_access_key_id : '',
- s3SecretAccessKey: isIcebergConfig ? config.iceberg.supabase.s3_secret_access_key : '',
- s3Region:
- projectSettings?.region ?? (isIcebergConfig ? config.iceberg.supabase.s3_region : ''),
- }
- }, [
- destinationData,
- pipelineData,
- catalogToken,
- projectSettings,
- editMode,
- availableDestinations,
- ])
-
- const form = useForm<z.infer<typeof FormSchema>>({
- mode: 'onChange',
- reValidateMode: 'onChange',
- resolver: zodResolver(FormSchema),
- defaultValues,
- })
- const { publicationName, type: selectedType, warehouseName } = form.watch()
- const isSaving =
- creatingDestinationPipeline ||
- updatingDestinationPipeline ||
- startingPipeline ||
- isCreatingS3AccessKey ||
- isCreatingNamespace
-
- const publicationNames = useMemo(() => publications?.map((pub) => pub.name) ?? [], [publications])
- const isSelectedPublicationMissing =
- isSuccessPublications && !!publicationName && !publicationNames.includes(publicationName)
-
- const selectedPublication = publications.find((pub) => pub.name === publicationName)
- const { data: checkPrimaryKeysExistsData, isPending: isLoadingCheck } = useCheckPrimaryKeysExists(
- {
- projectRef: project?.ref,
- connectionString: project?.connectionString,
- tables: selectedPublication?.tables ?? [],
- },
- { enabled: visible && !!selectedPublication }
+ const [selectedType, setSelectedType] = useState<DestinationType>(
+ unifiedReplication ? 'Read Replica' : 'BigQuery'
)
- const hasTablesWithNoPrimaryKeys = (checkPrimaryKeysExistsData?.offendingTables ?? []).length > 0
-
- const isSubmitDisabled =
- isSaving ||
- isSelectedPublicationMissing ||
- (!!selectedPublication && isLoadingCheck) ||
- hasTablesWithNoPrimaryKeys ||
- (!editMode && hasNoAvailableDestinations)
-
- // Helper function to handle namespace creation if needed
- const resolveNamespace = async (data: z.infer<typeof FormSchema>) => {
- if (data.namespace === CREATE_NEW_NAMESPACE) {
- if (!data.newNamespaceName) throw new Error('New namespace name is required')
-
- await createNamespace({
- projectRef,
- warehouse: data.warehouseName!,
- namespace: data.newNamespaceName,
- })
-
- return data.newNamespaceName
- }
- return data.namespace
- }
-
- // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty
- // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics
- // At first glance we could try grouping as edit / create bigquery, edit / create analytics
- // since the destination config seems rather similar between edit and create for the same type
- const submitPipeline = async (data: z.infer<typeof FormSchema>) => {
- if (!projectRef) return console.error('Project ref is required')
- if (!sourceId) return console.error('Source id is required')
- if (isSelectedPublicationMissing) {
- return toast.error('Please select another publication before continuing')
- }
-
- try {
- if (editMode && existingDestination) {
- if (!existingDestination.pipelineId) return console.error('Pipeline id is required')
-
- let destinationConfig: any = {}
- if (data.type === 'BigQuery') {
- const bigQueryConfig: any = {
- projectId: data.projectId,
- datasetId: data.datasetId,
- serviceAccountKey: data.serviceAccountKey,
- }
- if (!!data.maxStalenessMins) {
- bigQueryConfig.maxStalenessMins = data.maxStalenessMins
- }
- destinationConfig = { bigQuery: bigQueryConfig }
- } else if (data.type === 'Analytics Bucket') {
- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
-
- if (data.s3AccessKeyId === CREATE_NEW_KEY) {
- const newKeys = await createS3AccessKey({
- projectRef,
- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
- })
- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
- }
-
- // Resolve namespace (create if needed)
- const finalNamespace = await resolveNamespace(data)
-
- const icebergConfig: any = {
- projectRef: projectRef,
- warehouseName: data.warehouseName,
- namespace: finalNamespace,
- catalogToken: data.catalogToken,
- s3AccessKeyId: s3Keys.accessKey,
- s3SecretAccessKey: s3Keys.secretKey,
- s3Region: data.s3Region,
- }
- destinationConfig = { iceberg: icebergConfig }
- }
-
- const batchConfig: any = {}
- if (!!data.maxFillMs) batchConfig.maxFillMs = data.maxFillMs
- const hasBatchFields = Object.keys(batchConfig).length > 0
-
- await updateDestinationPipeline({
- destinationId: existingDestination.destinationId,
- pipelineId: existingDestination.pipelineId,
- projectRef,
- destinationName: data.name,
- destinationConfig,
- pipelineConfig: {
- publicationName: data.publicationName,
- ...(hasBatchFields ? { batch: batchConfig } : {}),
- },
- sourceId,
- })
- // Set request status only right before starting, then fire and close
- const snapshot =
- existingDestination.statusName ?? (existingDestination.enabled ? 'started' : 'stopped')
- if (existingDestination.enabled) {
- setRequestStatus(
- existingDestination.pipelineId,
- PipelineStatusRequestStatus.RestartRequested,
- snapshot
- )
- toast.success('Settings applied. Restarting the pipeline...')
- restartPipeline({ projectRef, pipelineId: existingDestination.pipelineId })
- } else {
- setRequestStatus(
- existingDestination.pipelineId,
- PipelineStatusRequestStatus.StartRequested,
- snapshot
- )
- toast.success('Settings applied. Starting the pipeline...')
- startPipeline({ projectRef, pipelineId: existingDestination.pipelineId })
- }
- onClose()
- } else {
- let destinationConfig: any = {}
-
- if (data.type === 'BigQuery') {
- const bigQueryConfig: any = {
- projectId: data.projectId,
- datasetId: data.datasetId,
- serviceAccountKey: data.serviceAccountKey,
- }
- if (!!data.maxStalenessMins) {
- bigQueryConfig.maxStalenessMins = data.maxStalenessMins
- }
- destinationConfig = { bigQuery: bigQueryConfig }
- } else if (data.type === 'Analytics Bucket') {
- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
-
- if (data.s3AccessKeyId === CREATE_NEW_KEY) {
- const newKeys = await createS3AccessKey({
- projectRef,
- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
- })
- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
- }
-
- // Resolve namespace (create if needed)
- const finalNamespace = await resolveNamespace(data)
-
- const icebergConfig: any = {
- projectRef: projectRef,
- warehouseName: data.warehouseName,
- namespace: finalNamespace,
- catalogToken: data.catalogToken,
- s3AccessKeyId: s3Keys.accessKey,
- s3SecretAccessKey: s3Keys.secretKey,
- s3Region: data.s3Region,
- }
- destinationConfig = { iceberg: icebergConfig }
- }
- const batchConfig: any = {}
- if (!!data.maxFillMs) batchConfig.maxFillMs = data.maxFillMs
- const hasBatchFields = Object.keys(batchConfig).length > 0
-
- const { pipeline_id: pipelineId } = await createDestinationPipeline({
- projectRef,
- destinationName: data.name,
- destinationConfig,
- sourceId,
- pipelineConfig: {
- publicationName: data.publicationName,
- ...(hasBatchFields ? { batch: batchConfig } : {}),
- },
- })
- // Set request status only right before starting, then fire and close
- setRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)
- toast.success('Destination created. Starting the pipeline...')
- startPipeline({ projectRef, pipelineId })
- onClose()
- }
- } catch (error) {
- const action = editMode ? 'apply and run' : 'create and start'
- toast.error(`Failed to ${action} destination`)
- }
- }
-
- const onSubmit = async (data: z.infer<typeof FormSchema>) => {
- if (!editMode) {
- setPendingFormValues(data)
- setShowDisclaimerDialog(true)
- return
- }
-
- await submitPipeline(data)
- }
-
- const handleDisclaimerDialogChange = (open: boolean) => {
- setShowDisclaimerDialog(open)
- if (!open) {
- setPendingFormValues(null)
- }
- }
-
- const handleDisclaimerConfirm = async () => {
- if (!pendingFormValues) return
-
- const values = pendingFormValues
- setPendingFormValues(null)
- setShowDisclaimerDialog(false)
- await submitPipeline(values)
- }
-
- useEffect(() => {
- if (editMode && destinationData && pipelineData && !isFormInteracting) {
- form.reset(defaultValues)
- }
- }, [destinationData, pipelineData, editMode, defaultValues, form, isFormInteracting])
-
- // Ensure the form always reflects the freshest data whenever the panel opens
- useEffect(() => {
- if (visible) {
- form.reset(defaultValues)
- setIsFormInteracting(false)
- }
- }, [visible, defaultValues, form])
-
- useEffect(() => {
- if (visible && projectRef && sourceId) {
- refetchPublications()
- }
- }, [visible, projectRef, sourceId, refetchPublications])
+ const editMode = !!existingDestination
return (
<>
@@ -443,10 +45,7 @@ export const DestinationPanel = ({
<SheetContent
size="default"
showClose={false}
- className={cn(
- 'md:!w-[700px]',
- publicationPanelVisible || newBucketSheetVisible ? 'right-20' : 'right-0'
- )}
+ className={cn(unifiedReplication ? 'md:!w-[850px]' : 'md:!w-[700px]')}
>
<div className="flex flex-col h-full" tabIndex={-1}>
<SheetHeader>
@@ -458,88 +57,29 @@ export const DestinationPanel = ({
</SheetDescription>
</SheetHeader>
- <SheetSection className="flex-grow overflow-auto px-0 py-0">
- {hasNoAvailableDestinations && !editMode ? (
- <NoDestinationsAvailable />
- ) : (
- <Form_Shadcn_ {...form}>
- <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
- <DestinationTypeSelection form={form} editMode={editMode} />
-
- <DialogSectionSeparator />
-
- <div className="p-5 flex flex-col gap-y-6">
- <p className="text-sm font-medium text-foreground">Destination details</p>
-
- <div className="space-y-4">
- <DestinationNameInput form={form} />
- <PublicationSelection
- form={form}
- sourceId={sourceId}
- visible={visible}
- onSelectNewPublication={() => setPublicationPanelVisible(true)}
- />
- </div>
- </div>
-
- <DialogSectionSeparator />
-
- {selectedType === 'BigQuery' && etlEnableBigQuery ? (
- <BigQueryFields form={form} />
- ) : selectedType === 'Analytics Bucket' && etlEnableIceberg ? (
- <AnalyticsBucketFields
- form={form}
- setIsFormInteracting={setIsFormInteracting}
- onSelectNewBucket={() => setNewBucketSheetVisible(true)}
- />
- ) : null}
-
- <DialogSectionSeparator />
-
- <AdvancedSettings form={form} />
- </form>
- </Form_Shadcn_>
- )}
- </SheetSection>
-
- <SheetFooter>
- <Button disabled={isSaving} type="default" onClick={onClose}>
- Cancel
- </Button>
- <Button
- disabled={isSubmitDisabled}
- loading={isSaving}
- form={formId}
- htmlType="submit"
- >
- {editMode
- ? existingDestination?.enabled
- ? 'Apply and restart'
- : 'Apply and start'
- : 'Create and start'}
- </Button>
- </SheetFooter>
+ <DestinationTypeSelection
+ editMode={editMode}
+ selectedType={selectedType}
+ setSelectedType={setSelectedType}
+ />
+
+ <DialogSectionSeparator />
+
+ {selectedType === 'Read Replica' ? (
+ <div className="w-full h-full flex items-center justify-center">
+ <p className="text-foreground-lighter text-sm">Coming soon</p>
+ </div>
+ ) : (
+ <DestinationForm
+ visible={visible}
+ selectedType={selectedType}
+ existingDestination={existingDestination}
+ onClose={onClose}
+ />
+ )}
</div>
</SheetContent>
</Sheet>
-
- <NewPublicationPanel
- sourceId={sourceId}
- visible={publicationPanelVisible}
- onClose={() => setPublicationPanelVisible(false)}
- />
-
- <CreateAnalyticsBucketSheet
- open={newBucketSheetVisible}
- onOpenChange={setNewBucketSheetVisible}
- />
-
- <ReplicationDisclaimerDialog
- open={showDisclaimerDialog}
- onOpenChange={handleDisclaimerDialogChange}
- isLoading={isSaving}
- onConfirm={handleDisclaimerConfirm}
- />
</>
)
}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.ts
new file mode 100644
index 00000000000..657395b8b66
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.types.ts
@@ -0,0 +1 @@
+export type DestinationType = 'Read Replica' | 'BigQuery' | 'Analytics Bucket'
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
index c7b9362a717..6f6e30b2a6e 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
@@ -1,26 +1,28 @@
-import type { UseFormReturn } from 'react-hook-form'
-
import { useFlag } from 'common'
-import { AnalyticsBucket, BigQuery } from 'icons'
-import {
- cn,
- FormControl_Shadcn_,
- FormField_Shadcn_,
- RadioGroupStacked,
- RadioGroupStackedItem,
-} from 'ui'
+import { AnalyticsBucket, BigQuery, Database } from 'icons'
+import { cn, RadioGroupStacked, RadioGroupStackedItem } from 'ui'
import { Admonition } from 'ui-patterns'
-import type { DestinationPanelSchemaType } from './DestinationPanel.schema'
+import { DestinationType } from './DestinationPanel.types'
type DestinationTypeSelectionProps = {
- form: UseFormReturn<DestinationPanelSchemaType>
editMode: boolean
+ selectedType: DestinationType
+ setSelectedType: (value: DestinationType) => void
}
-export const DestinationTypeSelection = ({ form, editMode }: DestinationTypeSelectionProps) => {
+export const DestinationTypeSelection = ({
+ editMode,
+ selectedType,
+ setSelectedType,
+}: DestinationTypeSelectionProps) => {
+ const unifiedReplication = useFlag('unifiedReplication')
const etlEnableBigQuery = useFlag('etlEnableBigQuery')
const etlEnableIceberg = useFlag('etlEnableIceberg')
+ const numberOfTypes = [unifiedReplication, etlEnableBigQuery, etlEnableIceberg].filter(
+ Boolean
+ ).length
+
return (
<div className="px-5 py-5">
<div className="flex flex-col gap-y-2 mb-4">
@@ -32,64 +34,71 @@ export const DestinationTypeSelection = ({ form, editMode }: DestinationTypeSele
/>
)}
</div>
- <FormField_Shadcn_
- name="type"
- control={form.control}
- render={({ field }) => (
- <FormControl_Shadcn_>
- <RadioGroupStacked
- disabled={editMode}
- value={field.value}
- onValueChange={(value) => field.onChange(value)}
- className={cn(
- 'grid grid-cols-2 [&>button>div]:py-4',
- '[&>button:first-of-type]:rounded-none [&>button:last-of-type]:rounded-none',
- '[&>button:first-of-type]:!rounded-l-lg [&>button:last-of-type]:!rounded-r-lg'
- )}
- >
- {((!editMode && etlEnableBigQuery) || (editMode && field.value === 'BigQuery')) && (
- <RadioGroupStackedItem
- label=""
- showIndicator={false}
- id="BigQuery"
- value="BigQuery"
- >
- <div className="flex flex-col gap-y-2">
- <BigQuery size={20} />
- <div className="flex flex-col gap-y-0.5 text-sm text-left">
- <p>BigQuery</p>
- <p className="text-foreground-lighter">
- Send data to Google Cloud's data warehouse for analytics and business
- intelligence
- </p>
- </div>
- </div>
- </RadioGroupStackedItem>
- )}
- {((!editMode && etlEnableIceberg) ||
- (editMode && field.value === 'Analytics Bucket')) && (
- <RadioGroupStackedItem
- label=""
- showIndicator={false}
- id="Analytics Bucket"
- value="Analytics Bucket"
- >
- <div className="flex flex-col gap-y-2">
- <AnalyticsBucket size={20} />
- <div className="flex flex-col gap-y-0.5 text-sm text-left">
- <p>Analytics Bucket</p>
- <p className="text-foreground-lighter">
- Send data to Apache Iceberg tables in your Supabase Storage for flexible
- analytics workflows
- </p>
- </div>
- </div>
- </RadioGroupStackedItem>
- )}
- </RadioGroupStacked>
- </FormControl_Shadcn_>
+ <RadioGroupStacked
+ disabled={editMode}
+ value={selectedType}
+ onValueChange={(value) => setSelectedType(value as DestinationType)}
+ className={cn(
+ 'grid [&>button>div]:py-4',
+ numberOfTypes === 3 ? 'grid-cols-3' : numberOfTypes === 2 ? 'grid-cols-2' : 'grid-cols-1',
+ '[&>button:first-of-type]:rounded-none [&>button:last-of-type]:rounded-none',
+ '[&>button:first-of-type]:!rounded-l-lg [&>button:last-of-type]:!rounded-r-lg'
+ )}
+ >
+ {((!editMode && unifiedReplication) || (editMode && selectedType === 'Read Replica')) && (
+ <RadioGroupStackedItem
+ label=""
+ showIndicator={false}
+ id="Read Replica"
+ value="Read Replica"
+ >
+ <div className="flex flex-col gap-y-2">
+ <Database size={20} />
+ <div className="flex flex-col gap-y-0.5 text-sm text-left">
+ <p>Read Replica</p>
+ <p className="text-foreground-lighter">
+ Deploy read-only databases across multiple regions for lower latency and better
+ resource management
+ </p>
+ </div>
+ </div>
+ </RadioGroupStackedItem>
+ )}
+
+ {((!editMode && etlEnableBigQuery) || (editMode && selectedType === 'BigQuery')) && (
+ <RadioGroupStackedItem label="" showIndicator={false} id="BigQuery" value="BigQuery">
+ <div className="flex flex-col gap-y-2">
+ <BigQuery size={20} />
+ <div className="flex flex-col gap-y-0.5 text-sm text-left">
+ <p>BigQuery</p>
+ <p className="text-foreground-lighter">
+ Send data to Google Cloud's data warehouse for analytics and business intelligence
+ </p>
+ </div>
+ </div>
+ </RadioGroupStackedItem>
+ )}
+
+ {((!editMode && etlEnableIceberg) || (editMode && selectedType === 'Analytics Bucket')) && (
+ <RadioGroupStackedItem
+ label=""
+ showIndicator={false}
+ id="Analytics Bucket"
+ value="Analytics Bucket"
+ >
+ <div className="flex flex-col gap-y-2">
+ <AnalyticsBucket size={20} />
+ <div className="flex flex-col gap-y-0.5 text-sm text-left">
+ <p>Analytics Bucket</p>
+ <p className="text-foreground-lighter">
+ Send data to Apache Iceberg tables in your Supabase Storage for flexible analytics
+ workflows
+ </p>
+ </div>
+ </div>
+ </RadioGroupStackedItem>
)}
- />
+ </RadioGroupStacked>
</div>
)
}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx
index 5e818630956..b3dbb5da599 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationRow.tsx
@@ -21,7 +21,7 @@ import { Button, Tooltip, TooltipContent, TooltipTrigger } from 'ui'
import { ShimmeringLoader } from 'ui-patterns/ShimmeringLoader'
import { DeleteDestination } from './DeleteDestination'
import { DestinationPanel } from './DestinationPanel/DestinationPanel'
-import { DestinationPanelSchemaType } from './DestinationPanel/DestinationPanel.schema'
+import { DestinationType } from './DestinationPanel/DestinationPanel.types'
import { getStatusName, PIPELINE_ERROR_MESSAGES } from './Pipeline.utils'
import { PipelineStatus } from './PipelineStatus'
import { PipelineStatusName, STATUS_REFRESH_FREQUENCY_MS } from './Replication.constants'
@@ -32,7 +32,7 @@ interface DestinationRowProps {
sourceId?: number
destinationId: number
destinationName: string
- type: DestinationPanelSchemaType['type'] | 'Other'
+ type: DestinationType | 'Other'
pipeline?: Pipeline
error: ResponseError | null
isLoading: boolean
@@ -223,7 +223,6 @@ export const DestinationRow = ({
<DestinationPanel
visible={showEditDestinationPanel}
onClose={() => setShowEditDestinationPanel(false)}
- sourceId={sourceId}
existingDestination={{
sourceId,
destinationId: destinationId,
diff --git a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
index 06cd96e310a..512b905fdc9 100644
--- a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
@@ -230,7 +230,6 @@ export const Destinations = () => {
<DestinationPanel
visible={showNewDestinationPanel}
- sourceId={sourceId}
onClose={() => setShowNewDestinationPanel(false)}
/>
</>
diff --git a/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx b/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx
index 1210d9e6dea..65f18ec09ad 100644
--- a/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx
+++ b/apps/studio/components/interfaces/Settings/Infrastructure/InfrastructureInfo.tsx
@@ -1,4 +1,5 @@
-import { useParams } from 'common'
+import { useFlag, useParams } from 'common'
+import { NoticeBar } from 'components/interfaces/DiskManagement/ui/NoticeBar'
import {
ScaffoldContainer,
ScaffoldDivider,
@@ -12,11 +13,13 @@ import { useProjectServiceVersionsQuery } from 'data/projects/project-service-ve
import { useReadReplicasQuery } from 'data/read-replicas/replicas-query'
import { useIsFeatureEnabled } from 'hooks/misc/useIsFeatureEnabled'
import { useIsOrioleDb, useSelectedProjectQuery } from 'hooks/misc/useSelectedProject'
+import Link from 'next/link'
import {
AlertDescription_Shadcn_,
AlertTitle_Shadcn_,
Alert_Shadcn_,
Badge,
+ Button,
Input,
Tooltip,
TooltipContent,
@@ -36,6 +39,8 @@ const InfrastructureInfo = () => {
const { ref } = useParams()
const { data: project } = useSelectedProjectQuery()
+ const unifiedReplication = useFlag('unifiedReplication')
+
const { projectAuthAll: authEnabled, projectSettingsDatabaseUpgrades: showDatabaseUpgrades } =
useIsFeatureEnabled(['project_auth:all', 'project_settings:database_upgrades'])
@@ -83,12 +88,31 @@ const InfrastructureInfo = () => {
return (
<>
<ScaffoldDivider />
- {project?.cloud_provider !== 'FLY' && (
- <>
- <InstanceConfiguration />
- <ScaffoldDivider />
- </>
- )}
+ {project?.cloud_provider !== 'FLY' &&
+ (unifiedReplication ? (
+ <ScaffoldContainer>
+ <ScaffoldSection isFullWidth>
+ <NoticeBar
+ visible={true}
+ type="default"
+ title="Management of read replicas has moved"
+ description="Read replicas is now managed under Replication in the Database section."
+ actions={
+ <Button type="default" asChild>
+ <Link href={`/project/${ref}/database/replication`} className="!no-underline">
+ Go to Replication
+ </Link>
+ </Button>
+ }
+ />
+ </ScaffoldSection>
+ </ScaffoldContainer>
+ ) : (
+ <>
+ <InstanceConfiguration />
+ <ScaffoldDivider />
+ </>
+ ))}
<ScaffoldContainer>
<ScaffoldSection>
diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
index 78476e137d7..8141dac26e6 100644
--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
@@ -6,6 +6,14 @@ import { handleError, post } from 'data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from 'types'
import { replicationKeys } from './keys'
+export type DestinationConfig =
+ | {
+ bigQuery: BigQueryDestinationConfig
+ }
+ | {
+ iceberg: IcebergDestinationConfig
+ }
+
export type BigQueryDestinationConfig = {
projectId: string
datasetId: string
@@ -23,22 +31,18 @@ export type IcebergDestinationConfig = {
s3Region: string
}
+export type BatchConfig = {
+ maxFillMs: number
+}
+
export type CreateDestinationPipelineParams = {
projectRef: string
destinationName: string
- destinationConfig:
- | {
- bigQuery: BigQueryDestinationConfig
- }
- | {
- iceberg: IcebergDestinationConfig
- }
+ destinationConfig: DestinationConfig
sourceId: number
pipelineConfig: {
publicationName: string
- batch?: {
- maxFillMs: number
- }
+ batch?: BatchConfig
}
}
diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
index ae7def269b4..12da1fcd1db 100644
--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
@@ -4,43 +4,19 @@ import { toast } from 'sonner'
import type { components } from 'api-types'
import { handleError, post } from 'data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from 'types'
+import { BatchConfig, DestinationConfig } from './create-destination-pipeline-mutation'
import { replicationKeys } from './keys'
-export type BigQueryDestinationConfig = {
- projectId: string
- datasetId: string
- serviceAccountKey: string
- maxStalenessMins?: number
-}
-
-export type IcebergDestinationConfig = {
- projectRef: string
- warehouseName: string
- namespace: string
- catalogToken: string
- s3AccessKeyId: string
- s3SecretAccessKey: string
- s3Region: string
-}
-
export type UpdateDestinationPipelineParams = {
destinationId: number
pipelineId: number
projectRef: string
destinationName: string
- destinationConfig:
- | {
- bigQuery: BigQueryDestinationConfig
- }
- | {
- iceberg: IcebergDestinationConfig
- }
+ destinationConfig: DestinationConfig
sourceId: number
pipelineConfig: {
publicationName: string
- batch?: {
- maxFillMs: number
- }
+ batch?: BatchConfig
}
}