Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(etl): Add UI elements to disable external replication (#45035)

Riccardo Busetti834787795786
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
index 4aa5afcbeae..53d1d953471 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationPanel.tsx
@@ -102,7 +102,7 @@ export const DestinationPanel = ({ onSuccessCreateReadReplica }: DestinationPane
<SheetDescription>
{editMode
? 'Update the configuration for this destination'
- : 'A destination is an external platform that automatically receives your database changes in real time.'}
+ : 'A destination can be a read replica or an external platform that receives your database changes in real time.'}
</SheetDescription>
</SheetHeader>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
index d918c3212aa..754ae1cd9be 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationTypeSelection.tsx
@@ -122,7 +122,7 @@ export const DestinationTypeSelection = () => {
{destinationType !== 'Read Replica' && (
<p className="mt-3 text-sm text-foreground-light">
- Replication is in alpha. Expect rapid changes and possible breaking updates.{' '}
+ External replication is in alpha. Expect rapid changes and possible breaking updates.{' '}
<InlineLink href="https://github.com/orgs/supabase/discussions/39416">
Leave feedback
</InlineLink>
diff --git a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
index b915d10160d..56e55e6876a 100644
--- a/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/Destinations.tsx
@@ -1,13 +1,17 @@
import { useQueryClient } from '@tanstack/react-query'
import { useParams } from 'common'
-import { Plus, Search, X } from 'lucide-react'
+import { MoreVertical, Plus, Search, X } from 'lucide-react'
import { parseAsStringEnum, useQueryState } from 'nuqs'
-import { useEffect, useRef, useState } from 'react'
+import { useEffect, useMemo, useRef, useState } from 'react'
import {
Button,
Card,
CardContent,
cn,
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuTrigger,
Table,
TableBody,
TableCell,
@@ -22,6 +26,7 @@ import { REPLICA_STATUS } from '../../Settings/Infrastructure/InfrastructureConf
import { DestinationPanel } from './DestinationPanel/DestinationPanel'
import { DestinationType } from './DestinationPanel/DestinationPanel.types'
import { DestinationRow } from './DestinationRow'
+import { DisableExternalReplicationDialog } from './DisableExternalReplicationDialog'
import { PIPELINE_ERROR_MESSAGES } from './Pipeline.utils'
import { ReadReplicaRow } from './ReadReplicas/ReadReplicaRow'
import { useIsETLBigQueryPrivateAlpha, useIsETLIcebergPrivateAlpha } from './useIsETLPrivateAlpha'
@@ -32,6 +37,7 @@ import { useReplicationDestinationsQuery } from '@/data/replication/destinations
import { replicationKeys } from '@/data/replication/keys'
import { fetchReplicationPipelineVersion } from '@/data/replication/pipeline-version-query'
import { useReplicationPipelinesQuery } from '@/data/replication/pipelines-query'
+import { useReplicationSourcesQuery } from '@/data/replication/sources-query'
import { useIsFeatureEnabled } from '@/hooks/misc/useIsFeatureEnabled'
import { DOCS_URL } from '@/lib/constants'
@@ -54,6 +60,8 @@ export const Destinations = () => {
const prefetchedRef = useRef(false)
const [filterString, setFilterString] = useState<string>('')
const [statusRefetchInterval, setStatusRefetchInterval] = useState<number | false>(5000)
+ const [showDisableExternalReplicationDialog, setShowDisableExternalReplicationDialog] =
+ useState(false)
const [_, setDestinationType] = useQueryState(
'destinationType',
@@ -102,10 +110,31 @@ export const Destinations = () => {
const { data: pipelinesData, isSuccess: isPipelinesSuccess } = useReplicationPipelinesQuery({
projectRef,
})
+ const pipelines = pipelinesData?.pipelines ?? []
+
+ const { data: sourcesData, isSuccess: isSourcesSuccess } = useReplicationSourcesQuery({
+ projectRef,
+ })
+ const externalReplicationSource = useMemo(
+ () => sourcesData?.sources.find((source) => source.name === projectRef),
+ [projectRef, sourcesData?.sources]
+ )
+ const canDisableExternalReplication =
+ isSourcesSuccess &&
+ isDestinationsSuccess &&
+ isPipelinesSuccess &&
+ !!externalReplicationSource &&
+ destinations.length === 0 &&
+ pipelines.length === 0
const isLoading = isDestinationsLoading || isDatabasesLoading
const hasErrorsFetchingData = isDestinationsError || isDatabasesError
+ const openDestinationPanel = () => {
+ if (!newDestinationDefaultType) return
+ setDestinationType(newDestinationDefaultType)
+ }
+
useEffect(() => {
if (
projectRef &&
@@ -176,11 +205,23 @@ export const Destinations = () => {
type="default"
icon={<Plus />}
disabled={!newDestinationDefaultType}
- onClick={() => setDestinationType(newDestinationDefaultType)}
+ onClick={openDestinationPanel}
>
Add destination
</Button>
<DocsButton href={`${DOCS_URL}/guides/database/replication`} />
+ {canDisableExternalReplication && (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button type="default" icon={<MoreVertical />} className="w-7" />
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="end" className="w-52">
+ <DropdownMenuItem onClick={() => setShowDisableExternalReplicationDialog(true)}>
+ Disable external replication
+ </DropdownMenuItem>
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )}
</div>
</div>
</div>
@@ -264,7 +305,8 @@ export const Destinations = () => {
</p>
<Button
icon={<Plus />}
- onClick={() => setDestinationType('Read Replica')}
+ disabled={!newDestinationDefaultType}
+ onClick={openDestinationPanel}
className="mt-4"
>
Add destination
@@ -275,6 +317,11 @@ export const Destinations = () => {
</div>
<DestinationPanel onSuccessCreateReadReplica={() => setStatusRefetchInterval(5000)} />
+
+ <DisableExternalReplicationDialog
+ open={showDisableExternalReplicationDialog}
+ setOpen={setShowDisableExternalReplicationDialog}
+ />
</>
)
}
diff --git a/apps/studio/components/interfaces/Database/Replication/DisableExternalReplicationDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DisableExternalReplicationDialog.tsx
new file mode 100644
index 00000000000..18586c89ce3
--- /dev/null
+++ b/apps/studio/components/interfaces/Database/Replication/DisableExternalReplicationDialog.tsx
@@ -0,0 +1,74 @@
+import { useParams } from 'common'
+import { toast } from 'sonner'
+import {
+ AlertDialog,
+ AlertDialogAction,
+ AlertDialogCancel,
+ AlertDialogContent,
+ AlertDialogDescription,
+ AlertDialogFooter,
+ AlertDialogHeader,
+ AlertDialogTitle,
+ Button,
+} from 'ui'
+
+import { useDeleteReplicationTenantMutation } from '@/data/replication/delete-tenant-mutation'
+
+interface DisableExternalReplicationDialogProps {
+ open: boolean
+ setOpen: (value: boolean) => void
+}
+
+export const DisableExternalReplicationDialog = ({
+ open,
+ setOpen,
+}: DisableExternalReplicationDialogProps) => {
+ const { ref: projectRef } = useParams()
+
+ const { mutateAsync: deleteReplicationTenant, isPending: isSubmitting } =
+ useDeleteReplicationTenantMutation({
+ onSuccess: () => {
+ toast.success('External replication has been disabled')
+ setOpen(false)
+ },
+ })
+
+ const onConfirm = async () => {
+ if (!projectRef) return console.error('Project ref is required')
+ await deleteReplicationTenant({ projectRef })
+ }
+
+ return (
+ <AlertDialog open={open} onOpenChange={(open) => !isSubmitting && setOpen(open)}>
+ <AlertDialogContent size="medium">
+ <AlertDialogHeader>
+ <AlertDialogTitle>Confirm to disable external replication</AlertDialogTitle>
+ <AlertDialogDescription className="space-y-2 text-sm">
+ <p>
+ This will remove the <code className="text-code-inline">etl</code> schema and all
+ connected resources from your database. Any active pipelines sending changes to
+ external destinations will stop.
+ </p>
+ <p>Read replicas are not affected.</p>
+ </AlertDialogDescription>
+ </AlertDialogHeader>
+ <AlertDialogFooter>
+ <AlertDialogCancel disabled={isSubmitting}>Cancel</AlertDialogCancel>
+ <AlertDialogAction variant="danger" asChild>
+ <Button
+ type="danger"
+ loading={isSubmitting}
+ disabled={isSubmitting}
+ onClick={(e) => {
+ e.preventDefault()
+ onConfirm()
+ }}
+ >
+ Disable external replication
+ </Button>
+ </AlertDialogAction>
+ </AlertDialogFooter>
+ </AlertDialogContent>
+ </AlertDialog>
+ )
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/EnableReplicationCallout.tsx b/apps/studio/components/interfaces/Database/Replication/EnableReplicationCallout.tsx
index 8fad5e633be..f71c1f78eab 100644
--- a/apps/studio/components/interfaces/Database/Replication/EnableReplicationCallout.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/EnableReplicationCallout.tsx
@@ -28,11 +28,11 @@ const EnableReplicationModal = () => {
const { mutate: createTenantSource, isPending: creatingTenantSource } =
useCreateTenantSourceMutation({
onSuccess: () => {
- toast.success('Replication has been successfully enabled!')
+ toast.success('External replication has been successfully enabled!')
setOpen(false)
},
onError: (error) => {
- toast.error(`Failed to enable replication: ${error.message}`)
+ toast.error(`Failed to enable external replication: ${error.message}`)
},
})
@@ -45,12 +45,12 @@ const EnableReplicationModal = () => {
<Dialog open={open} onOpenChange={setOpen}>
<DialogTrigger asChild>
<Button type="primary" className="w-min">
- Enable replication
+ Enable external replication
</Button>
</DialogTrigger>
<DialogContent>
<DialogHeader>
- <DialogTitle>Enable Replication</DialogTitle>
+ <DialogTitle>Enable external replication</DialogTitle>
</DialogHeader>
<DialogSectionSeparator />
<DialogSection className="flex flex-col gap-y-2 !p-0">
@@ -74,7 +74,7 @@ const EnableReplicationModal = () => {
Cancel
</Button>
<Button type="primary" loading={creatingTenantSource} onClick={onEnableReplication}>
- Enable replication
+ Enable external replication
</Button>
</DialogFooter>
</DialogContent>
@@ -94,10 +94,10 @@ export const EnableReplicationCallout = ({
return (
<div className={cn('border rounded-md p-4 md:p-12 flex flex-col gap-y-4', className)}>
<div className="flex flex-col gap-y-1">
- <h4>Replicate data to external destinations in real-time</h4>
+ <h4>Replicate data to external destinations in real time</h4>
<p className="text-sm text-foreground-light">
- {hasAccess ? 'Enable replication' : 'Upgrade to the Pro plan'} to start replicating your
- database changes to {type ?? 'data warehouses and analytics platforms'}
+ {hasAccess ? 'Enable external replication' : 'Upgrade to the Pro plan'} to start
+ replicating your database changes to {type ?? 'data warehouses and analytics platforms'}
</p>
</div>
<div className="flex gap-x-2">
diff --git a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
index 0f32addeece..7f9802d2928 100644
--- a/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/ReplicationPipelineStatus/ReplicationPipelineStatus.tsx
@@ -444,7 +444,7 @@ export const ReplicationPipelineStatus = () => {
tooltip={{
content: {
side: 'left',
- text: hasErroredTables ? 'No tables require manual retry' : undefined,
+ text: !hasErroredTables ? 'No tables require manual retry' : undefined,
},
}}
>
diff --git a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/AnalyticsBucketDetails.constants.ts b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/AnalyticsBucketDetails.constants.ts
index 7efe6904c95..fe4274afc4d 100644
--- a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/AnalyticsBucketDetails.constants.ts
+++ b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/AnalyticsBucketDetails.constants.ts
@@ -26,7 +26,7 @@ export const DESCRIPTIONS: Record<string, string> = {
}
// [Joshen] For context we've decided to decouple ETL from Analytics Buckets for now
-// So this flag just hides all "connect table" ETL flow related UI
+// So this flag just hides all ETL-related user flows in Analytics Buckets
// Depending on future decision if we intend to keep it that way, then we might be able
-// to clean up + deprecate ConnectTablesDialog and other ETL related UI within Analytics Buckets
+// to clean up + deprecate the remaining ETL-related UI within Analytics Buckets
export const HIDE_REPLICATION_USER_FLOW = true
diff --git a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/BucketHeader.tsx b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/BucketHeader.tsx
index ee023453bf2..538b015fed4 100644
--- a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/BucketHeader.tsx
+++ b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/BucketHeader.tsx
@@ -1,9 +1,4 @@
-import { noop } from 'lodash'
-
-import { HIDE_REPLICATION_USER_FLOW } from './AnalyticsBucketDetails.constants'
-import { ConnectTablesDialog } from './ConnectTablesDialog'
import { CreateTableInstructionsDialog } from './CreateTable/CreateTableInstructionsDialog'
-import { FormattedWrapperTable } from '@/components/interfaces/Integrations/Wrappers/Wrappers.utils'
import {
ScaffoldHeader,
ScaffoldSectionDescription,
@@ -12,19 +7,9 @@ import {
interface BucketHeaderProps {
showActions?: boolean
- namespaces?: {
- namespace: string
- schema: string
- tables: FormattedWrapperTable[]
- }[]
- onSuccessConnectTables?: () => void
}
-export const BucketHeader = ({
- showActions = true,
- namespaces = [],
- onSuccessConnectTables = noop,
-}: BucketHeaderProps) => {
+export const BucketHeader = ({ showActions = true }: BucketHeaderProps) => {
return (
<ScaffoldHeader className="pt-0 flex flex-row justify-between items-end gap-x-8">
<div>
@@ -35,13 +20,7 @@ export const BucketHeader = ({
</div>
{showActions && (
<div className="flex items-center gap-x-2">
- {HIDE_REPLICATION_USER_FLOW ? (
- <CreateTableInstructionsDialog />
- ) : (
- namespaces.length > 0 && (
- <ConnectTablesDialog onSuccessConnectTables={onSuccessConnectTables} />
- )
- )}
+ <CreateTableInstructionsDialog />
</div>
)}
</ScaffoldHeader>
diff --git a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/ConnectTablesDialog.tsx b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/ConnectTablesDialog.tsx
deleted file mode 100644
index 9bd60d028bd..00000000000
--- a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/ConnectTablesDialog.tsx
+++ /dev/null
@@ -1,481 +0,0 @@
-import { zodResolver } from '@hookform/resolvers/zod'
-import { PermissionAction } from '@supabase/shared-types/out/constants'
-import { useParams } from 'common'
-import { AnimatePresence, motion } from 'framer-motion'
-import { Loader2, Plus } from 'lucide-react'
-import { useEffect, useMemo, useState } from 'react'
-import { SubmitHandler, useForm } from 'react-hook-form'
-import { toast } from 'sonner'
-import {
- Button,
- Dialog,
- DialogContent,
- DialogDescription,
- DialogFooter,
- DialogHeader,
- DialogSection,
- DialogSectionSeparator,
- DialogTitle,
- DialogTrigger,
- Form_Shadcn_,
- FormControl_Shadcn_,
- FormField_Shadcn_,
- Progress,
-} from 'ui'
-import { Admonition } from 'ui-patterns'
-import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
-import { MultiSelector } from 'ui-patterns/multi-select'
-import z from 'zod'
-
-import {
- getAnalyticsBucketPublicationName,
- getAnalyticsBucketsDestinationName,
-} from './AnalyticsBucketDetails.utils'
-import { useAnalyticsBucketAssociatedEntities } from './useAnalyticsBucketAssociatedEntities'
-import { useAnalyticsBucketWrapperInstance } from './useAnalyticsBucketWrapperInstance'
-import { useIsETLPrivateAlpha } from '@/components/interfaces/Database/Replication/useIsETLPrivateAlpha'
-import { convertKVStringArrayToJson } from '@/components/interfaces/Integrations/Wrappers/Wrappers.utils'
-import { getKeys, useAPIKeysQuery } from '@/data/api-keys/api-keys-query'
-import { useProjectSettingsV2Query } from '@/data/config/project-settings-v2-query'
-import { useCreateDestinationPipelineMutation } from '@/data/replication/create-destination-pipeline-mutation'
-import { useCreateTenantSourceMutation } from '@/data/replication/create-tenant-source-mutation'
-import { useCreatePublicationMutation } from '@/data/replication/publication-create-mutation'
-import { useUpdatePublicationMutation } from '@/data/replication/publication-update-mutation'
-import { useReplicationSourcesQuery } from '@/data/replication/sources-query'
-import { useStartPipelineMutation } from '@/data/replication/start-pipeline-mutation'
-import { useReplicationTablesQuery } from '@/data/replication/tables-query'
-import { getDecryptedValues } from '@/data/vault/vault-secret-decrypted-value-query'
-import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
-import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
-
-/**
- * [Joshen] So far this is purely just setting up a "Connect from empty state" flow
- * Doing it bit by bit as this is quite an unknown territory, will adjust as we figure out
- * limitations, correctness, etc, etc. ETL is also only available on staging so its quite hard
- * to test things locally (Local set up is technically available but quite high friction)
- *
- * What's missing afaict:
- * - Deleting namespaces
- * - Removing tables
- * - Adding more tables
- * - Error handling due to multiple async processes
- */
-
-const formId = 'connect-tables-form'
-const FormSchema = z.object({
- tables: z.array(z.string()).min(1, 'Select at least one table'),
-})
-
-type ConnectTablesForm = z.infer<typeof FormSchema>
-
-enum PROGRESS_STAGE {
- CREATE_PUBLICATION = 'CREATE_PUBLICATION',
- CREATE_PIPELINE = 'CREATE_PIPELINE',
- START_PIPELINE = 'START_PIPELINE',
- UPDATE_PUBLICATION = 'CREATE_REPLICATION',
-}
-
-const PROGRESS_INDICATORS = {
- CREATE: [
- {
- step: PROGRESS_STAGE.CREATE_PUBLICATION,
- getDescription: (numTables: number) =>
- `Creating replication publication with ${numTables} table${numTables > 1 ? 's' : ''}...`,
- },
- { step: PROGRESS_STAGE.CREATE_PIPELINE, description: `Creating replication pipeline` },
- { step: PROGRESS_STAGE.START_PIPELINE, description: `Starting replication pipeline` },
- ],
- UPDATE: [
- {
- step: PROGRESS_STAGE.UPDATE_PUBLICATION,
- getDescription: (numTables: number) =>
- `Updating replication publication with ${numTables} table${numTables > 1 ? 's' : ''}...`,
- },
- { step: PROGRESS_STAGE.START_PIPELINE, description: 'Restarting replication pipeline...' },
- ],
-}
-
-interface ConnectTablesDialogProps {
- onSuccessConnectTables: () => void
-}
-
-/** [Joshen] This component is currently not user-facing atm, might opt to clean up as we're likely not going to use this UI flow */
-export const ConnectTablesDialog = ({ onSuccessConnectTables }: ConnectTablesDialogProps) => {
- const { ref: projectRef, bucketId } = useParams()
- const [visible, setVisible] = useState(false)
-
- const { sourceId, pipeline, publication } = useAnalyticsBucketAssociatedEntities({
- projectRef,
- bucketId,
- })
- const isEditingExistingPublication = !!publication && !!pipeline
-
- return (
- <Dialog open={visible} onOpenChange={setVisible}>
- <DialogTrigger asChild>
- <Button size="tiny" type="primary" icon={<Plus />} onClick={() => setVisible(true)}>
- {isEditingExistingPublication ? 'Add tables' : 'Connect tables'}
- </Button>
- </DialogTrigger>
-
- {!sourceId ? (
- <EnableReplicationDialogContent onClose={() => setVisible(false)} />
- ) : (
- <ConnectTablesDialogContent
- visible={visible}
- onClose={() => setVisible(false)}
- onSuccessConnectTables={onSuccessConnectTables}
- />
- )}
- </Dialog>
- )
-}
-
-export const ConnectTablesDialogContent = ({
- visible,
- onClose,
- onSuccessConnectTables,
-}: ConnectTablesDialogProps & { visible: boolean; onClose: () => void }) => {
- const { ref: projectRef, bucketId } = useParams()
- const { data: project } = useSelectedProjectQuery()
-
- const [isConnecting, setIsConnecting] = useState(false)
- const [connectingStep, setConnectingStep] = useState<PROGRESS_STAGE>()
-
- const form = useForm<ConnectTablesForm>({
- resolver: zodResolver(FormSchema),
- defaultValues: { tables: [] },
- })
- const { tables: selectedTables } = form.watch()
-
- const { data: wrapperInstance } = useAnalyticsBucketWrapperInstance({ bucketId: bucketId })
- const wrapperValues = convertKVStringArrayToJson(wrapperInstance?.server_options ?? [])
-
- const { data: projectSettings } = useProjectSettingsV2Query({ projectRef })
- const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
- const { data: apiKeys } = useAPIKeysQuery(
- { projectRef, reveal: true },
- { enabled: canReadAPIKeys }
- )
- const { serviceKey } = getKeys(apiKeys)
-
- const { sourceId, pipeline, publication } = useAnalyticsBucketAssociatedEntities({
- projectRef,
- bucketId,
- })
- const isEditingExistingPublication = !!publication && !!pipeline
-
- const { data: tables } = useReplicationTablesQuery({ projectRef, sourceId })
-
- const { mutateAsync: createPublication } = useCreatePublicationMutation()
- const { mutateAsync: updatePublication } = useUpdatePublicationMutation()
- const { mutateAsync: createDestinationPipeline } = useCreateDestinationPipelineMutation()
- const { mutateAsync: startPipeline } = useStartPipelineMutation()
-
- const progressIndicator = useMemo(
- () => (isEditingExistingPublication ? PROGRESS_INDICATORS.UPDATE : PROGRESS_INDICATORS.CREATE),
- // [Joshen] This is to prevent the progressIndicator from flipping to UPDATE in the middle of CREATE
- // since the publication and pipelines are getting created in the middle of CREATE
- // eslint-disable-next-line react-hooks/exhaustive-deps
- [isConnecting]
- )
- const totalProgressSteps = progressIndicator.length
- const currentStep = progressIndicator.findIndex((x) => x.step === connectingStep) + 1
- const progressDescription = progressIndicator.find((x) => x.step === connectingStep)
-
- const onSubmitNewPublication: SubmitHandler<ConnectTablesForm> = async (values) => {
- if (!projectRef) return console.error('Project ref is required')
- if (!bucketId) return toast.error('Bucket ID is required')
- if (!sourceId) return toast.error('Replication has not been enabled for your project')
-
- try {
- setIsConnecting(true)
-
- // Step 1: Create publication
- setConnectingStep(PROGRESS_STAGE.CREATE_PUBLICATION)
- const publicationName = getAnalyticsBucketPublicationName(bucketId)
- const publicationTables = values.tables.map((table) => {
- const [schema, name] = table.split('.')
- return { schema, name }
- })
- await createPublication({
- projectRef,
- sourceId,
- name: publicationName,
- tables: publicationTables,
- })
-
- // Step 2: Create destination pipeline
- setConnectingStep(PROGRESS_STAGE.CREATE_PIPELINE)
- const keysToDecrypt = Object.entries(wrapperValues)
- .filter(([name]) =>
- ['vault_aws_access_key_id', 'vault_aws_secret_access_key'].includes(name)
- )
- .map(([_, keyId]) => keyId)
- const decryptedValues = await getDecryptedValues({
- projectRef,
- connectionString: project?.connectionString,
- ids: keysToDecrypt,
- })
-
- const warehouseName = bucketId
- const catalogToken = serviceKey?.api_key ?? ''
- const s3AccessKeyId = decryptedValues[wrapperValues['vault_aws_access_key_id']]
- const s3SecretAccessKey = decryptedValues[wrapperValues['vault_aws_secret_access_key']]
- const s3Region = projectSettings?.region ?? ''
-
- const icebergConfiguration = {
- projectRef,
- warehouseName,
- catalogToken,
- s3AccessKeyId,
- s3SecretAccessKey,
- s3Region,
- }
- const destinationName = getAnalyticsBucketsDestinationName(bucketId)
- const { pipeline_id: pipelineId } = await createDestinationPipeline({
- projectRef,
- destinationName,
- destinationConfig: { iceberg: icebergConfiguration },
- sourceId,
- pipelineConfig: { publicationName },
- })
-
- // Step 3: Start the destination pipeline
- setConnectingStep(PROGRESS_STAGE.START_PIPELINE)
- await startPipeline({ projectRef, pipelineId })
-
- onSuccessConnectTables?.()
- toast.success(`Connected ${values.tables.length} tables to Analytics bucket!`)
- form.reset()
- onClose()
- } catch (error: any) {
- // [Joshen] JFYI there's several async processes here so if something goes wrong midway - we need to figure out how to roll back cleanly
- // e.g publication gets created, but namespace creation fails -> should the old publication get deleted?
- // Another question is probably whether all of these step by step logic should be at the API level instead of client level
- // Same issue present within DestinationPanel - it's alright for now as we do an Alpha but this needs to be addressed before GA
- toast.error(`Failed to connect tables: ${error.message}`)
- } finally {
- setIsConnecting(false)
- setConnectingStep(undefined)
- }
- }
-
- const onSubmitUpdatePublication: SubmitHandler<ConnectTablesForm> = async (values) => {
- if (!projectRef) return console.error('Project ref is required')
- if (!sourceId) return toast.error('Replication has not been enabled on this project')
- if (!bucketId) return toast.error('Bucket ID is required')
- if (!publication) return toast.error('Unable to find existing publication')
- if (!pipeline) return toast.error('Unable to find existing pipeline')
-
- try {
- setIsConnecting(true)
-
- const tablesToBeAdded = values.tables.map((table) => {
- const [schema, name] = table.split('.')
- return { schema, name }
- })
- setConnectingStep(PROGRESS_STAGE.UPDATE_PUBLICATION)
- const publicationTables = publication.tables.concat(tablesToBeAdded)
- await updatePublication({
- projectRef,
- sourceId,
- publicationName: publication.name,
- tables: publicationTables,
- })
-
- setConnectingStep(PROGRESS_STAGE.START_PIPELINE)
- await startPipeline({ projectRef, pipelineId: pipeline.id })
-
- onSuccessConnectTables?.()
- toast.success('Successfully updated connected tables! Pipeline is being restarted')
- onClose()
- } catch (error: any) {
- toast.error(`Failed to update tables: ${error.message}`)
- } finally {
- setIsConnecting(false)
- setConnectingStep(undefined)
- }
- }
-
- const onSubmit: SubmitHandler<ConnectTablesForm> = async (values) => {
- if (isEditingExistingPublication) {
- onSubmitUpdatePublication(values)
- } else {
- onSubmitNewPublication(values)
- }
- }
-
- useEffect(() => {
- form.reset()
- // eslint-disable-next-line react-hooks/exhaustive-deps
- }, [visible])
-
- return (
- <DialogContent>
- <DialogHeader>
- <DialogTitle>
- {isEditingExistingPublication ? 'Connect more tables' : 'Connect tables'}
- </DialogTitle>
- </DialogHeader>
-
- <DialogSectionSeparator />
-
- <Form_Shadcn_ {...form}>
- <form id={formId} onSubmit={form.handleSubmit(onSubmit)}>
- <DialogSection className="flex flex-col gap-y-4">
- <p className="text-sm">
- Select the database tables to send data from. A destination analytics table will be
- created for each, and data will replicate automatically.
- </p>
- </DialogSection>
- <DialogSectionSeparator />
- <DialogSection className="overflow-visible">
- <FormField_Shadcn_
- control={form.control}
- name="tables"
- render={({ field }) => (
- <FormItemLayout label="Tables">
- <FormControl_Shadcn_>
- <MultiSelector
- values={field.value}
- onValuesChange={field.onChange}
- disabled={isConnecting}
- >
- <MultiSelector.Trigger
- deletableBadge
- badgeLimit="wrap"
- mode="inline-combobox"
- label="Select tables..."
- />
- <MultiSelector.Content>
- <MultiSelector.List>
- {tables?.map((table) => {
- const alreadyConnected = (publication?.tables ?? []).some(
- (x) => x.schema === table.schema && x.name === table.name
- )
- return (
- <MultiSelector.Item
- disabled={alreadyConnected}
- className="[&>div]:flex [&>div]:items-center [&>div]:justify-between"
- key={`${table.schema}.${table.name}`}
- value={`${table.schema}.${table.name}`}
- >
- <span>{`${table.schema}.${table.name}`}</span>
- {alreadyConnected && <span>Connected to analytics bucket</span>}
- </MultiSelector.Item>
- )
- })}
- </MultiSelector.List>
- </MultiSelector.Content>
- </MultiSelector>
- </FormControl_Shadcn_>
- </FormItemLayout>
- )}
- />
- </DialogSection>
- </form>
- </Form_Shadcn_>
-
- <AnimatePresence mode="wait">
- {isConnecting && !!progressDescription && (
- <motion.div
- initial={{ height: 0, opacity: 0 }}
- animate={{ height: 'auto', opacity: 1 }}
- exit={{ height: 0, opacity: 0 }}
- transition={{ duration: 0.2, ease: 'easeOut' }}
- >
- <DialogSectionSeparator />
- <DialogSection>
- <div className="flex items-center gap-x-2 mb-2">
- <p className="text-sm text-foreground-light">
- {'getDescription' in progressDescription
- ? progressDescription.getDescription?.(selectedTables.length)
- : progressDescription.description}
- </p>
- <Loader2 size={14} className="animate-spin" />
- </div>
- <Progress value={(currentStep / (totalProgressSteps + 1)) * 100} />
- </DialogSection>
- </motion.div>
- )}
- </AnimatePresence>
-
- <DialogFooter>
- <Button type="default" disabled={isConnecting} onClick={onClose}>
- Cancel
- </Button>
- <Button form={formId} htmlType="submit" disabled={isConnecting}>
- Connect
- </Button>
- </DialogFooter>
- </DialogContent>
- )
-}
-
-const EnableReplicationDialogContent = ({ onClose }: { onClose: () => void }) => {
- const { ref: projectRef } = useParams()
- const enablePgReplicate = useIsETLPrivateAlpha()
- const { error } = useReplicationSourcesQuery({ projectRef })
- const noAccessToReplication =
- !enablePgReplicate || error?.message.includes('feature flag is required')
-
- const { mutateAsync: createTenantSource, isPending: creatingTenantSource } =
- useCreateTenantSourceMutation()
-
- const onEnableReplication = async () => {
- if (!projectRef) return console.error('Project ref is required')
- await createTenantSource({ projectRef })
- }
-
- return (
- <DialogContent>
- <DialogHeader>
- <DialogTitle>Database replication needs to be enabled</DialogTitle>
- <DialogDescription>
- Replication is used to sync data from your Postgres tables
- </DialogDescription>
- </DialogHeader>
- <DialogSectionSeparator />
- <DialogSection className="flex flex-col gap-y-2 !p-0">
- <Admonition
- type="warning"
- className="rounded-none border-0"
- title={
- noAccessToReplication
- ? 'Replication is currently unavailable for your project'
- : 'Replication is currently in Alpha'
- }
- >
- {noAccessToReplication ? (
- <p className="text-sm !leading-normal">
- Access to database replication is currently not available yet for public use. If
- you're interested, do reach out to us via support!
- </p>
- ) : (
- <>
- <p className="text-sm !leading-normal">
- This feature is in active development and may change as we gather feedback.
- Availability and behavior can evolve while in Alpha.
- </p>
- <p className="text-sm !leading-normal">
- Pricing has not been finalized yet. You can enable replication now; we’ll announce
- pricing later and notify you before any charges apply.
- </p>
- </>
- )}
- </Admonition>
- </DialogSection>
- <DialogFooter>
- <Button type="default" disabled={creatingTenantSource} onClick={() => onClose()}>
- {noAccessToReplication ? 'Understood' : 'Cancel'}
- </Button>
- {!noAccessToReplication && (
- <Button type="primary" loading={creatingTenantSource} onClick={onEnableReplication}>
- Enable replication
- </Button>
- )}
- </DialogFooter>
- </DialogContent>
- )
-}
diff --git a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/index.tsx b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/index.tsx
index a0b49ed3105..3d4ffbc6f9f 100644
--- a/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/index.tsx
+++ b/apps/studio/components/interfaces/Storage/AnalyticsBuckets/AnalyticsBucketDetails/index.tsx
@@ -1,6 +1,6 @@
import { useParams } from 'common'
import { uniq } from 'lodash'
-import { Loader2, SquarePlus } from 'lucide-react'
+import { Loader2 } from 'lucide-react'
import Link from 'next/link'
import { useRouter } from 'next/router'
import { parseAsBoolean, useQueryState } from 'nuqs'
@@ -14,7 +14,6 @@ import { DeleteAnalyticsBucketModal } from '../DeleteAnalyticsBucketModal'
import { useSelectedAnalyticsBucket } from '../useSelectedAnalyticsBucket'
import { HIDE_REPLICATION_USER_FLOW } from './AnalyticsBucketDetails.constants'
import { BucketHeader } from './BucketHeader'
-import { ConnectTablesDialog } from './ConnectTablesDialog'
import { CreateTableInstructions } from './CreateTable/CreateTableInstructions'
import { NamespaceWithTables } from './NamespaceWithTables'
import { SimpleConfigurationDetails } from './SimpleConfigurationDetails'
@@ -202,13 +201,7 @@ export const AnalyticBucketDetails = () => {
) : state === 'added' && wrapperInstance ? (
<>
<ScaffoldSection isFullWidth>
- <BucketHeader
- namespaces={namespaces}
- onSuccessConnectTables={() => {
- setPollIntervalNamespaces(4000)
- setPollIntervalNamespaceTables(4000)
- }}
- />
+ <BucketHeader />
{isLoadingNamespaces || isLoadingWrapperInstance ? (
<GenericTableLoader headers={['Name']} />
@@ -228,20 +221,7 @@ export const AnalyticBucketDetails = () => {
title="Connecting table(s) to bucket"
description="Tables will be shown here once the connection is complete"
/>
- ) : (
- <EmptyStatePresentational
- icon={SquarePlus}
- title="Connect database tables"
- description="Stream table data for continuous backups and analysis"
- >
- <ConnectTablesDialog
- onSuccessConnectTables={() => {
- setPollIntervalNamespaces(4000)
- setPollIntervalNamespaceTables(4000)
- }}
- />
- </EmptyStatePresentational>
- )}
+ ) : null}
</>
) : (
<>
diff --git a/apps/studio/data/replication/delete-tenant-mutation.ts b/apps/studio/data/replication/delete-tenant-mutation.ts
new file mode 100644
index 00000000000..d7696c6225c
--- /dev/null
+++ b/apps/studio/data/replication/delete-tenant-mutation.ts
@@ -0,0 +1,65 @@
+import { useMutation, useQueryClient } from '@tanstack/react-query'
+import { toast } from 'sonner'
+
+import { replicationKeys } from './keys'
+import { del, handleError } from '@/data/fetchers'
+import type { ResponseError, UseCustomMutationOptions } from '@/types'
+
+export type DeleteReplicationTenantParams = {
+ projectRef: string
+}
+
+export async function deleteReplicationTenant(
+ { projectRef }: DeleteReplicationTenantParams,
+ signal?: AbortSignal
+) {
+ if (!projectRef) throw new Error('projectRef is required')
+
+ const { data, error } = await del('/platform/replication/{ref}/tenants', {
+ params: { path: { ref: projectRef } },
+ signal,
+ })
+ if (error) handleError(error)
+
+ return data
+}
+
+type DeleteReplicationTenantData = Awaited<ReturnType<typeof deleteReplicationTenant>>
+
+export const useDeleteReplicationTenantMutation = ({
+ onSuccess,
+ onError,
+ ...options
+}: Omit<
+ UseCustomMutationOptions<
+ DeleteReplicationTenantData,
+ ResponseError,
+ DeleteReplicationTenantParams
+ >,
+ 'mutationFn'
+> = {}) => {
+ const queryClient = useQueryClient()
+
+ return useMutation<DeleteReplicationTenantData, ResponseError, DeleteReplicationTenantParams>({
+ mutationFn: (vars) => deleteReplicationTenant(vars),
+ async onSuccess(data, variables, context) {
+ const { projectRef } = variables
+
+ await Promise.all([
+ queryClient.invalidateQueries({ queryKey: replicationKeys.sources(projectRef) }),
+ queryClient.invalidateQueries({ queryKey: replicationKeys.destinations(projectRef) }),
+ queryClient.invalidateQueries({ queryKey: replicationKeys.pipelines(projectRef) }),
+ ])
+
+ await onSuccess?.(data, variables, context)
+ },
+ async onError(data, variables, context) {
+ if (onError === undefined) {
+ toast.error(`Failed to disable external replication: ${data.message}`)
+ } else {
+ onError(data, variables, context)
+ }
+ },
+ ...options,
+ })
+}
diff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.ts
index d2501aff841..30cb5676d1b 100644
--- a/packages/api-types/types/platform.d.ts
+++ b/packages/api-types/types/platform.d.ts
@@ -3835,6 +3835,26 @@ export interface paths {
patch?: never
trace?: never
}
+ '/platform/replication/{ref}/tenants': {
+ parameters: {
+ query?: never
+ header?: never
+ path?: never
+ cookie?: never
+ }
+ get?: never
+ put?: never
+ post?: never
+ /**
+ * Delete tenant
+ * @description Delete the replication tenant for the project. Requires bearer auth.
+ */
+ delete: operations['ReplicationTenantsController_deleteTenant']
+ options?: never
+ head?: never
+ patch?: never
+ trace?: never
+ }
'/platform/replication/{ref}/tenants-sources': {
parameters: {
query?: never
@@ -6127,6 +6147,23 @@ export interface components {
DeclineAuthorizationResponse: {
id: string
}
+ DeleteDestinationPipelineResponse: {
+ /**
+ * @description Whether the destination was deleted. True when no other pipelines remain attached to it.
+ * @example true
+ */
+ destination_deleted: boolean
+ /**
+ * @description Destination id
+ * @example 2001
+ */
+ destination_id: number
+ /**
+ * @description Pipeline id
+ * @example 1012
+ */
+ pipeline_id: number
+ }
DeleteOAuthAppResponse: {
client_id: string
created_at: string
@@ -23703,7 +23740,9 @@ export interface operations {
headers: {
[name: string]: unknown
}
- content?: never
+ content: {
+ 'application/json': components['schemas']['DeleteDestinationPipelineResponse']
+ }
}
/** @description Unauthorized */
401: {
@@ -23719,6 +23758,20 @@ export interface operations {
}
content?: never
}
+ /** @description Pipeline or destination not found. */
+ 404: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Pipeline is still active. */
+ 409: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
/** @description Rate limit exceeded */
429: {
headers: {
@@ -23878,6 +23931,20 @@ export interface operations {
}
content?: never
}
+ /** @description Destination not found. */
+ 404: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Destination has an active or attached pipeline. */
+ 409: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
/** @description Rate limit exceeded */
429: {
headers: {
@@ -24198,6 +24265,20 @@ export interface operations {
}
content?: never
}
+ /** @description Pipeline not found. */
+ 404: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Pipeline is still active. */
+ 409: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
/** @description Rate limit exceeded */
429: {
headers: {
@@ -24946,6 +25027,13 @@ export interface operations {
}
content?: never
}
+ /** @description Source not found. */
+ 404: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
/** @description Rate limit exceeded */
429: {
headers: {
@@ -25015,6 +25103,69 @@ export interface operations {
}
}
}
+ ReplicationTenantsController_deleteTenant: {
+ parameters: {
+ query?: never
+ header?: never
+ path: {
+ /** @description Project ref */
+ ref: string
+ }
+ cookie?: never
+ }
+ requestBody?: never
+ responses: {
+ /** @description Tenant deleted. */
+ 200: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Unauthorized */
+ 401: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Forbidden action */
+ 403: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Tenant not found. */
+ 404: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Tenant has active pipelines. */
+ 409: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Rate limit exceeded */
+ 429: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ /** @description Unexpected error while deleting tenant. */
+ 500: {
+ headers: {
+ [name: string]: unknown
+ }
+ content?: never
+ }
+ }
+ }
ReplicationTenantsSourcesController_createTenantSource: {
parameters: {
query?: never