Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

ref(etl): Update configuration and add new parameters (#43126)

Riccardo Busettid41ab5b4ff18
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
index 3971a14d673..475b47d8246 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx
@@ -1,6 +1,5 @@
import type { ChangeEvent } from 'react'
import type { UseFormReturn } from 'react-hook-form'
-
import {
Accordion_Shadcn_,
AccordionContent_Shadcn_,
@@ -11,8 +10,13 @@ import {
FormField_Shadcn_,
Input_Shadcn_,
PrePostTab,
+ Select_Shadcn_,
+ SelectContent_Shadcn_,
+ SelectItem_Shadcn_,
+ SelectTrigger_Shadcn_,
} from 'ui'
import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
+
import { DestinationType } from '../DestinationPanel.types'
import { type DestinationPanelSchemaType } from './DestinationForm.schema'
@@ -53,11 +57,11 @@ export const AdvancedSettings = ({
description={
<>
<p>
- Maximum time the pipeline waits to collect additional changes before
- flushing a batch.
+ Maximum time pipeline waits to collect additional changes before flushing a
+ batch.
</p>
<p>
- Lower values reduce replication latency; higher values improve batching
+ Lower values reduce replication latency, higher values improve batching
efficiency.
</p>
</>
@@ -122,7 +126,7 @@ export const AdvancedSettings = ({
Number of parallel connections each table copy can use during initial sync.
</p>
<p>
- Higher values can copy large tables faster, but consume more database
+ More connections speed up large table copies, but use more database
connections.
</p>
</>
@@ -143,47 +147,118 @@ export const AdvancedSettings = ({
)}
/>
- {/* BigQuery-specific: Max staleness */}
+ <FormField_Shadcn_
+ control={form.control}
+ name="invalidatedSlotBehavior"
+ render={({ field }) => (
+ <FormItemLayout
+ label="Invalidated slot behavior"
+ layout="horizontal"
+ description="Behavior when the replication slot is invalidated"
+ >
+ <FormControl_Shadcn_>
+ <Select_Shadcn_ value={field.value ?? 'error'} onValueChange={field.onChange}>
+ <SelectTrigger_Shadcn_ className="capitalize">
+ {field.value ?? 'error'}
+ </SelectTrigger_Shadcn_>
+ <SelectContent_Shadcn_>
+ <SelectItem_Shadcn_ value="error" className="[&>span]:top-2.5">
+ <p>Error</p>
+ <p className="text-foreground-lighter">
+ Blocks startup for manual recovery
+ </p>
+ </SelectItem_Shadcn_>
+ <SelectItem_Shadcn_ value="recreate" className="[&>span]:top-2.5">
+ <p>Recreate</p>
+ <p className="text-foreground-lighter">
+ Rebuilds the slot and restarts replication from scratch
+ </p>
+ </SelectItem_Shadcn_>
+ </SelectContent_Shadcn_>
+ </Select_Shadcn_>
+ </FormControl_Shadcn_>
+ </FormItemLayout>
+ )}
+ />
+
{type === 'BigQuery' && (
- <FormField_Shadcn_
- control={form.control}
- name="maxStalenessMins"
- render={({ field }) => (
- <FormItemLayout
- label={
- <div className="flex flex-col gap-y-2">
- <span>Maximum staleness</span>
- <Badge className="w-min">BigQuery only</Badge>
- </div>
- }
- layout="horizontal"
- description={
- <>
- <p>
- Maximum allowed age for BigQuery cached metadata before reading base
- tables.
- </p>
- <p>
- Lower values improve freshness; higher values can reduce query cost and
- latency.
- </p>
- </>
- }
- >
- <FormControl_Shadcn_>
- <PrePostTab postTab="minutes">
- <Input_Shadcn_
- {...field}
- type="number"
- value={field.value ?? ''}
- onChange={handleNumberChange(field)}
- placeholder="Default: None (No staleness limit)"
- />
- </PrePostTab>
- </FormControl_Shadcn_>
- </FormItemLayout>
- )}
- />
+ <>
+ <FormField_Shadcn_
+ control={form.control}
+ name="connectionPoolSize"
+ render={({ field }) => (
+ <FormItemLayout
+ label={
+ <div className="flex flex-col gap-y-2">
+ <span>Connection pool size</span>
+ <Badge className="w-min">BigQuery only</Badge>
+ </div>
+ }
+ layout="horizontal"
+ description={
+ <>
+ <p>Size of the BigQuery Storage Write API connection pool.</p>
+ <p>
+ More connections allow more parallel writes, but consume more resources.
+ </p>
+ </>
+ }
+ >
+ <FormControl_Shadcn_>
+ <PrePostTab postTab="connections">
+ <Input_Shadcn_
+ {...field}
+ type="number"
+ value={field.value ?? ''}
+ onChange={handleNumberChange(field)}
+ placeholder="Default: 4"
+ />
+ </PrePostTab>
+ </FormControl_Shadcn_>
+ </FormItemLayout>
+ )}
+ />
+
+ <FormField_Shadcn_
+ control={form.control}
+ name="maxStalenessMins"
+ render={({ field }) => (
+ <FormItemLayout
+ label={
+ <div className="flex flex-col gap-y-2">
+ <span>Maximum staleness</span>
+ <Badge className="w-min">BigQuery only</Badge>
+ </div>
+ }
+ layout="horizontal"
+ description={
+ <>
+ <p>
+ Maximum allowed age for BigQuery cached metadata before reading base
+ tables.
+ </p>
+ <p>
+ Lower values improve freshness, higher values can reduce query cost and
+ latency.
+ </p>
+ </>
+ }
+ >
+ <FormControl_Shadcn_>
+ <PrePostTab postTab="minutes">
+ <Input_Shadcn_
+ {...field}
+ type="number"
+ value={field.value ?? ''}
+ onChange={handleNumberChange(field)}
+ placeholder="Default: None (No staleness limit)"
+ />
+ </PrePostTab>
+ </FormControl_Shadcn_>
+ </FormItemLayout>
+ )}
+ />
+ </>
)}
</AccordionContent_Shadcn_>
</AccordionItem_Shadcn_>
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
index a8949f27729..d2ff933585d 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts
@@ -17,10 +17,12 @@ export const DestinationPanelFormSchema = z.object({
.int()
.min(1, 'Max copy connections per table should be greater than 0')
.optional(),
+ invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(),
// BigQuery fields
projectId: z.string().optional(),
datasetId: z.string().optional(),
serviceAccountKey: z.string().optional(),
+ connectionPoolSize: z.number().int().min(1).optional(),
maxStalenessMins: z.number().nonnegative().optional(),
// Analytics Bucket fields, only warehouse name and namespace are visible + editable fields
warehouseName: z.string().optional(),
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
index 77a95a7cc4d..e9801c0f202 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts
@@ -1,8 +1,20 @@
+import { UseMutateAsyncFunction } from '@tanstack/react-query'
+import { snakeCase } from 'lodash'
import z from 'zod'
import { DestinationType } from '../DestinationPanel.types'
import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
import { DestinationPanelFormSchema } from './DestinationForm.schema'
+import {
+ BigQueryDestinationConfig,
+ DestinationConfig,
+ IcebergDestinationConfig,
+} from '@/data/replication/create-destination-pipeline-mutation'
+import {
+ type CreateS3AccessKeyCredentialVariables,
+ type S3AccessKeyCreateData,
+} from '@/data/storage/s3-access-key-create-mutation'
+import { ResponseError } from '@/types'
// Helper function to build destination config for validation
export const buildDestinationConfigForValidation = ({
@@ -22,7 +34,8 @@ export const buildDestinationConfigForValidation = ({
projectId: data.projectId ?? '',
datasetId: data.datasetId ?? '',
serviceAccountKey: data.serviceAccountKey ?? '',
- ...(data.maxStalenessMins !== undefined ? { maxStalenessMins: data.maxStalenessMins } : {}),
+ connectionPoolSize: data.connectionPoolSize,
+ maxStalenessMins: data.maxStalenessMins,
},
}
} else if (selectedType === 'Analytics Bucket') {
@@ -54,3 +67,67 @@ export const buildDestinationConfigForValidation = ({
throw new Error('Invalid destination type')
}
}
+
+export const buildDestinationConfig = async ({
+ projectRef,
+ selectedType,
+ data,
+ warehouseName,
+ createS3AccessKey,
+ resolveNamespace,
+}: {
+ projectRef?: string
+ selectedType: DestinationType
+ data: z.infer<typeof DestinationPanelFormSchema>
+ warehouseName?: string
+ createS3AccessKey: UseMutateAsyncFunction<
+ S3AccessKeyCreateData,
+ ResponseError,
+ CreateS3AccessKeyCredentialVariables,
+ unknown
+ >
+ resolveNamespace: (
+ data: z.infer<typeof DestinationPanelFormSchema>
+ ) => Promise<string | undefined>
+}) => {
+ if (!projectRef) throw new Error('Project ref is required')
+
+ let destinationConfig: DestinationConfig | undefined = undefined
+
+ if (selectedType === 'BigQuery') {
+ const bigQueryConfig: BigQueryDestinationConfig = {
+ projectId: data.projectId ?? '',
+ datasetId: data.datasetId ?? '',
+ serviceAccountKey: data.serviceAccountKey ?? '',
+ connectionPoolSize: data.connectionPoolSize,
+ maxStalenessMins: data.maxStalenessMins,
+ }
+ destinationConfig = { bigQuery: bigQueryConfig }
+ } else if (selectedType === 'Analytics Bucket') {
+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
+
+ if (data.s3AccessKeyId === CREATE_NEW_KEY) {
+ const newKeys = await createS3AccessKey({
+ projectRef,
+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
+ })
+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
+ }
+
+ // Resolve namespace (create if needed)
+ const finalNamespace = await resolveNamespace(data)
+
+ const icebergConfig: IcebergDestinationConfig = {
+ projectRef: projectRef,
+ warehouseName: data.warehouseName ?? '',
+ namespace: finalNamespace,
+ catalogToken: data.catalogToken ?? '',
+ s3AccessKeyId: s3Keys.accessKey ?? '',
+ s3SecretAccessKey: s3Keys.secretKey ?? '',
+ s3Region: data.s3Region ?? '',
+ }
+ destinationConfig = { iceberg: icebergConfig }
+ }
+
+ return destinationConfig
+}
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
index 8d99386278e..f9291d4db11 100644
--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx
@@ -6,9 +6,6 @@ import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'
import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'
import {
BatchConfig,
- BigQueryDestinationConfig,
- DestinationConfig,
- IcebergDestinationConfig,
useCreateDestinationPipelineMutation,
} from 'data/replication/create-destination-pipeline-mutation'
import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'
@@ -27,7 +24,6 @@ import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespac
import { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'
import { AnimatePresence, motion } from 'framer-motion'
import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'
-import { snakeCase } from 'lodash'
import { Loader2 } from 'lucide-react'
import { useEffect, useMemo, useRef, useState } from 'react'
import { useForm } from 'react-hook-form'
@@ -41,9 +37,12 @@ import * as z from 'zod'
import { DestinationType } from '../DestinationPanel.types'
import { AdvancedSettings } from './AdvancedSettings'
-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
+import { CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'
-import { buildDestinationConfigForValidation } from './DestinationForm.utils'
+import {
+ buildDestinationConfig,
+ buildDestinationConfigForValidation,
+} from './DestinationForm.utils'
import { DestinationNameInput } from './DestinationNameInput'
import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'
import { NewPublicationPanel } from './NewPublicationPanel'
@@ -176,10 +175,16 @@ export const DestinationForm = ({
maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,
maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,
maxCopyConnectionsPerTable: pipelineData?.config?.max_copy_connections_per_table ?? undefined,
+ invalidatedSlotBehavior:
+ (pipelineData?.config as { invalidated_slot_behavior?: 'error' | 'recreate' } | undefined)
+ ?.invalidated_slot_behavior ?? undefined,
// BigQuery fields
projectId: isBigQueryConfig ? config.big_query.project_id : '',
datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',
serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',
+ connectionPoolSize:
+ (config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query
+ ?.connection_pool_size ?? undefined,
maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, // Default: null
// Analytics Bucket fields
warehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',
@@ -305,6 +310,7 @@ export const DestinationForm = ({
maxFillMs: data.maxFillMs,
maxTableSyncWorkers: data.maxTableSyncWorkers,
maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior: data.invalidatedSlotBehavior,
}),
])
@@ -349,10 +355,6 @@ export const DestinationForm = ({
return !hasCriticalFailures
}
- // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty
- // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics
- // At first glance we could try grouping as edit / create bigquery, edit / create analytics
- // since the destination config seems rather similar between edit and create for the same type
const submitPipeline = async (data: z.infer<typeof FormSchema>) => {
if (!projectRef) return console.error('Project ref is required')
if (!sourceId) return console.error('Source id is required')
@@ -361,56 +363,31 @@ export const DestinationForm = ({
}
try {
- if (editMode && existingDestination) {
- if (!existingDestination.pipelineId) return console.error('Pipeline id is required')
+ const destinationConfig = await buildDestinationConfig({
+ projectRef,
+ selectedType,
+ warehouseName,
+ data,
+ createS3AccessKey,
+ resolveNamespace,
+ })
- let destinationConfig: DestinationConfig | undefined = undefined
+ if (!destinationConfig) throw new Error('Destination configuration is missing')
- if (selectedType === 'BigQuery') {
- const bigQueryConfig: BigQueryDestinationConfig = {
- projectId: data.projectId ?? '',
- datasetId: data.datasetId ?? '',
- serviceAccountKey: data.serviceAccountKey ?? '',
- }
- if (!!data.maxStalenessMins) {
- bigQueryConfig.maxStalenessMins = data.maxStalenessMins
- }
- destinationConfig = { bigQuery: bigQueryConfig }
- } else if (selectedType === 'Analytics Bucket') {
- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
-
- if (data.s3AccessKeyId === CREATE_NEW_KEY) {
- const newKeys = await createS3AccessKey({
- projectRef,
- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
- })
- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
- }
+ const batchConfig: BatchConfig | undefined =
+ data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : undefined
+ const hasBatchFields = batchConfig !== undefined
- // Resolve namespace (create if needed)
- const finalNamespace = await resolveNamespace(data)
-
- const icebergConfig: IcebergDestinationConfig = {
- projectRef: projectRef,
- warehouseName: data.warehouseName ?? '',
- namespace: finalNamespace,
- catalogToken: data.catalogToken ?? '',
- s3AccessKeyId: s3Keys.accessKey ?? '',
- s3SecretAccessKey: s3Keys.secretKey ?? '',
- s3Region: data.s3Region ?? '',
- }
- destinationConfig = { iceberg: icebergConfig }
- }
-
- const batchConfig: BatchConfig | undefined =
- data.maxFillMs !== undefined
- ? {
- ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),
- }
- : undefined
- const hasBatchFields = batchConfig !== undefined
+ const pipelineConfig = {
+ publicationName: data.publicationName,
+ maxTableSyncWorkers: data.maxTableSyncWorkers,
+ maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior: data.invalidatedSlotBehavior,
+ ...(hasBatchFields ? { batch: batchConfig } : {}),
+ }
- if (!destinationConfig) throw new Error('Destination configuration is missing')
+ if (editMode && existingDestination) {
+ if (!existingDestination.pipelineId) return console.error('Pipeline id is required')
await updateDestinationPipeline({
destinationId: existingDestination.destinationId,
@@ -418,12 +395,7 @@ export const DestinationForm = ({
projectRef,
destinationName: data.name,
destinationConfig,
- pipelineConfig: {
- publicationName: data.publicationName,
- maxTableSyncWorkers: data.maxTableSyncWorkers,
- maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
- ...(hasBatchFields ? { batch: batchConfig } : {}),
- },
+ pipelineConfig,
sourceId,
})
// Set request status only right before starting, then fire and close
@@ -448,64 +420,12 @@ export const DestinationForm = ({
}
onClose()
} else {
- let destinationConfig: DestinationConfig | undefined = undefined
-
- if (selectedType === 'BigQuery') {
- const bigQueryConfig: BigQueryDestinationConfig = {
- projectId: data.projectId ?? '',
- datasetId: data.datasetId ?? '',
- serviceAccountKey: data.serviceAccountKey ?? '',
- }
- if (!!data.maxStalenessMins) {
- bigQueryConfig.maxStalenessMins = data.maxStalenessMins
- }
- destinationConfig = { bigQuery: bigQueryConfig }
- } else if (selectedType === 'Analytics Bucket') {
- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }
-
- if (data.s3AccessKeyId === CREATE_NEW_KEY) {
- const newKeys = await createS3AccessKey({
- projectRef,
- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,
- })
- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }
- }
-
- // Resolve namespace (create if needed)
- const finalNamespace = await resolveNamespace(data)
-
- const icebergConfig: IcebergDestinationConfig = {
- projectRef: projectRef,
- warehouseName: data.warehouseName ?? '',
- namespace: finalNamespace,
- catalogToken: data.catalogToken ?? '',
- s3AccessKeyId: s3Keys.accessKey ?? '',
- s3SecretAccessKey: s3Keys.secretKey ?? '',
- s3Region: data.s3Region ?? '',
- }
- destinationConfig = { iceberg: icebergConfig }
- }
- const batchConfig: BatchConfig | undefined =
- data.maxFillMs !== undefined
- ? {
- ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),
- }
- : undefined
- const hasBatchFields = batchConfig !== undefined
-
- if (!destinationConfig) throw new Error('Destination configuration is missing')
-
const { pipeline_id: pipelineId } = await createDestinationPipeline({
projectRef,
destinationName: data.name,
destinationConfig,
+ pipelineConfig,
sourceId,
- pipelineConfig: {
- publicationName: data.publicationName,
- maxTableSyncWorkers: data.maxTableSyncWorkers,
- maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,
- ...(hasBatchFields ? { batch: batchConfig } : {}),
- },
})
// Set request status only right before starting, then fire and close
setRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)
diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
index 8a0cd04291c..96e81175306 100644
--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts
@@ -18,6 +18,7 @@ export type BigQueryDestinationConfig = {
projectId: string
datasetId: string
serviceAccountKey: string
+ connectionPoolSize?: number
maxStalenessMins?: number
}
@@ -45,6 +46,7 @@ export type CreateDestinationPipelineParams = {
batch?: BatchConfig
maxTableSyncWorkers?: number
maxCopyConnectionsPerTable?: number
+ invalidatedSlotBehavior?: 'error' | 'recreate'
}
}
@@ -53,7 +55,13 @@ async function createDestinationPipeline(
projectRef,
destinationName: destinationName,
destinationConfig,
- pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },
+ pipelineConfig: {
+ publicationName,
+ batch,
+ maxTableSyncWorkers,
+ maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior,
+ },
sourceId,
}: CreateDestinationPipelineParams,
signal?: AbortSignal
@@ -64,16 +72,18 @@ async function createDestinationPipeline(
let destination_config: components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
if ('bigQuery' in destinationConfig) {
- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery
+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =
+ destinationConfig.bigQuery
destination_config = {
big_query: {
project_id: projectId,
dataset_id: datasetId,
service_account_key: serviceAccountKey,
- ...(maxStalenessMins !== null ? { max_staleness_mins: maxStalenessMins } : {}),
+ connection_pool_size: connectionPoolSize,
+ max_staleness_mins: maxStalenessMins,
},
- }
+ } as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']
} else if ('iceberg' in destinationConfig) {
const {
projectRef: icebergProjectRef,
@@ -102,35 +112,27 @@ async function createDestinationPipeline(
throw new Error('Invalid destination config: must specify either bigQuery or iceberg')
}
+ const pipeline_config = {
+ publication_name: publicationName,
+ max_table_sync_workers: maxTableSyncWorkers,
+ max_copy_connections_per_table: maxCopyConnectionsPerTable,
+ invalidated_slot_behavior: invalidatedSlotBehavior,
+ batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,
+ }
+
const { data, error } = await post('/platform/replication/{ref}/destinations-pipelines', {
params: { path: { ref: projectRef } },
body: {
source_id: sourceId,
destination_name: destinationName,
destination_config,
- pipeline_config: {
- publication_name: publicationName,
- ...(maxTableSyncWorkers !== undefined
- ? { max_table_sync_workers: maxTableSyncWorkers }
- : {}),
- ...(maxCopyConnectionsPerTable !== undefined
- ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }
- : {}),
- ...(batch
- ? {
- batch: {
- ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),
- },
- }
- : {}),
- },
+ pipeline_config:
+ pipeline_config as components['schemas']['CreateReplicationDestinationPipelineBody']['pipeline_config'],
},
signal,
})
- if (error) {
- handleError(error)
- }
+ if (error) handleError(error)
return data
}
diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
index a48f475357c..d86f1287d6a 100644
--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts
+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts
@@ -19,6 +19,7 @@ export type UpdateDestinationPipelineParams = {
batch?: BatchConfig
maxTableSyncWorkers?: number
maxCopyConnectionsPerTable?: number
+ invalidatedSlotBehavior?: 'error' | 'recreate'
}
}
@@ -29,7 +30,13 @@ async function updateDestinationPipeline(
projectRef,
destinationName: destinationName,
destinationConfig,
- pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },
+ pipelineConfig: {
+ publicationName,
+ batch,
+ maxTableSyncWorkers,
+ maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior,
+ },
sourceId,
}: UpdateDestinationPipelineParams,
signal?: AbortSignal
@@ -40,15 +47,17 @@ async function updateDestinationPipeline(
let destination_config: components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']
if ('bigQuery' in destinationConfig) {
- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery
+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =
+ destinationConfig.bigQuery
destination_config = {
big_query: {
project_id: projectId,
dataset_id: datasetId,
service_account_key: serviceAccountKey,
- ...(maxStalenessMins !== null ? { max_staleness_mins: maxStalenessMins } : {}),
+ connection_pool_size: connectionPoolSize,
+ max_staleness_mins: maxStalenessMins,
},
- }
+ } as components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']
} else if ('iceberg' in destinationConfig) {
const {
projectRef: icebergProjectRef,
@@ -76,6 +85,14 @@ async function updateDestinationPipeline(
throw new Error('Invalid destination config: must specify either bigQuery or iceberg')
}
+ const pipeline_config = {
+ publication_name: publicationName,
+ max_table_sync_workers: maxTableSyncWorkers,
+ max_copy_connections_per_table: maxCopyConnectionsPerTable,
+ invalidated_slot_behavior: invalidatedSlotBehavior,
+ batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,
+ }
+
const { data, error } = await post(
'/platform/replication/{ref}/destinations-pipelines/{destination_id}/{pipeline_id}',
{
@@ -84,26 +101,13 @@ async function updateDestinationPipeline(
destination_config,
source_id: sourceId,
destination_name: destinationName,
- pipeline_config: {
- publication_name: publicationName,
- ...(maxTableSyncWorkers !== undefined
- ? { max_table_sync_workers: maxTableSyncWorkers }
- : {}),
- ...(maxCopyConnectionsPerTable !== undefined
- ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }
- : {}),
- ...(batch
- ? {
- batch: {
- ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),
- },
- }
- : {}),
- },
+ pipeline_config:
+ pipeline_config as components['schemas']['UpdateReplicationDestinationPipelineBody']['pipeline_config'],
},
signal,
}
)
+
if (error) handleError(error)
return data
}
diff --git a/apps/studio/data/replication/validate-destination-mutation.ts b/apps/studio/data/replication/validate-destination-mutation.ts
index f9468b2ae3a..37c59fafde9 100644
--- a/apps/studio/data/replication/validate-destination-mutation.ts
+++ b/apps/studio/data/replication/validate-destination-mutation.ts
@@ -1,8 +1,8 @@
import { useMutation } from '@tanstack/react-query'
-
import type { components } from 'api-types'
import { handleError, post } from 'data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from 'types'
+
import { DestinationConfig } from './create-destination-pipeline-mutation'
type ValidateDestinationParams = {
@@ -23,16 +23,18 @@ async function validateDestination(
let config: components['schemas']['ValidateReplicationDestinationBody']['config']
if ('bigQuery' in destinationConfig) {
- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery
+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =
+ destinationConfig.bigQuery
config = {
big_query: {
project_id: projectId,
dataset_id: datasetId,
service_account_key: serviceAccountKey,
- ...(maxStalenessMins !== undefined ? { max_staleness_mins: maxStalenessMins } : {}),
+ connection_pool_size: connectionPoolSize,
+ max_staleness_mins: maxStalenessMins,
},
- }
+ } as components['schemas']['ValidateReplicationDestinationBody']['config']
} else if ('iceberg' in destinationConfig) {
const {
projectRef: icebergProjectRef,
diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.ts
index af822a5bd07..a248559aecf 100644
--- a/apps/studio/data/replication/validate-pipeline-mutation.ts
+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts
@@ -1,6 +1,5 @@
import { useMutation } from '@tanstack/react-query'
import { components } from 'api-types'
-
import { handleError, post } from 'data/fetchers'
import type { ResponseError, UseCustomMutationOptions } from 'types'
@@ -11,6 +10,7 @@ type ValidatePipelineParams = {
maxFillMs?: number
maxTableSyncWorkers?: number
maxCopyConnectionsPerTable?: number
+ invalidatedSlotBehavior?: 'error' | 'recreate'
}
type ValidatePipelineResponse = components['schemas']['ValidatePipelineResponse']
@@ -22,6 +22,7 @@ async function validatePipeline(
maxFillMs,
maxTableSyncWorkers,
maxCopyConnectionsPerTable,
+ invalidatedSlotBehavior,
}: ValidatePipelineParams,
signal?: AbortSignal
): Promise<ValidatePipelineResponse> {
@@ -30,16 +31,19 @@ async function validatePipeline(
const batchConfig = maxFillMs !== undefined ? { max_fill_ms: maxFillMs } : undefined
+ const config = {
+ publication_name: publicationName,
+ max_table_sync_workers: maxTableSyncWorkers,
+ max_copy_connections_per_table: maxCopyConnectionsPerTable,
+ invalidated_slot_behavior: invalidatedSlotBehavior,
+ batch: batchConfig,
+ }
+
const { data, error } = await post('/platform/replication/{ref}/pipelines/validate', {
params: { path: { ref: projectRef } },
body: {
source_id: sourceId,
- config: {
- publication_name: publicationName,
- max_table_sync_workers: maxTableSyncWorkers,
- max_copy_connections_per_table: maxCopyConnectionsPerTable,
- batch: batchConfig,
- },
+ config: config as components['schemas']['ValidateReplicationPipelineBody']['config'],
},
signal,
})
diff --git a/apps/studio/data/storage/s3-access-key-create-mutation.ts b/apps/studio/data/storage/s3-access-key-create-mutation.ts
index 362d47ef557..2536773648e 100644
--- a/apps/studio/data/storage/s3-access-key-create-mutation.ts
+++ b/apps/studio/data/storage/s3-access-key-create-mutation.ts
@@ -1,11 +1,11 @@
import { useMutation, useQueryClient } from '@tanstack/react-query'
-import { toast } from 'sonner'
-
import { handleError, post } from 'data/fetchers'
+import { toast } from 'sonner'
import type { ResponseError, UseCustomMutationOptions } from 'types'
+
import { storageCredentialsKeys } from './s3-access-key-keys'
-type CreateS3AccessKeyCredentialVariables = {
+export type CreateS3AccessKeyCredentialVariables = {
description: string
projectRef?: string
}
@@ -25,7 +25,7 @@ const createS3AccessKeyCredential = async ({
return data
}
-type S3AccessKeyCreateData = Awaited<ReturnType<typeof createS3AccessKeyCredential>>
+export type S3AccessKeyCreateData = Awaited<ReturnType<typeof createS3AccessKeyCredential>>
export function useS3AccessKeyCreateMutation({
onSuccess,
diff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.ts
index 9d917f93f37..2915887577f 100644
--- a/packages/api-types/types/platform.d.ts
+++ b/packages/api-types/types/platform.d.ts
@@ -5296,16 +5296,16 @@ export interface components {
config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -5372,16 +5372,16 @@ export interface components {
destination_config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -5452,6 +5452,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -5479,6 +5485,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -8461,16 +8473,16 @@ export interface components {
config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -8549,16 +8561,16 @@ export interface components {
config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -8761,6 +8773,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -8820,6 +8838,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -10198,16 +10222,16 @@ export interface components {
config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -10274,16 +10298,16 @@ export interface components {
destination_config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -10354,6 +10378,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -10381,6 +10411,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */
@@ -10712,16 +10748,16 @@ export interface components {
config:
| {
big_query: {
+ /**
+ * @description Number of concurrent BigQuery Storage Write API connections.
+ * @example 8
+ */
+ connection_pool_size?: number
/**
* @description BigQuery dataset id
* @example analytics
*/
dataset_id: string
- /**
- * @description Maximum number of concurrent write streams
- * @example 8
- */
- max_concurrent_streams?: number
/**
* @description Maximum data staleness in minutes
* @example 5
@@ -10789,6 +10825,12 @@ export interface components {
*/
max_fill_ms?: number
}
+ /**
+ * @description Behavior when the replication slot is invalidated
+ * @example error
+ * @enum {string}
+ */
+ invalidated_slot_behavior?: 'error' | 'recreate'
/** @description Maximum number of copy connections per table */
max_copy_connections_per_table?: number
/** @description Maximum number of table sync workers */