The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
ref(etl): Update configuration and add new parameters (#43126)
Riccardo Busettid41ab5b4ff18
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxindex 3971a14d673..475b47d8246 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx@@ -1,6 +1,5 @@import type { ChangeEvent } from 'react'import type { UseFormReturn } from 'react-hook-form'-import {Accordion_Shadcn_,AccordionContent_Shadcn_,@@ -11,8 +10,13 @@ import {FormField_Shadcn_,Input_Shadcn_,PrePostTab,+ Select_Shadcn_,+ SelectContent_Shadcn_,+ SelectItem_Shadcn_,+ SelectTrigger_Shadcn_,} from 'ui'import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'+import { DestinationType } from '../DestinationPanel.types'import { type DestinationPanelSchemaType } from './DestinationForm.schema'@@ -53,11 +57,11 @@ export const AdvancedSettings = ({description={<><p>- Maximum time the pipeline waits to collect additional changes before- flushing a batch.+ Maximum time pipeline waits to collect additional changes before flushing a+ batch.</p><p>- Lower values reduce replication latency; higher values improve batching+ Lower values reduce replication latency, higher values improve batchingefficiency.</p></>@@ -122,7 +126,7 @@ export const AdvancedSettings = ({Number of parallel connections each table copy can use during initial sync.</p><p>- Higher values can copy large tables faster, but consume more database+ More connections speed up large table copies, but use more databaseconnections.</p></>@@ -143,47 +147,118 @@ export const AdvancedSettings = ({)}/>- {/* BigQuery-specific: Max staleness */}+ <FormField_Shadcn_+ control={form.control}+ name="invalidatedSlotBehavior"+ render={({ field }) => (+ <FormItemLayout+ label="Invalidated slot behavior"+ layout="horizontal"+ description="Behavior when the replication slot is invalidated"+ >+ <FormControl_Shadcn_>+ <Select_Shadcn_ value={field.value ?? 'error'} onValueChange={field.onChange}>+ <SelectTrigger_Shadcn_ className="capitalize">+ {field.value ?? 'error'}+ </SelectTrigger_Shadcn_>+ <SelectContent_Shadcn_>+ <SelectItem_Shadcn_ value="error" className="[&>span]:top-2.5">+ <p>Error</p>+ <p className="text-foreground-lighter">+ Blocks startup for manual recovery+ </p>+ </SelectItem_Shadcn_>+ <SelectItem_Shadcn_ value="recreate" className="[&>span]:top-2.5">+ <p>Recreate</p>+ <p className="text-foreground-lighter">+ Rebuilds the slot and restarts replication from scratch+ </p>+ </SelectItem_Shadcn_>+ </SelectContent_Shadcn_>+ </Select_Shadcn_>+ </FormControl_Shadcn_>+ </FormItemLayout>+ )}+ />+{type === 'BigQuery' && (- <FormField_Shadcn_- control={form.control}- name="maxStalenessMins"- render={({ field }) => (- <FormItemLayout- label={- <div className="flex flex-col gap-y-2">- <span>Maximum staleness</span>- <Badge className="w-min">BigQuery only</Badge>- </div>- }- layout="horizontal"- description={- <>- <p>- Maximum allowed age for BigQuery cached metadata before reading base- tables.- </p>- <p>- Lower values improve freshness; higher values can reduce query cost and- latency.- </p>- </>- }- >- <FormControl_Shadcn_>- <PrePostTab postTab="minutes">- <Input_Shadcn_- {...field}- type="number"- value={field.value ?? ''}- onChange={handleNumberChange(field)}- placeholder="Default: None (No staleness limit)"- />- </PrePostTab>- </FormControl_Shadcn_>- </FormItemLayout>- )}- />+ <>+ <FormField_Shadcn_+ control={form.control}+ name="connectionPoolSize"+ render={({ field }) => (+ <FormItemLayout+ label={+ <div className="flex flex-col gap-y-2">+ <span>Connection pool size</span>+ <Badge className="w-min">BigQuery only</Badge>+ </div>+ }+ layout="horizontal"+ description={+ <>+ <p>Size of the BigQuery Storage Write API connection pool.</p>+ <p>+ More connections allow more parallel writes, but consume more resources.+ </p>+ </>+ }+ >+ <FormControl_Shadcn_>+ <PrePostTab postTab="connections">+ <Input_Shadcn_+ {...field}+ type="number"+ value={field.value ?? ''}+ onChange={handleNumberChange(field)}+ placeholder="Default: 4"+ />+ </PrePostTab>+ </FormControl_Shadcn_>+ </FormItemLayout>+ )}+ />++ <FormField_Shadcn_+ control={form.control}+ name="maxStalenessMins"+ render={({ field }) => (+ <FormItemLayout+ label={+ <div className="flex flex-col gap-y-2">+ <span>Maximum staleness</span>+ <Badge className="w-min">BigQuery only</Badge>+ </div>+ }+ layout="horizontal"+ description={+ <>+ <p>+ Maximum allowed age for BigQuery cached metadata before reading base+ tables.+ </p>+ <p>+ Lower values improve freshness, higher values can reduce query cost and+ latency.+ </p>+ </>+ }+ >+ <FormControl_Shadcn_>+ <PrePostTab postTab="minutes">+ <Input_Shadcn_+ {...field}+ type="number"+ value={field.value ?? ''}+ onChange={handleNumberChange(field)}+ placeholder="Default: None (No staleness limit)"+ />+ </PrePostTab>+ </FormControl_Shadcn_>+ </FormItemLayout>+ )}+ />+ </>)}</AccordionContent_Shadcn_></AccordionItem_Shadcn_>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.tsindex a8949f27729..d2ff933585d 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts@@ -17,10 +17,12 @@ export const DestinationPanelFormSchema = z.object({.int().min(1, 'Max copy connections per table should be greater than 0').optional(),+ invalidatedSlotBehavior: z.enum(['error', 'recreate']).optional(),// BigQuery fieldsprojectId: z.string().optional(),datasetId: z.string().optional(),serviceAccountKey: z.string().optional(),+ connectionPoolSize: z.number().int().min(1).optional(),maxStalenessMins: z.number().nonnegative().optional(),// Analytics Bucket fields, only warehouse name and namespace are visible + editable fieldswarehouseName: z.string().optional(),diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.tsindex 77a95a7cc4d..e9801c0f202 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts@@ -1,8 +1,20 @@+import { UseMutateAsyncFunction } from '@tanstack/react-query'+import { snakeCase } from 'lodash'import z from 'zod'import { DestinationType } from '../DestinationPanel.types'import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'import { DestinationPanelFormSchema } from './DestinationForm.schema'+import {+ BigQueryDestinationConfig,+ DestinationConfig,+ IcebergDestinationConfig,+} from '@/data/replication/create-destination-pipeline-mutation'+import {+ type CreateS3AccessKeyCredentialVariables,+ type S3AccessKeyCreateData,+} from '@/data/storage/s3-access-key-create-mutation'+import { ResponseError } from '@/types'// Helper function to build destination config for validationexport const buildDestinationConfigForValidation = ({@@ -22,7 +34,8 @@ export const buildDestinationConfigForValidation = ({projectId: data.projectId ?? '',datasetId: data.datasetId ?? '',serviceAccountKey: data.serviceAccountKey ?? '',- ...(data.maxStalenessMins !== undefined ? { maxStalenessMins: data.maxStalenessMins } : {}),+ connectionPoolSize: data.connectionPoolSize,+ maxStalenessMins: data.maxStalenessMins,},}} else if (selectedType === 'Analytics Bucket') {@@ -54,3 +67,67 @@ export const buildDestinationConfigForValidation = ({throw new Error('Invalid destination type')}}++export const buildDestinationConfig = async ({+ projectRef,+ selectedType,+ data,+ warehouseName,+ createS3AccessKey,+ resolveNamespace,+}: {+ projectRef?: string+ selectedType: DestinationType+ data: z.infer<typeof DestinationPanelFormSchema>+ warehouseName?: string+ createS3AccessKey: UseMutateAsyncFunction<+ S3AccessKeyCreateData,+ ResponseError,+ CreateS3AccessKeyCredentialVariables,+ unknown+ >+ resolveNamespace: (+ data: z.infer<typeof DestinationPanelFormSchema>+ ) => Promise<string | undefined>+}) => {+ if (!projectRef) throw new Error('Project ref is required')++ let destinationConfig: DestinationConfig | undefined = undefined++ if (selectedType === 'BigQuery') {+ const bigQueryConfig: BigQueryDestinationConfig = {+ projectId: data.projectId ?? '',+ datasetId: data.datasetId ?? '',+ serviceAccountKey: data.serviceAccountKey ?? '',+ connectionPoolSize: data.connectionPoolSize,+ maxStalenessMins: data.maxStalenessMins,+ }+ destinationConfig = { bigQuery: bigQueryConfig }+ } else if (selectedType === 'Analytics Bucket') {+ let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }++ if (data.s3AccessKeyId === CREATE_NEW_KEY) {+ const newKeys = await createS3AccessKey({+ projectRef,+ description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,+ })+ s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }+ }++ // Resolve namespace (create if needed)+ const finalNamespace = await resolveNamespace(data)++ const icebergConfig: IcebergDestinationConfig = {+ projectRef: projectRef,+ warehouseName: data.warehouseName ?? '',+ namespace: finalNamespace,+ catalogToken: data.catalogToken ?? '',+ s3AccessKeyId: s3Keys.accessKey ?? '',+ s3SecretAccessKey: s3Keys.secretKey ?? '',+ s3Region: data.s3Region ?? '',+ }+ destinationConfig = { iceberg: icebergConfig }+ }++ return destinationConfig+}diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsxindex 8d99386278e..f9291d4db11 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx@@ -6,9 +6,6 @@ import { getKeys, useAPIKeysQuery } from 'data/api-keys/api-keys-query'import { useProjectSettingsV2Query } from 'data/config/project-settings-v2-query'import {BatchConfig,- BigQueryDestinationConfig,- DestinationConfig,- IcebergDestinationConfig,useCreateDestinationPipelineMutation,} from 'data/replication/create-destination-pipeline-mutation'import { useReplicationDestinationByIdQuery } from 'data/replication/destination-by-id-query'@@ -27,7 +24,6 @@ import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespacimport { useS3AccessKeyCreateMutation } from 'data/storage/s3-access-key-create-mutation'import { AnimatePresence, motion } from 'framer-motion'import { useAsyncCheckPermissions } from 'hooks/misc/useCheckPermissions'-import { snakeCase } from 'lodash'import { Loader2 } from 'lucide-react'import { useEffect, useMemo, useRef, useState } from 'react'import { useForm } from 'react-hook-form'@@ -41,9 +37,12 @@ import * as z from 'zod'import { DestinationType } from '../DestinationPanel.types'import { AdvancedSettings } from './AdvancedSettings'-import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'+import { CREATE_NEW_NAMESPACE } from './DestinationForm.constants'import { DestinationPanelFormSchema as FormSchema } from './DestinationForm.schema'-import { buildDestinationConfigForValidation } from './DestinationForm.utils'+import {+ buildDestinationConfig,+ buildDestinationConfigForValidation,+} from './DestinationForm.utils'import { DestinationNameInput } from './DestinationNameInput'import { AnalyticsBucketFields, BigQueryFields } from './DestinationPanelFields'import { NewPublicationPanel } from './NewPublicationPanel'@@ -176,10 +175,16 @@ export const DestinationForm = ({maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,maxCopyConnectionsPerTable: pipelineData?.config?.max_copy_connections_per_table ?? undefined,+ invalidatedSlotBehavior:+ (pipelineData?.config as { invalidated_slot_behavior?: 'error' | 'recreate' } | undefined)+ ?.invalidated_slot_behavior ?? undefined,// BigQuery fieldsprojectId: isBigQueryConfig ? config.big_query.project_id : '',datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',serviceAccountKey: isBigQueryConfig ? config.big_query.service_account_key : '',+ connectionPoolSize:+ (config as { big_query?: { connection_pool_size?: number } } | undefined)?.big_query+ ?.connection_pool_size ?? undefined,maxStalenessMins: isBigQueryConfig ? config.big_query.max_staleness_mins : undefined, // Default: null// Analytics Bucket fieldswarehouseName: isIcebergConfig ? config.iceberg.supabase.warehouse_name : '',@@ -305,6 +310,7 @@ export const DestinationForm = ({maxFillMs: data.maxFillMs,maxTableSyncWorkers: data.maxTableSyncWorkers,maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,+ invalidatedSlotBehavior: data.invalidatedSlotBehavior,}),])@@ -349,10 +355,6 @@ export const DestinationForm = ({return !hasCriticalFailures}- // [Joshen] I reckon this function can be refactored to be a bit more modular, it's currently pretty- // complicated with 4 different types of flows -> edit bigquery/analytics, and create bigquery/analytics- // At first glance we could try grouping as edit / create bigquery, edit / create analytics- // since the destination config seems rather similar between edit and create for the same typeconst submitPipeline = async (data: z.infer<typeof FormSchema>) => {if (!projectRef) return console.error('Project ref is required')if (!sourceId) return console.error('Source id is required')@@ -361,56 +363,31 @@ export const DestinationForm = ({}try {- if (editMode && existingDestination) {- if (!existingDestination.pipelineId) return console.error('Pipeline id is required')+ const destinationConfig = await buildDestinationConfig({+ projectRef,+ selectedType,+ warehouseName,+ data,+ createS3AccessKey,+ resolveNamespace,+ })- let destinationConfig: DestinationConfig | undefined = undefined+ if (!destinationConfig) throw new Error('Destination configuration is missing')- if (selectedType === 'BigQuery') {- const bigQueryConfig: BigQueryDestinationConfig = {- projectId: data.projectId ?? '',- datasetId: data.datasetId ?? '',- serviceAccountKey: data.serviceAccountKey ?? '',- }- if (!!data.maxStalenessMins) {- bigQueryConfig.maxStalenessMins = data.maxStalenessMins- }- destinationConfig = { bigQuery: bigQueryConfig }- } else if (selectedType === 'Analytics Bucket') {- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }-- if (data.s3AccessKeyId === CREATE_NEW_KEY) {- const newKeys = await createS3AccessKey({- projectRef,- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,- })- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }- }+ const batchConfig: BatchConfig | undefined =+ data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : undefined+ const hasBatchFields = batchConfig !== undefined- // Resolve namespace (create if needed)- const finalNamespace = await resolveNamespace(data)-- const icebergConfig: IcebergDestinationConfig = {- projectRef: projectRef,- warehouseName: data.warehouseName ?? '',- namespace: finalNamespace,- catalogToken: data.catalogToken ?? '',- s3AccessKeyId: s3Keys.accessKey ?? '',- s3SecretAccessKey: s3Keys.secretKey ?? '',- s3Region: data.s3Region ?? '',- }- destinationConfig = { iceberg: icebergConfig }- }-- const batchConfig: BatchConfig | undefined =- data.maxFillMs !== undefined- ? {- ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),- }- : undefined- const hasBatchFields = batchConfig !== undefined+ const pipelineConfig = {+ publicationName: data.publicationName,+ maxTableSyncWorkers: data.maxTableSyncWorkers,+ maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,+ invalidatedSlotBehavior: data.invalidatedSlotBehavior,+ ...(hasBatchFields ? { batch: batchConfig } : {}),+ }- if (!destinationConfig) throw new Error('Destination configuration is missing')+ if (editMode && existingDestination) {+ if (!existingDestination.pipelineId) return console.error('Pipeline id is required')await updateDestinationPipeline({destinationId: existingDestination.destinationId,@@ -418,12 +395,7 @@ export const DestinationForm = ({projectRef,destinationName: data.name,destinationConfig,- pipelineConfig: {- publicationName: data.publicationName,- maxTableSyncWorkers: data.maxTableSyncWorkers,- maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,- ...(hasBatchFields ? { batch: batchConfig } : {}),- },+ pipelineConfig,sourceId,})// Set request status only right before starting, then fire and close@@ -448,64 +420,12 @@ export const DestinationForm = ({}onClose()} else {- let destinationConfig: DestinationConfig | undefined = undefined-- if (selectedType === 'BigQuery') {- const bigQueryConfig: BigQueryDestinationConfig = {- projectId: data.projectId ?? '',- datasetId: data.datasetId ?? '',- serviceAccountKey: data.serviceAccountKey ?? '',- }- if (!!data.maxStalenessMins) {- bigQueryConfig.maxStalenessMins = data.maxStalenessMins- }- destinationConfig = { bigQuery: bigQueryConfig }- } else if (selectedType === 'Analytics Bucket') {- let s3Keys = { accessKey: data.s3AccessKeyId, secretKey: data.s3SecretAccessKey }-- if (data.s3AccessKeyId === CREATE_NEW_KEY) {- const newKeys = await createS3AccessKey({- projectRef,- description: `Autogenerated key for replication to ${snakeCase(warehouseName)}`,- })- s3Keys = { accessKey: newKeys.access_key, secretKey: newKeys.secret_key }- }-- // Resolve namespace (create if needed)- const finalNamespace = await resolveNamespace(data)-- const icebergConfig: IcebergDestinationConfig = {- projectRef: projectRef,- warehouseName: data.warehouseName ?? '',- namespace: finalNamespace,- catalogToken: data.catalogToken ?? '',- s3AccessKeyId: s3Keys.accessKey ?? '',- s3SecretAccessKey: s3Keys.secretKey ?? '',- s3Region: data.s3Region ?? '',- }- destinationConfig = { iceberg: icebergConfig }- }- const batchConfig: BatchConfig | undefined =- data.maxFillMs !== undefined- ? {- ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),- }- : undefined- const hasBatchFields = batchConfig !== undefined-- if (!destinationConfig) throw new Error('Destination configuration is missing')-const { pipeline_id: pipelineId } = await createDestinationPipeline({projectRef,destinationName: data.name,destinationConfig,+ pipelineConfig,sourceId,- pipelineConfig: {- publicationName: data.publicationName,- maxTableSyncWorkers: data.maxTableSyncWorkers,- maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,- ...(hasBatchFields ? { batch: batchConfig } : {}),- },})// Set request status only right before starting, then fire and closesetRequestStatus(pipelineId, PipelineStatusRequestStatus.StartRequested, undefined)diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex 8a0cd04291c..96e81175306 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -18,6 +18,7 @@ export type BigQueryDestinationConfig = {projectId: stringdatasetId: stringserviceAccountKey: string+ connectionPoolSize?: numbermaxStalenessMins?: number}@@ -45,6 +46,7 @@ export type CreateDestinationPipelineParams = {batch?: BatchConfigmaxTableSyncWorkers?: numbermaxCopyConnectionsPerTable?: number+ invalidatedSlotBehavior?: 'error' | 'recreate'}}@@ -53,7 +55,13 @@ async function createDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },+ pipelineConfig: {+ publicationName,+ batch,+ maxTableSyncWorkers,+ maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,+ },sourceId,}: CreateDestinationPipelineParams,signal?: AbortSignal@@ -64,16 +72,18 @@ async function createDestinationPipeline(let destination_config: components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =+ destinationConfig.bigQuerydestination_config = {big_query: {project_id: projectId,dataset_id: datasetId,service_account_key: serviceAccountKey,- ...(maxStalenessMins !== null ? { max_staleness_mins: maxStalenessMins } : {}),+ connection_pool_size: connectionPoolSize,+ max_staleness_mins: maxStalenessMins,},- }+ } as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']} else if ('iceberg' in destinationConfig) {const {projectRef: icebergProjectRef,@@ -102,35 +112,27 @@ async function createDestinationPipeline(throw new Error('Invalid destination config: must specify either bigQuery or iceberg')}+ const pipeline_config = {+ publication_name: publicationName,+ max_table_sync_workers: maxTableSyncWorkers,+ max_copy_connections_per_table: maxCopyConnectionsPerTable,+ invalidated_slot_behavior: invalidatedSlotBehavior,+ batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,+ }+const { data, error } = await post('/platform/replication/{ref}/destinations-pipelines', {params: { path: { ref: projectRef } },body: {source_id: sourceId,destination_name: destinationName,destination_config,- pipeline_config: {- publication_name: publicationName,- ...(maxTableSyncWorkers !== undefined- ? { max_table_sync_workers: maxTableSyncWorkers }- : {}),- ...(maxCopyConnectionsPerTable !== undefined- ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }- : {}),- ...(batch- ? {- batch: {- ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),- },- }- : {}),- },+ pipeline_config:+ pipeline_config as components['schemas']['CreateReplicationDestinationPipelineBody']['pipeline_config'],},signal,})- if (error) {- handleError(error)- }+ if (error) handleError(error)return data}diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex a48f475357c..d86f1287d6a 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -19,6 +19,7 @@ export type UpdateDestinationPipelineParams = {batch?: BatchConfigmaxTableSyncWorkers?: numbermaxCopyConnectionsPerTable?: number+ invalidatedSlotBehavior?: 'error' | 'recreate'}}@@ -29,7 +30,13 @@ async function updateDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },+ pipelineConfig: {+ publicationName,+ batch,+ maxTableSyncWorkers,+ maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,+ },sourceId,}: UpdateDestinationPipelineParams,signal?: AbortSignal@@ -40,15 +47,17 @@ async function updateDestinationPipeline(let destination_config: components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =+ destinationConfig.bigQuerydestination_config = {big_query: {project_id: projectId,dataset_id: datasetId,service_account_key: serviceAccountKey,- ...(maxStalenessMins !== null ? { max_staleness_mins: maxStalenessMins } : {}),+ connection_pool_size: connectionPoolSize,+ max_staleness_mins: maxStalenessMins,},- }+ } as components['schemas']['UpdateReplicationDestinationPipelineBody']['destination_config']} else if ('iceberg' in destinationConfig) {const {projectRef: icebergProjectRef,@@ -76,6 +85,14 @@ async function updateDestinationPipeline(throw new Error('Invalid destination config: must specify either bigQuery or iceberg')}+ const pipeline_config = {+ publication_name: publicationName,+ max_table_sync_workers: maxTableSyncWorkers,+ max_copy_connections_per_table: maxCopyConnectionsPerTable,+ invalidated_slot_behavior: invalidatedSlotBehavior,+ batch: batch ? { max_fill_ms: batch.maxFillMs } : undefined,+ }+const { data, error } = await post('/platform/replication/{ref}/destinations-pipelines/{destination_id}/{pipeline_id}',{@@ -84,26 +101,13 @@ async function updateDestinationPipeline(destination_config,source_id: sourceId,destination_name: destinationName,- pipeline_config: {- publication_name: publicationName,- ...(maxTableSyncWorkers !== undefined- ? { max_table_sync_workers: maxTableSyncWorkers }- : {}),- ...(maxCopyConnectionsPerTable !== undefined- ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }- : {}),- ...(batch- ? {- batch: {- ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),- },- }- : {}),- },+ pipeline_config:+ pipeline_config as components['schemas']['UpdateReplicationDestinationPipelineBody']['pipeline_config'],},signal,})+if (error) handleError(error)return data}diff --git a/apps/studio/data/replication/validate-destination-mutation.ts b/apps/studio/data/replication/validate-destination-mutation.tsindex f9468b2ae3a..37c59fafde9 100644--- a/apps/studio/data/replication/validate-destination-mutation.ts+++ b/apps/studio/data/replication/validate-destination-mutation.ts@@ -1,8 +1,8 @@import { useMutation } from '@tanstack/react-query'-import type { components } from 'api-types'import { handleError, post } from 'data/fetchers'import type { ResponseError, UseCustomMutationOptions } from 'types'+import { DestinationConfig } from './create-destination-pipeline-mutation'type ValidateDestinationParams = {@@ -23,16 +23,18 @@ async function validateDestination(let config: components['schemas']['ValidateReplicationDestinationBody']['config']if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, maxStalenessMins } = destinationConfig.bigQuery+ const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =+ destinationConfig.bigQueryconfig = {big_query: {project_id: projectId,dataset_id: datasetId,service_account_key: serviceAccountKey,- ...(maxStalenessMins !== undefined ? { max_staleness_mins: maxStalenessMins } : {}),+ connection_pool_size: connectionPoolSize,+ max_staleness_mins: maxStalenessMins,},- }+ } as components['schemas']['ValidateReplicationDestinationBody']['config']} else if ('iceberg' in destinationConfig) {const {projectRef: icebergProjectRef,diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.tsindex af822a5bd07..a248559aecf 100644--- a/apps/studio/data/replication/validate-pipeline-mutation.ts+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts@@ -1,6 +1,5 @@import { useMutation } from '@tanstack/react-query'import { components } from 'api-types'-import { handleError, post } from 'data/fetchers'import type { ResponseError, UseCustomMutationOptions } from 'types'@@ -11,6 +10,7 @@ type ValidatePipelineParams = {maxFillMs?: numbermaxTableSyncWorkers?: numbermaxCopyConnectionsPerTable?: number+ invalidatedSlotBehavior?: 'error' | 'recreate'}type ValidatePipelineResponse = components['schemas']['ValidatePipelineResponse']@@ -22,6 +22,7 @@ async function validatePipeline(maxFillMs,maxTableSyncWorkers,maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,}: ValidatePipelineParams,signal?: AbortSignal): Promise<ValidatePipelineResponse> {@@ -30,16 +31,19 @@ async function validatePipeline(const batchConfig = maxFillMs !== undefined ? { max_fill_ms: maxFillMs } : undefined+ const config = {+ publication_name: publicationName,+ max_table_sync_workers: maxTableSyncWorkers,+ max_copy_connections_per_table: maxCopyConnectionsPerTable,+ invalidated_slot_behavior: invalidatedSlotBehavior,+ batch: batchConfig,+ }+const { data, error } = await post('/platform/replication/{ref}/pipelines/validate', {params: { path: { ref: projectRef } },body: {source_id: sourceId,- config: {- publication_name: publicationName,- max_table_sync_workers: maxTableSyncWorkers,- max_copy_connections_per_table: maxCopyConnectionsPerTable,- batch: batchConfig,- },+ config: config as components['schemas']['ValidateReplicationPipelineBody']['config'],},signal,})diff --git a/apps/studio/data/storage/s3-access-key-create-mutation.ts b/apps/studio/data/storage/s3-access-key-create-mutation.tsindex 362d47ef557..2536773648e 100644--- a/apps/studio/data/storage/s3-access-key-create-mutation.ts+++ b/apps/studio/data/storage/s3-access-key-create-mutation.ts@@ -1,11 +1,11 @@import { useMutation, useQueryClient } from '@tanstack/react-query'-import { toast } from 'sonner'-import { handleError, post } from 'data/fetchers'+import { toast } from 'sonner'import type { ResponseError, UseCustomMutationOptions } from 'types'+import { storageCredentialsKeys } from './s3-access-key-keys'-type CreateS3AccessKeyCredentialVariables = {+export type CreateS3AccessKeyCredentialVariables = {description: stringprojectRef?: string}@@ -25,7 +25,7 @@ const createS3AccessKeyCredential = async ({return data}-type S3AccessKeyCreateData = Awaited<ReturnType<typeof createS3AccessKeyCredential>>+export type S3AccessKeyCreateData = Awaited<ReturnType<typeof createS3AccessKeyCredential>>export function useS3AccessKeyCreateMutation({onSuccess,diff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.tsindex 9d917f93f37..2915887577f 100644--- a/packages/api-types/types/platform.d.ts+++ b/packages/api-types/types/platform.d.ts@@ -5296,16 +5296,16 @@ export interface components {config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -5372,16 +5372,16 @@ export interface components {destination_config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -5452,6 +5452,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -5479,6 +5485,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -8461,16 +8473,16 @@ export interface components {config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -8549,16 +8561,16 @@ export interface components {config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -8761,6 +8773,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -8820,6 +8838,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -10198,16 +10222,16 @@ export interface components {config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -10274,16 +10298,16 @@ export interface components {destination_config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -10354,6 +10378,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -10381,6 +10411,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */@@ -10712,16 +10748,16 @@ export interface components {config:| {big_query: {+ /**+ * @description Number of concurrent BigQuery Storage Write API connections.+ * @example 8+ */+ connection_pool_size?: number/*** @description BigQuery dataset id* @example analytics*/dataset_id: string- /**- * @description Maximum number of concurrent write streams- * @example 8- */- max_concurrent_streams?: number/*** @description Maximum data staleness in minutes* @example 5@@ -10789,6 +10825,12 @@ export interface components {*/max_fill_ms?: number}+ /**+ * @description Behavior when the replication slot is invalidated+ * @example error+ * @enum {string}+ */+ invalidated_slot_behavior?: 'error' | 'recreate'/** @description Maximum number of copy connections per table */max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */