The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
refactor(studio): centralise replication payload builders (#49842)
Danny Whitea351a36e9b35
diff --git a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsxindex 5c9b209f0fe..1c9948cb6e0 100644--- a/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx+++ b/apps/studio/components/interfaces/Database/Replication/BatchRestartDialog.tsx@@ -14,9 +14,10 @@ import {import { PipelineStatusName } from './Replication.constants'import { RestartCostEstimate } from './RestartCostEstimate'-import { getTableCopyTargets, type TableSyncCopyConfig } from './TableSyncCopy.utils'+import { getTableCopyTargets } from './TableSyncCopy.utils'import { ReplicationPipelineTableStatus } from '@/data/replication/pipeline-replication-status-query'import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation'+import type { TableSyncCopyConfig } from '@/data/replication/types'interface BatchRestartDialogProps {open: booleandiff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.tsindex d36e4287aab..3ed2cb9a00b 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.utils.ts@@ -24,7 +24,10 @@ import {} from './DuckLake/DuckLake.constants'import { type DucklakeApiConfig } from './DuckLake/DuckLake.utils'import { type SnowflakeApiConfig } from './Snowflake/Snowflake.utils'-import {+import { type ReplicationDestinationByIdData } from '@/data/replication/destination-by-id-query'+import { type ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'+import { type ReplicationPublication } from '@/data/replication/publications-query'+import type {BatchConfig,BigQueryDestinationConfig,ClickHouseDestinationConfig,@@ -35,10 +38,7 @@ import {IcebergDestinationConfig,SnowflakeDestinationConfig,TableSyncCopyConfig,-} from '@/data/replication/create-destination-pipeline-mutation'-import { type ReplicationDestinationByIdData } from '@/data/replication/destination-by-id-query'-import { type ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'-import { type ReplicationPublication } from '@/data/replication/publications-query'+} from '@/data/replication/types'import { type ValidationFailure } from '@/data/replication/validate-destination-mutation'import {type CreateS3AccessKeyCredentialVariables,diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsxindex b472551c66f..199b1d6ee85 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/PipelineCostDialog.tsx@@ -25,11 +25,11 @@ import {getTableCopyTargets,summarizeTableCopyEstimate,type ReplicationTableIdentity,- type TableSyncCopyConfig,} from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'import { InlineLink } from '@/components/ui/InlineLink'import { useReplicationCostEstimateQuery } from '@/data/replication/cost-estimate-query'import { useReplicationSourceId } from '@/data/replication/sources-query'+import type { TableSyncCopyConfig } from '@/data/replication/types'import { useLatest } from '@/hooks/misc/useLatest'import { DOCS_URL } from '@/lib/constants'import { formatBytes, formatCurrency } from '@/lib/helpers'diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.tsindex a89977494cc..5fca795b9a5 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/useDestinationForm.ts@@ -12,13 +12,11 @@ import {buildDestinationConfigForValidation,buildTableSyncCopyConfig,} from './DestinationForm.utils'-import {- useCreateDestinationPipelineMutation,- type BatchConfig,-} from '@/data/replication/create-destination-pipeline-mutation'+import { useCreateDestinationPipelineMutation } from '@/data/replication/create-destination-pipeline-mutation'import type { ReplicationPipelineByIdData } from '@/data/replication/pipeline-by-id-query'import { useReplicationSourcesQuery } from '@/data/replication/sources-query'import { useStartPipelineMutation } from '@/data/replication/start-pipeline-mutation'+import { type BatchConfig } from '@/data/replication/types'import { useUpdateDestinationPipelineMutation } from '@/data/replication/update-destination-pipeline-mutation'import {useValidateDestinationMutation,diff --git a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsxindex 59ce94d5f69..771e6eb034c 100644--- a/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx+++ b/apps/studio/components/interfaces/Database/Replication/RestartTableDialog.tsx@@ -13,12 +13,9 @@ import {import { PipelineStatusName } from './Replication.constants'import { RestartCostEstimate } from './RestartCostEstimate'-import {- shouldCopyTable,- type ReplicationTableIdentity,- type TableSyncCopyConfig,-} from './TableSyncCopy.utils'+import { shouldCopyTable, type ReplicationTableIdentity } from './TableSyncCopy.utils'import { useRollbackTablesMutation } from '@/data/replication/rollback-tables-mutation'+import type { TableSyncCopyConfig } from '@/data/replication/types'interface RestartTableDialogProps {open: booleandiff --git a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.tsindex 023afa18441..83f0748fd1a 100644--- a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts+++ b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.test.ts@@ -4,8 +4,8 @@ import {getTableCopyTargets,shouldCopyTable,summarizeTableCopyEstimate,- type TableSyncCopyConfig,} from './TableSyncCopy.utils'+import type { TableSyncCopyConfig } from '@/data/replication/types'const tables = [{ id: 101, schema: 'public', name: 'orders' },diff --git a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.tsindex 71fe5b9f0e1..6e004d88238 100644--- a/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts+++ b/apps/studio/components/interfaces/Database/Replication/TableSyncCopy.utils.ts@@ -1,8 +1,4 @@-export type TableSyncCopyConfig =- | { type: 'include_all_tables' }- | { type: 'skip_all_tables' }- | { type: 'include_tables'; table_ids: number[] }- | { type: 'skip_tables'; table_ids: number[] }+import type { TableSyncCopyConfig } from '@/data/replication/types'export type ReplicationTableIdentity = {id: numberdiff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.test.tsindex 5264a361700..d112d7cc56a 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.test.ts@@ -1,9 +1,14 @@import { describe, expect, it } from 'vitest'import {+ buildBigQueryApiConfig,buildDucklakeApiConfig,- buildPipelineApiConfig,} from './create-destination-pipeline-mutation'+import {+ buildBigQueryUpdateApiConfig,+ buildDucklakeUpdateApiConfig,+} from './update-destination-pipeline-mutation'+import { buildPipelineApiConfig } from './utils'describe('buildPipelineApiConfig', () => {it('maps selective initial-copy configuration to the ETL API shape', () => {@@ -27,6 +32,33 @@ describe('buildPipelineApiConfig', () => {})})+describe('buildBigQueryApiConfig', () => {+ const baseConfig = {+ projectId: 'my-project',+ datasetId: 'analytics',+ serviceAccountKey: '{}',+ }++ it('maps the destination config to the API shape', () => {+ expect(buildBigQueryApiConfig(baseConfig)).toEqual({+ big_query: {+ project_id: 'my-project',+ dataset_id: 'analytics',+ service_account_key: '{}',+ connection_pool_size: undefined,+ max_staleness_mins: undefined,+ },+ })+ })++ it('omits blank service_account_key on update, but not on create', () => {+ const config = { ...baseConfig, serviceAccountKey: '' }++ expect(buildBigQueryApiConfig(config).big_query.service_account_key).toBe('')+ expect(buildBigQueryUpdateApiConfig(config).big_query.service_account_key).toBeUndefined()+ })+})+describe('buildDucklakeApiConfig', () => {it('maps a "Use Supabase" config with catalog-level pool size + metadata schema', () => {expect(@@ -106,21 +138,18 @@ describe('buildDucklakeApiConfig', () => {it('omits blank custom secret fields when requested', () => {expect(- buildDucklakeApiConfig(- {- catalogUrl: ' ',- dataPath: 's3://bucket/path',- poolSize: 4,- s3AccessKeyId: '',- s3SecretAccessKey: '\n',- s3Region: 'eu-west-1',- s3Endpoint: 's3.example.com',- s3UrlStyle: 'path',- s3UseSsl: true,- metadataSchema: 'ducklake',- },- { omitBlankSecrets: true }- )+ buildDucklakeUpdateApiConfig({+ catalogUrl: ' ',+ dataPath: 's3://bucket/path',+ poolSize: 4,+ s3AccessKeyId: '',+ s3SecretAccessKey: '\n',+ s3Region: 'eu-west-1',+ s3Endpoint: 's3.example.com',+ s3UrlStyle: 'path',+ s3UseSsl: true,+ metadataSchema: 'ducklake',+ })).toEqual({ducklake: {catalog_url: undefined,diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex 4ada0784bd8..b583ab6fd77 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -2,102 +2,54 @@ import { useMutation, useQueryClient } from '@tanstack/react-query'import type { components } from 'api-types'import { toast } from 'sonner'-import { optionalSecret } from './destination-secret-utils'import { replicationKeys } from './keys'-import type { TableSyncCopyConfig } from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'+import type {+ BigQueryDestinationConfig,+ DestinationConfig,+ DucklakeDestinationConfig,+ PipelineConfig,+} from './types'+import { buildPipelineApiConfig, isDucklakeSupabaseConfig } from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'-export type { TableSyncCopyConfig } from '@/components/interfaces/Database/Replication/TableSyncCopy.utils'+type CreateDestinationPipelineBody =+ components['schemas']['CreateReplicationDestinationPipelineBody']+type CreateDestinationApiConfig = CreateDestinationPipelineBody['destination_config']-export type DestinationConfig =- | { bigQuery: BigQueryDestinationConfig }- | { iceberg: IcebergDestinationConfig }- | { ducklake: DucklakeDestinationConfig }- | { snowflake: SnowflakeDestinationConfig }- | { clickHouse: ClickHouseDestinationConfig }+type CreateBigQueryApiConfig = Extract<CreateDestinationApiConfig, { big_query: unknown }>+type CreateDucklakeApiConfig = Extract<CreateDestinationApiConfig, { ducklake: unknown }>-export type BigQueryDestinationConfig = {- projectId: string- datasetId: string- serviceAccountKey: string- connectionPoolSize?: number- maxStalenessMins?: number-}--export type IcebergDestinationConfig = {- projectRef: string- warehouseName: string- namespace?: string- catalogToken: string- s3AccessKeyId: string- s3SecretAccessKey: string- s3Region: string-}--// "Custom parameters" DuckLake: caller provides the PostgreSQL catalog URL and the-// S3-compatible storage credentials directly.-export type DucklakeManualDestinationConfig = {- catalogUrl: string- dataPath: string- poolSize?: number- s3AccessKeyId: string- s3SecretAccessKey: string- s3Region: string- s3Endpoint: string- s3UrlStyle?: 'path' | 'vhost'- s3UseSsl?: boolean- metadataSchema?: string-}--// "Use Supabase" DuckLake: caller provides Supabase project refs and a bucket; the platform-// API resolves these into a catalog URL + provisioned S3 credentials before persisting.-export type DucklakeSupabaseDestinationConfig = {- catalogProjectRef: string- storageProjectRef: string- bucket: string- path?: string- poolSize?: number- metadataSchema?: string-}--export type DucklakeDestinationConfig =- | DucklakeManualDestinationConfig- | DucklakeSupabaseDestinationConfig--function isDucklakeSupabaseConfig(- config: DucklakeDestinationConfig-): config is DucklakeSupabaseDestinationConfig {- return 'catalogProjectRef' in config-}--const maybeOmitBlankSecret = (value: string | undefined, omitBlankSecrets: boolean) => {- if (omitBlankSecrets) return optionalSecret(value)-- return value+// Maps the studio-side BigQuery config to the snake_case `{ big_query: ... }` payload accepted+// by the platform API. Shared by the create and validate mutations.+export function buildBigQueryApiConfig(config: BigQueryDestinationConfig): CreateBigQueryApiConfig {+ return {+ big_query: {+ project_id: config.projectId,+ dataset_id: config.datasetId,+ service_account_key: config.serviceAccountKey,+ connection_pool_size: config.connectionPoolSize,+ max_staleness_mins: config.maxStalenessMins,+ },+ }}// Maps the studio-side DuckLake config to the snake_case `{ ducklake: ... }` payload accepted// by the platform API. Shared by the create / update / validate mutations.-export function buildDucklakeApiConfig(- config: DucklakeDestinationConfig,- options: { omitBlankSecrets?: boolean } = {}-) {- const omitBlankSecrets = options.omitBlankSecrets ?? false-+export function buildDucklakeApiConfig(config: DucklakeDestinationConfig): CreateDucklakeApiConfig {if (isDucklakeSupabaseConfig(config)) {return {ducklake: {// pool_size / metadata_schema live on the catalog so they apply to the selected// Supabase Postgres catalog (the API resolves catalog-level values over top-level).catalog: {- type: 'supabase_project' as const,+ type: 'supabase_project',project_ref: config.catalogProjectRef,pool_size: config.poolSize,metadata_schema: config.metadataSchema,},storage: {- type: 'supabase_storage' as const,+ type: 'supabase_storage',project_ref: config.storageProjectRef,bucket: config.bucket,...(config.path ? { path: config.path } : {}),@@ -108,11 +60,11 @@ export function buildDucklakeApiConfig(return {ducklake: {- catalog_url: maybeOmitBlankSecret(config.catalogUrl, omitBlankSecrets),+ catalog_url: config.catalogUrl,data_path: config.dataPath,pool_size: config.poolSize,- s3_access_key_id: maybeOmitBlankSecret(config.s3AccessKeyId, omitBlankSecrets),- s3_secret_access_key: maybeOmitBlankSecret(config.s3SecretAccessKey, omitBlankSecrets),+ s3_access_key_id: config.s3AccessKeyId,+ s3_secret_access_key: config.s3SecretAccessKey,s3_region: config.s3Region,s3_endpoint: config.s3Endpoint,s3_url_style: config.s3UrlStyle,@@ -122,100 +74,16 @@ export function buildDucklakeApiConfig(}}-export type SnowflakeDestinationConfig = {- accountId: string- user: string- privateKey: string- privateKeyPassphrase?: string- database: string- schema: string- role?: string-}--export type ClickHouseDestinationConfig = {- url: string- user: string- password?: string- database: string- engine?: 'merge_tree' | 'replacing_merge_tree'-}--export type BatchConfig = {- maxFillMs?: number- maxBytes?: number- memoryBudgetRatio?: number-}--export type PipelineConfig = {- publicationName: string- batch?: BatchConfig- maxTableSyncWorkers?: number- maxCopyConnectionsPerTable?: number- invalidatedSlotBehavior?: 'error' | 'recreate'- tableSyncCopy: TableSyncCopyConfig-}--export const buildPipelineApiConfig = ({- publicationName,- batch,- maxTableSyncWorkers,- maxCopyConnectionsPerTable,- invalidatedSlotBehavior,- tableSyncCopy,-}: PipelineConfig) => ({- publication_name: publicationName,- max_table_sync_workers: maxTableSyncWorkers,- max_copy_connections_per_table: maxCopyConnectionsPerTable,- invalidated_slot_behavior: invalidatedSlotBehavior,- table_sync_copy: tableSyncCopy,- batch: batch- ? {- max_fill_ms: batch.maxFillMs,- max_bytes: batch.maxBytes,- memory_budget_ratio: batch.memoryBudgetRatio,- }- : undefined,-})--export type CreateDestinationPipelineParams = {- projectRef: string- destinationName: string+export const buildCreateDestinationApiConfig = (destinationConfig: DestinationConfig- sourceId: number- pipelineConfig: PipelineConfig-}--async function createDestinationPipeline(- {- projectRef,- destinationName: destinationName,- destinationConfig,- pipelineConfig,- sourceId,- }: CreateDestinationPipelineParams,- signal?: AbortSignal-) {- if (!projectRef) throw new Error('projectRef is required')-- // Build destination_config based on the type- let destination_config: components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']-+): CreateDestinationApiConfig => {if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =- destinationConfig.bigQuery+ return buildBigQueryApiConfig(destinationConfig.bigQuery)+ }- destination_config = {- big_query: {- project_id: projectId,- dataset_id: datasetId,- service_account_key: serviceAccountKey,- connection_pool_size: connectionPoolSize,- max_staleness_mins: maxStalenessMins,- },- } as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']- } else if ('iceberg' in destinationConfig) {+ if ('iceberg' in destinationConfig) {const {- projectRef: icebergProjectRef,+ projectRef,namespace,warehouseName,catalogToken,@@ -224,11 +92,11 @@ async function createDestinationPipeline(s3Region,} = destinationConfig.iceberg- destination_config = {+ return {iceberg: {supabase: {namespace,- project_ref: icebergProjectRef,+ project_ref: projectRef,warehouse_name: warehouseName,catalog_token: catalogToken,s3_access_key_id: s3AccessKeyId,@@ -237,15 +105,17 @@ async function createDestinationPipeline(},},}- } else if ('ducklake' in destinationConfig) {- destination_config = buildDucklakeApiConfig(- destinationConfig.ducklake- ) as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']- } else if ('snowflake' in destinationConfig) {+ }++ if ('ducklake' in destinationConfig) {+ return buildDucklakeApiConfig(destinationConfig.ducklake)+ }++ if ('snowflake' in destinationConfig) {const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =destinationConfig.snowflake- destination_config = {+ return {snowflake: {account_id: accountId,user,@@ -255,25 +125,42 @@ async function createDestinationPipeline(schema,role,},- } as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']- } else if ('clickHouse' in destinationConfig) {+ }+ }++ if ('clickHouse' in destinationConfig) {const { url, user, password, database, engine } = destinationConfig.clickHouse- destination_config = {- clickhouse: {- url,- user,- password,- database,- engine,- },- } as components['schemas']['CreateReplicationDestinationPipelineBody']['destination_config']- } else {- throw new Error(- 'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'- )+ return { clickhouse: { url, user, password, database, engine } }}+ throw new Error(+ 'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'+ )+}++export type CreateDestinationPipelineParams = {+ projectRef: string+ destinationName: string+ destinationConfig: DestinationConfig+ sourceId: number+ pipelineConfig: PipelineConfig+}++async function createDestinationPipeline(+ {+ projectRef,+ destinationName: destinationName,+ destinationConfig,+ pipelineConfig,+ sourceId,+ }: CreateDestinationPipelineParams,+ signal?: AbortSignal+) {+ if (!projectRef) throw new Error('projectRef is required')++ const destination_config = buildCreateDestinationApiConfig(destinationConfig)+const pipeline_config = buildPipelineApiConfig(pipelineConfig)const { data, error } = await post('/platform/replication/{ref}/destinations-pipelines', {@@ -282,8 +169,7 @@ async function createDestinationPipeline(source_id: sourceId,destination_name: destinationName,destination_config,- pipeline_config:- pipeline_config as components['schemas']['CreateReplicationDestinationPipelineBody']['pipeline_config'],+ pipeline_config,},signal,})diff --git a/apps/studio/data/replication/types.ts b/apps/studio/data/replication/types.tsnew file mode 100644index 00000000000..7b6d875de52--- /dev/null+++ b/apps/studio/data/replication/types.ts@@ -0,0 +1,95 @@+import { components } from 'api-types'++type CreateDestinationPipelineBody =+ components['schemas']['CreateReplicationDestinationPipelineBody']+export type CreatePipelineApiConfig = CreateDestinationPipelineBody['pipeline_config']++export type BatchConfig = {+ maxFillMs?: number+ maxBytes?: number+ memoryBudgetRatio?: number+}++export type TableSyncCopyConfig = NonNullable<CreatePipelineApiConfig['table_sync_copy']>++export type PipelineConfig = {+ publicationName: string+ batch?: BatchConfig+ maxTableSyncWorkers?: number+ maxCopyConnectionsPerTable?: number+ invalidatedSlotBehavior?: 'error' | 'recreate'+ tableSyncCopy: TableSyncCopyConfig+}++export type DestinationConfig =+ | { bigQuery: BigQueryDestinationConfig }+ | { iceberg: IcebergDestinationConfig }+ | { ducklake: DucklakeDestinationConfig }+ | { snowflake: SnowflakeDestinationConfig }+ | { clickHouse: ClickHouseDestinationConfig }++// "Custom parameters" DuckLake: caller provides the PostgreSQL catalog URL and the+// S3-compatible storage credentials directly.+export type DucklakeManualDestinationConfig = {+ catalogUrl: string+ dataPath: string+ poolSize?: number+ s3AccessKeyId: string+ s3SecretAccessKey: string+ s3Region: string+ s3Endpoint: string+ s3UrlStyle?: 'path' | 'vhost'+ s3UseSsl?: boolean+ metadataSchema?: string+}++// "Use Supabase" DuckLake: caller provides Supabase project refs and a bucket; the platform+// API resolves these into a catalog URL + provisioned S3 credentials before persisting.+export type DucklakeSupabaseDestinationConfig = {+ catalogProjectRef: string+ storageProjectRef: string+ bucket: string+ path?: string+ poolSize?: number+ metadataSchema?: string+}++export type DucklakeDestinationConfig =+ | DucklakeManualDestinationConfig+ | DucklakeSupabaseDestinationConfig++export type BigQueryDestinationConfig = {+ projectId: string+ datasetId: string+ serviceAccountKey: string+ connectionPoolSize?: number+ maxStalenessMins?: number+}++export type IcebergDestinationConfig = {+ projectRef: string+ warehouseName: string+ namespace?: string+ catalogToken: string+ s3AccessKeyId: string+ s3SecretAccessKey: string+ s3Region: string+}++export type SnowflakeDestinationConfig = {+ accountId: string+ user: string+ privateKey: string+ privateKeyPassphrase?: string+ database: string+ schema: string+ role?: string+}++export type ClickHouseDestinationConfig = {+ url: string+ user: string+ password?: string+ database: string+ engine?: 'merge_tree' | 'replacing_merge_tree'+}diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex 8356324f701..033a315584f 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -1,65 +1,88 @@import { useMutation, useQueryClient } from '@tanstack/react-query'-import type { components } from 'api-types'+import { components } from 'api-types'import { toast } from 'sonner'-import {- buildDucklakeApiConfig,- buildPipelineApiConfig,- DestinationConfig,- PipelineConfig,-} from './create-destination-pipeline-mutation'import { optionalSecret } from './destination-secret-utils'import { replicationKeys } from './keys'+import type {+ BigQueryDestinationConfig,+ DestinationConfig,+ DucklakeDestinationConfig,+ PipelineConfig,+} from './types'+import { buildPipelineApiConfig, isDucklakeSupabaseConfig } from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'-export type UpdateDestinationPipelineParams = {- destinationId: number- pipelineId: number- projectRef: string- destinationName: string- destinationConfig: DestinationConfig- sourceId: number- pipelineConfig: PipelineConfig-}-type UpdateDestinationPipelineBody =components['schemas']['UpdateReplicationDestinationPipelineBody']-type UpdateDestinationConfig = UpdateDestinationPipelineBody['destination_config']-type UpdatePipelineConfig = UpdateDestinationPipelineBody['pipeline_config']+type UpdateDestinationApiConfig = UpdateDestinationPipelineBody['destination_config']-async function updateDestinationPipeline(- {- destinationId: destinationId,- pipelineId,- projectRef,- destinationName: destinationName,- destinationConfig,- pipelineConfig,- sourceId,- }: UpdateDestinationPipelineParams,- signal?: AbortSignal-) {- if (!projectRef) throw new Error('projectRef is required')+type UpdateBigQueryApiConfig = Extract<UpdateDestinationApiConfig, { big_query: unknown }>+type UpdateDucklakeApiConfig = Extract<UpdateDestinationApiConfig, { ducklake: unknown }>- // Build destination_config based on the type- let destination_config: UpdateDestinationConfig+export function buildBigQueryUpdateApiConfig(+ config: BigQueryDestinationConfig+): UpdateBigQueryApiConfig {+ return {+ big_query: {+ project_id: config.projectId,+ dataset_id: config.datasetId,+ service_account_key: optionalSecret(config.serviceAccountKey),+ connection_pool_size: config.connectionPoolSize,+ max_staleness_mins: config.maxStalenessMins,+ },+ }+}- if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =- destinationConfig.bigQuery- destination_config = {- big_query: {- project_id: projectId,- dataset_id: datasetId,- service_account_key: optionalSecret(serviceAccountKey),- connection_pool_size: connectionPoolSize,- max_staleness_mins: maxStalenessMins,+export function buildDucklakeUpdateApiConfig(+ config: DucklakeDestinationConfig+): UpdateDucklakeApiConfig {+ if (isDucklakeSupabaseConfig(config)) {+ return {+ ducklake: {+ catalog: {+ type: 'supabase_project',+ project_ref: config.catalogProjectRef,+ pool_size: config.poolSize,+ metadata_schema: config.metadataSchema,+ },+ storage: {+ type: 'supabase_storage',+ project_ref: config.storageProjectRef,+ bucket: config.bucket,+ ...(config.path ? { path: config.path } : {}),+ },},- } as UpdateDestinationConfig- } else if ('iceberg' in destinationConfig) {+ }+ }++ return {+ ducklake: {+ catalog_url: optionalSecret(config.catalogUrl),+ data_path: config.dataPath,+ pool_size: config.poolSize,+ s3_access_key_id: optionalSecret(config.s3AccessKeyId),+ s3_secret_access_key: optionalSecret(config.s3SecretAccessKey),+ s3_region: config.s3Region,+ s3_endpoint: config.s3Endpoint,+ s3_url_style: config.s3UrlStyle,+ s3_use_ssl: config.s3UseSsl,+ metadata_schema: config.metadataSchema,+ },+ }+}++export const buildUpdateDestinationApiConfig = (+ destinationConfig: DestinationConfig+): UpdateDestinationApiConfig => {+ if ('bigQuery' in destinationConfig) {+ return buildBigQueryUpdateApiConfig(destinationConfig.bigQuery)+ }++ if ('iceberg' in destinationConfig) {const {- projectRef: icebergProjectRef,+ projectRef,warehouseName,namespace,catalogToken,@@ -67,27 +90,31 @@ async function updateDestinationPipeline(s3SecretAccessKey,s3Region,} = destinationConfig.iceberg- destination_config = {++ return {iceberg: {supabase: {- project_ref: icebergProjectRef,+ project_ref: projectRef,warehouse_name: warehouseName,- namespace: namespace,+ namespace,catalog_token: optionalSecret(catalogToken),s3_access_key_id: optionalSecret(s3AccessKeyId),s3_secret_access_key: optionalSecret(s3SecretAccessKey),s3_region: s3Region,},},- } as UpdateDestinationConfig- } else if ('ducklake' in destinationConfig) {- destination_config = buildDucklakeApiConfig(destinationConfig.ducklake, {- omitBlankSecrets: true,- }) as UpdateDestinationConfig- } else if ('snowflake' in destinationConfig) {+ }+ }++ if ('ducklake' in destinationConfig) {+ return buildDucklakeUpdateApiConfig(destinationConfig.ducklake)+ }++ if ('snowflake' in destinationConfig) {const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =destinationConfig.snowflake- destination_config = {++ return {snowflake: {account_id: accountId,user,@@ -97,10 +124,13 @@ async function updateDestinationPipeline(schema,role,},- } as UpdateDestinationConfig- } else if ('clickHouse' in destinationConfig) {+ }+ }++ if ('clickHouse' in destinationConfig) {const { url, user, password, database, engine } = destinationConfig.clickHouse- destination_config = {++ return {clickhouse: {url,user,@@ -108,13 +138,40 @@ async function updateDestinationPipeline(database,engine,},- } as UpdateDestinationConfig- } else {- throw new Error(- 'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'- )+ }}+ throw new Error(+ 'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'+ )+}++export type UpdateDestinationPipelineParams = {+ destinationId: number+ pipelineId: number+ projectRef: string+ destinationName: string+ destinationConfig: DestinationConfig+ sourceId: number+ pipelineConfig: PipelineConfig+}++async function updateDestinationPipeline(+ {+ destinationId: destinationId,+ pipelineId,+ projectRef,+ destinationName: destinationName,+ destinationConfig,+ pipelineConfig,+ sourceId,+ }: UpdateDestinationPipelineParams,+ signal?: AbortSignal+) {+ if (!projectRef) throw new Error('projectRef is required')++ const destination_config = buildUpdateDestinationApiConfig(destinationConfig)+const pipeline_config = buildPipelineApiConfig(pipelineConfig)const { data, error } = await post(@@ -125,7 +182,7 @@ async function updateDestinationPipeline(destination_config,source_id: sourceId,destination_name: destinationName,- pipeline_config: pipeline_config as UpdatePipelineConfig,+ pipeline_config,},signal,}diff --git a/apps/studio/data/replication/utils.ts b/apps/studio/data/replication/utils.tsindex 31289a682e7..6ad7c043055 100644--- a/apps/studio/data/replication/utils.ts+++ b/apps/studio/data/replication/utils.ts@@ -1,3 +1,9 @@+import type {+ CreatePipelineApiConfig,+ DucklakeDestinationConfig,+ DucklakeSupabaseDestinationConfig,+ PipelineConfig,+} from './types'import { MAX_RETRY_FAILURE_COUNT } from '@/data/query-client'import { ResponseError } from '@/types'@@ -36,3 +42,31 @@ export const checkReplicationFeatureFlagRetry = (return false}++export function isDucklakeSupabaseConfig(+ config: DucklakeDestinationConfig+): config is DucklakeSupabaseDestinationConfig {+ return 'catalogProjectRef' in config+}++export const buildPipelineApiConfig = ({+ publicationName,+ batch,+ maxTableSyncWorkers,+ maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,+ tableSyncCopy,+}: PipelineConfig): CreatePipelineApiConfig => ({+ publication_name: publicationName,+ max_table_sync_workers: maxTableSyncWorkers,+ max_copy_connections_per_table: maxCopyConnectionsPerTable,+ invalidated_slot_behavior: invalidatedSlotBehavior,+ table_sync_copy: tableSyncCopy,+ batch: batch+ ? {+ max_fill_ms: batch.maxFillMs,+ max_bytes: batch.maxBytes,+ memory_budget_ratio: batch.memoryBudgetRatio,+ }+ : undefined,+})diff --git a/apps/studio/data/replication/validate-destination-mutation.ts b/apps/studio/data/replication/validate-destination-mutation.tsindex 3e263ac79b3..0d1fc23d5c9 100644--- a/apps/studio/data/replication/validate-destination-mutation.ts+++ b/apps/studio/data/replication/validate-destination-mutation.ts@@ -1,11 +1,9 @@import { useMutation } from '@tanstack/react-query'import type { components } from 'api-types'-import {- buildDucklakeApiConfig,- DestinationConfig,- TableSyncCopyConfig,-} from './create-destination-pipeline-mutation'+import { buildCreateDestinationApiConfig } from './create-destination-pipeline-mutation'+import type { DestinationConfig, TableSyncCopyConfig } from './types'+import { buildPipelineApiConfig } from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'@@ -40,108 +38,28 @@ async function validateDestination(): Promise<ValidateDestinationResponse> {if (!projectRef) throw new Error('projectRef is required')- // Build destination_config based on the type- let config: components['schemas']['ValidateReplicationDestinationBody']['config']-- if ('bigQuery' in destinationConfig) {- const { projectId, datasetId, serviceAccountKey, connectionPoolSize, maxStalenessMins } =- destinationConfig.bigQuery-- config = {- big_query: {- project_id: projectId,- dataset_id: datasetId,- service_account_key: serviceAccountKey,- connection_pool_size: connectionPoolSize,- max_staleness_mins: maxStalenessMins,- },- } as components['schemas']['ValidateReplicationDestinationBody']['config']- } else if ('iceberg' in destinationConfig) {- const {- projectRef: icebergProjectRef,- namespace,- warehouseName,- catalogToken,- s3AccessKeyId,- s3SecretAccessKey,- s3Region,- } = destinationConfig.iceberg-- config = {- iceberg: {- supabase: {- namespace,- project_ref: icebergProjectRef,- warehouse_name: warehouseName,- catalog_token: catalogToken,- s3_access_key_id: s3AccessKeyId,- s3_secret_access_key: s3SecretAccessKey,- s3_region: s3Region,- },- },- }- } else if ('ducklake' in destinationConfig) {- config = buildDucklakeApiConfig(- destinationConfig.ducklake- ) as components['schemas']['ValidateReplicationDestinationBody']['config']- } else if ('snowflake' in destinationConfig) {- const { accountId, user, privateKey, privateKeyPassphrase, database, schema, role } =- destinationConfig.snowflake-- config = {- snowflake: {- account_id: accountId,- user,- private_key: privateKey,- private_key_passphrase: privateKeyPassphrase,- database,- schema,- role,- },- } as components['schemas']['ValidateReplicationDestinationBody']['config']- } else if ('clickHouse' in destinationConfig) {- const { url, user, password, database, engine } = destinationConfig.clickHouse-- config = {- clickhouse: {- url,- user,- password,- database,- engine,- },- } as components['schemas']['ValidateReplicationDestinationBody']['config']- } else {- throw new Error(- 'Invalid destination config: must specify bigQuery, iceberg, ducklake, snowflake, or clickHouse'- )- }-- const batchConfig = maxFillMs !== undefined ? { max_fill_ms: maxFillMs } : undefined- const pipelineConfig =- publicationName === undefined- ? undefined- : {- publication_name: publicationName,- max_table_sync_workers: maxTableSyncWorkers,- max_copy_connections_per_table: maxCopyConnectionsPerTable,- invalidated_slot_behavior: invalidatedSlotBehavior,- table_sync_copy: tableSyncCopy,- batch: batchConfig,- }-const { data, error } = await post('/platform/replication/{ref}/destinations/validate', {params: { path: { ref: projectRef } },body: {- config,+ config: buildCreateDestinationApiConfig(destinationConfig),source_id: sourceId,- pipeline_config: pipelineConfig,+ pipeline_config:+ publicationName === undefined+ ? undefined+ : buildPipelineApiConfig({+ publicationName,+ maxTableSyncWorkers,+ maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,+ tableSyncCopy: tableSyncCopy ?? { type: 'include_all_tables' },+ batch: maxFillMs === undefined ? undefined : { maxFillMs },+ }),},signal,})if (error) handleError(error)- return data as ValidateDestinationResponse+ return data}type ValidateDestinationData = Awaited<ReturnType<typeof validateDestination>>diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.tsindex 81fc461046a..92019fe2484 100644--- a/apps/studio/data/replication/validate-pipeline-mutation.ts+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts@@ -1,7 +1,8 @@import { useMutation } from '@tanstack/react-query'import { components } from 'api-types'-import type { TableSyncCopyConfig } from './create-destination-pipeline-mutation'+import { type TableSyncCopyConfig } from './types'+import { buildPipelineApiConfig } from './utils'import { handleError, post } from '@/data/fetchers'import type { ResponseError, UseCustomMutationOptions } from '@/types'@@ -33,28 +34,24 @@ async function validatePipeline(if (!projectRef) throw new Error('projectRef is required')if (!sourceId) throw new Error('sourceId is required')- const batchConfig = maxFillMs !== undefined ? { max_fill_ms: maxFillMs } : undefined-- const config = {- publication_name: publicationName,- max_table_sync_workers: maxTableSyncWorkers,- max_copy_connections_per_table: maxCopyConnectionsPerTable,- invalidated_slot_behavior: invalidatedSlotBehavior,- table_sync_copy: tableSyncCopy,- batch: batchConfig,- }-const { data, error } = await post('/platform/replication/{ref}/pipelines/validate', {params: { path: { ref: projectRef } },body: {source_id: sourceId,- config: config as components['schemas']['ValidateReplicationPipelineBody']['config'],+ config: buildPipelineApiConfig({+ publicationName,+ maxTableSyncWorkers,+ maxCopyConnectionsPerTable,+ invalidatedSlotBehavior,+ tableSyncCopy,+ batch: maxFillMs === undefined ? undefined : { maxFillMs },+ }),},signal,})if (error) handleError(error)- return data as ValidatePipelineResponse+ return data}type ValidatePipelineData = Awaited<ReturnType<typeof validatePipeline>>