The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(etl): Add new setting for controlling table copy (#42853)
Riccardo Busetti67ed8f18eeca
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxindex 9b14461e8ac..ee640523a61 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx@@ -74,26 +74,26 @@ export const AdvancedSettings = ({<FormField_Shadcn_control={form.control}- name="maxSize"+ name="maxTableSyncWorkers"render={({ field }) => (<FormItemLayout- label="Batch size"+ label="Table sync workers"layout="horizontal"description={<>- <p>Number of rows to send in a batch.</p>- <p>Larger batches use more memory, with the risk of running out of memory.</p>+ <p>Number of tables to copy in parallel during the initial sync.</p>+ <p>Uses one replication slot per worker (N + 1 total when fully active).</p></>}><FormControl_Shadcn_>- <PrePostTab postTab="rows">+ <PrePostTab postTab="workers"><Input_Shadcn_{...field}type="number"value={field.value ?? ''}onChange={handleNumberChange(field)}- placeholder="Default: 100000"+ placeholder="Default: 4"/></PrePostTab></FormControl_Shadcn_>@@ -103,20 +103,22 @@ export const AdvancedSettings = ({<FormField_Shadcn_control={form.control}- name="maxTableSyncWorkers"+ name="maxCopyConnectionsPerTable"render={({ field }) => (<FormItemLayout- label="Table sync workers"+ label="Copy connections per table"layout="horizontal"description={<>- <p>Number of tables to copy in parallel during the initial sync.</p>- <p>Uses one replication slot per worker (N + 1 total when fully active).</p>+ <p>+ Number of copy connections each table sync worker can use in parallel when+ copying a table.+ </p></>}><FormControl_Shadcn_>- <PrePostTab postTab="workers">+ <PrePostTab postTab="connections"><Input_Shadcn_{...field}type="number"diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.tsindex 312d3798a53..a8949f27729 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts@@ -7,12 +7,16 @@ export const DestinationPanelFormSchema = z.object({name: z.string().min(1, 'Name is required'),publicationName: z.string().min(1, 'Publication is required'),maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),- maxSize: z.number().min(1, 'Max batch size should be greater than 0').int().optional(),maxTableSyncWorkers: z.number().min(1, 'Max table sync workers should be greater than 0').int().optional(),+ maxCopyConnectionsPerTable: z+ .number()+ .int()+ .min(1, 'Max copy connections per table should be greater than 0')+ .optional(),// BigQuery fieldsprojectId: z.string().optional(),datasetId: z.string().optional(),diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsxindex 5449a1982df..8d99386278e 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx@@ -19,8 +19,8 @@ import { useReplicationSourcesQuery } from 'data/replication/sources-query'import { useStartPipelineMutation } from 'data/replication/start-pipeline-mutation'import { useUpdateDestinationPipelineMutation } from 'data/replication/update-destination-pipeline-mutation'import {- type ValidationFailure,useValidateDestinationMutation,+ type ValidationFailure,} from 'data/replication/validate-destination-mutation'import { useValidatePipelineMutation } from 'data/replication/validate-pipeline-mutation'import { useIcebergNamespaceCreateMutation } from 'data/storage/iceberg-namespace-create-mutation'@@ -174,8 +174,8 @@ export const DestinationForm = ({name: destinationData?.name ?? '',publicationName: pipelineData?.config.publication_name ?? '',maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,- maxSize: pipelineData?.config?.batch?.max_size ?? undefined,maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,+ maxCopyConnectionsPerTable: pipelineData?.config?.max_copy_connections_per_table ?? undefined,// BigQuery fieldsprojectId: isBigQueryConfig ? config.big_query.project_id : '',datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',@@ -303,8 +303,8 @@ export const DestinationForm = ({sourceId,publicationName: data.publicationName,maxFillMs: data.maxFillMs,- maxSize: data.maxSize,maxTableSyncWorkers: data.maxTableSyncWorkers,+ maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,}),])@@ -403,10 +403,9 @@ export const DestinationForm = ({}const batchConfig: BatchConfig | undefined =- data.maxFillMs !== undefined || data.maxSize !== undefined+ data.maxFillMs !== undefined? {...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),- ...(data.maxSize !== undefined ? { maxSize: data.maxSize } : {}),}: undefinedconst hasBatchFields = batchConfig !== undefined@@ -422,6 +421,7 @@ export const DestinationForm = ({pipelineConfig: {publicationName: data.publicationName,maxTableSyncWorkers: data.maxTableSyncWorkers,+ maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,...(hasBatchFields ? { batch: batchConfig } : {}),},sourceId,@@ -486,10 +486,9 @@ export const DestinationForm = ({destinationConfig = { iceberg: icebergConfig }}const batchConfig: BatchConfig | undefined =- data.maxFillMs !== undefined || data.maxSize !== undefined+ data.maxFillMs !== undefined? {...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),- ...(data.maxSize !== undefined ? { maxSize: data.maxSize } : {}),}: undefinedconst hasBatchFields = batchConfig !== undefined@@ -504,6 +503,7 @@ export const DestinationForm = ({pipelineConfig: {publicationName: data.publicationName,maxTableSyncWorkers: data.maxTableSyncWorkers,+ maxCopyConnectionsPerTable: data.maxCopyConnectionsPerTable,...(hasBatchFields ? { batch: batchConfig } : {}),},})diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex 4d900622284..8a0cd04291c 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -1,9 +1,9 @@import { useMutation, useQueryClient } from '@tanstack/react-query'-import { toast } from 'sonner'-import type { components } from 'api-types'import { handleError, post } from 'data/fetchers'+import { toast } from 'sonner'import type { ResponseError, UseCustomMutationOptions } from 'types'+import { replicationKeys } from './keys'export type DestinationConfig =@@ -33,7 +33,6 @@ export type IcebergDestinationConfig = {export type BatchConfig = {maxFillMs?: number- maxSize?: number}export type CreateDestinationPipelineParams = {@@ -45,6 +44,7 @@ export type CreateDestinationPipelineParams = {publicationName: stringbatch?: BatchConfigmaxTableSyncWorkers?: number+ maxCopyConnectionsPerTable?: number}}@@ -53,7 +53,7 @@ async function createDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch, maxTableSyncWorkers },+ pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },sourceId,}: CreateDestinationPipelineParams,signal?: AbortSignal@@ -113,11 +113,13 @@ async function createDestinationPipeline(...(maxTableSyncWorkers !== undefined? { max_table_sync_workers: maxTableSyncWorkers }: {}),+ ...(maxCopyConnectionsPerTable !== undefined+ ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }+ : {}),...(batch? {batch: {...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),- ...(batch.maxSize !== undefined ? { max_size: batch.maxSize } : {}),},}: {}),diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex 8bdb50f0acc..a48f475357c 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -1,9 +1,9 @@import { useMutation, useQueryClient } from '@tanstack/react-query'-import { toast } from 'sonner'-import type { components } from 'api-types'import { handleError, post } from 'data/fetchers'+import { toast } from 'sonner'import type { ResponseError, UseCustomMutationOptions } from 'types'+import { BatchConfig, DestinationConfig } from './create-destination-pipeline-mutation'import { replicationKeys } from './keys'@@ -18,6 +18,7 @@ export type UpdateDestinationPipelineParams = {publicationName: stringbatch?: BatchConfigmaxTableSyncWorkers?: number+ maxCopyConnectionsPerTable?: number}}@@ -28,7 +29,7 @@ async function updateDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch, maxTableSyncWorkers },+ pipelineConfig: { publicationName, batch, maxTableSyncWorkers, maxCopyConnectionsPerTable },sourceId,}: UpdateDestinationPipelineParams,signal?: AbortSignal@@ -88,11 +89,13 @@ async function updateDestinationPipeline(...(maxTableSyncWorkers !== undefined? { max_table_sync_workers: maxTableSyncWorkers }: {}),+ ...(maxCopyConnectionsPerTable !== undefined+ ? { max_copy_connections_per_table: maxCopyConnectionsPerTable }+ : {}),...(batch? {batch: {...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),- ...(batch.maxSize !== undefined ? { max_size: batch.maxSize } : {}),},}: {}),diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.tsindex 9c85c9b2f12..af822a5bd07 100644--- a/apps/studio/data/replication/validate-pipeline-mutation.ts+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts@@ -9,8 +9,8 @@ type ValidatePipelineParams = {sourceId: numberpublicationName: stringmaxFillMs?: number- maxSize?: numbermaxTableSyncWorkers?: number+ maxCopyConnectionsPerTable?: number}type ValidatePipelineResponse = components['schemas']['ValidatePipelineResponse']@@ -20,18 +20,15 @@ async function validatePipeline(sourceId,publicationName,maxFillMs,- maxSize,maxTableSyncWorkers,+ maxCopyConnectionsPerTable,}: ValidatePipelineParams,signal?: AbortSignal): Promise<ValidatePipelineResponse> {if (!projectRef) throw new Error('projectRef is required')if (!sourceId) throw new Error('sourceId is required')- const batchConfig =- maxFillMs !== undefined || maxSize !== undefined- ? { max_fill_ms: maxFillMs, max_size: maxSize }- : undefined+ const batchConfig = maxFillMs !== undefined ? { max_fill_ms: maxFillMs } : undefinedconst { data, error } = await post('/platform/replication/{ref}/pipelines/validate', {params: { path: { ref: projectRef } },@@ -40,6 +37,7 @@ async function validatePipeline(config: {publication_name: publicationName,max_table_sync_workers: maxTableSyncWorkers,+ max_copy_connections_per_table: maxCopyConnectionsPerTable,batch: batchConfig,},},diff --git a/packages/api-types/types/api.d.ts b/packages/api-types/types/api.d.tsindex 32207dabcdb..0e5003ce695 100644--- a/packages/api-types/types/api.d.ts+++ b/packages/api-types/types/api.d.ts@@ -934,7 +934,8 @@ export interface paths {path?: nevercookie?: never}- get?: never+ /** Get database disk attributes */+ get: operations['v1-get-database-disk']put?: never/** Modify database disk */post: operations['v1-modify-database-disk']@@ -2806,6 +2807,23 @@ export interface components {type: 'io2'}}+ DiskResponse: {+ attributes:+ | {+ iops: number+ size_gb: number+ throughput_mibps?: number+ /** @enum {string} */+ type: 'gp3'+ }+ | {+ iops: number+ size_gb: number+ /** @enum {string} */+ type: 'io2'+ }+ last_modified_at?: string+ }DiskUtilMetricsResponse: {metrics: {fs_avail_bytes: number@@ -4821,6 +4839,7 @@ export interface components {* @description Deprecated. Use `status` instead.*/healthy: boolean+ replication_connected: boolean}| {db_schema: string@@ -8291,6 +8310,56 @@ export interface operations {}}}+ 'v1-get-database-disk': {+ parameters: {+ query?: never+ header?: never+ path: {+ /** @description Project ref */+ ref: string+ }+ cookie?: never+ }+ requestBody?: never+ responses: {+ 200: {+ headers: {+ [name: string]: unknown+ }+ content: {+ 'application/json': components['schemas']['DiskResponse']+ }+ }+ /** @description Unauthorized */+ 401: {+ headers: {+ [name: string]: unknown+ }+ content?: never+ }+ /** @description Forbidden action */+ 403: {+ headers: {+ [name: string]: unknown+ }+ content?: never+ }+ /** @description Rate limit exceeded */+ 429: {+ headers: {+ [name: string]: unknown+ }+ content?: never+ }+ /** @description Failed to get database disk attributes */+ 500: {+ headers: {+ [name: string]: unknown+ }+ content?: never+ }+ }+ }'v1-modify-database-disk': {parameters: {query?: neverdiff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.tsindex f695cc39e5b..9d917f93f37 100644--- a/packages/api-types/types/platform.d.ts+++ b/packages/api-types/types/platform.d.ts@@ -5451,12 +5451,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -5481,12 +5478,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -8766,12 +8760,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -8828,12 +8819,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -10365,12 +10353,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -10395,12 +10380,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**@@ -10806,12 +10788,9 @@ export interface components {* @example 200*/max_fill_ms?: number- /**- * @description Maximum size of the batch- * @example 200- */- max_size?: number}+ /** @description Maximum number of copy connections per table */+ max_copy_connections_per_table?: number/** @description Maximum number of table sync workers */max_table_sync_workers?: number/**