The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(etl): Expose more configuration parameters (#41758)
Riccardo Busetti1238a19fe440
diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsxindex 2fd126068ff..9b14461e8ac 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/AdvancedSettings.tsx@@ -48,9 +48,14 @@ export const AdvancedSettings = ({name="maxFillMs"render={({ field }) => (<FormItemLayout- label="Batch wait time"layout="horizontal"- description="How long to wait for more changes before sending. Shorter times mean more real-time updates but higher overhead."+ label="Batch wait time"+ description={+ <>+ <p>Time to wait for additional changes before sending.</p>+ <p>Shorter times imply faster updates, but higher overhead.</p>+ </>+ }><FormControl_Shadcn_><PrePostTab postTab="milliseconds">@@ -59,7 +64,65 @@ export const AdvancedSettings = ({type="number"value={field.value ?? ''}onChange={handleNumberChange(field)}- placeholder="Default: 10000 (10 seconds)"+ placeholder="Default: 10000"+ />+ </PrePostTab>+ </FormControl_Shadcn_>+ </FormItemLayout>+ )}+ />++ <FormField_Shadcn_+ control={form.control}+ name="maxSize"+ render={({ field }) => (+ <FormItemLayout+ label="Batch size"+ layout="horizontal"+ description={+ <>+ <p>Number of rows to send in a batch.</p>+ <p>Larger batches use more memory, with the risk of running out of memory.</p>+ </>+ }+ >+ <FormControl_Shadcn_>+ <PrePostTab postTab="rows">+ <Input_Shadcn_+ {...field}+ type="number"+ value={field.value ?? ''}+ onChange={handleNumberChange(field)}+ placeholder="Default: 100000"+ />+ </PrePostTab>+ </FormControl_Shadcn_>+ </FormItemLayout>+ )}+ />++ <FormField_Shadcn_+ control={form.control}+ name="maxTableSyncWorkers"+ render={({ field }) => (+ <FormItemLayout+ label="Table sync workers"+ layout="horizontal"+ description={+ <>+ <p>Number of tables to copy in parallel during the initial sync.</p>+ <p>Uses one replication slot per worker (N + 1 total when fully active).</p>+ </>+ }+ >+ <FormControl_Shadcn_>+ <PrePostTab postTab="workers">+ <Input_Shadcn_+ {...field}+ type="number"+ value={field.value ?? ''}+ onChange={handleNumberChange(field)}+ placeholder="Default: 4"/></PrePostTab></FormControl_Shadcn_>@@ -81,7 +144,15 @@ export const AdvancedSettings = ({</div>}layout="horizontal"- description="Maximum age of cached data before BigQuery reads from base tables at query time. Lower values ensure fresher results but may increase query costs. Leave empty for no staleness limit."+ description={+ <>+ <p>+ Maximum age of cached data before BigQuery reads from base tables at query+ time.+ </p>+ <p>Lower values return fresher results, but may increase query costs.</p>+ </>+ }><FormControl_Shadcn_><PrePostTab postTab="minutes">@@ -90,7 +161,7 @@ export const AdvancedSettings = ({type="number"value={field.value ?? ''}onChange={handleNumberChange(field)}- placeholder="Default: none"+ placeholder="Default: None (No staleness limit)"/></PrePostTab></FormControl_Shadcn_>diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.tsindex 97e09b56a95..312d3798a53 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/DestinationForm.schema.ts@@ -7,6 +7,12 @@ export const DestinationPanelFormSchema = z.object({name: z.string().min(1, 'Name is required'),publicationName: z.string().min(1, 'Publication is required'),maxFillMs: z.number().min(1, 'Max Fill milliseconds should be greater than 0').int().optional(),+ maxSize: z.number().min(1, 'Max batch size should be greater than 0').int().optional(),+ maxTableSyncWorkers: z+ .number()+ .min(1, 'Max table sync workers should be greater than 0')+ .int()+ .optional(),// BigQuery fieldsprojectId: z.string().optional(),datasetId: z.string().optional(),diff --git a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsxindex 9143453ec5a..25eb91dc02f 100644--- a/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx+++ b/apps/studio/components/interfaces/Database/Replication/DestinationPanel/DestinationForm/index.tsx@@ -173,7 +173,9 @@ export const DestinationForm = ({// Common fieldsname: destinationData?.name ?? '',publicationName: pipelineData?.config.publication_name ?? '',- maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? 10000, // Default: 10 seconds+ maxFillMs: pipelineData?.config?.batch?.max_fill_ms ?? undefined,+ maxSize: pipelineData?.config?.batch?.max_size ?? undefined,+ maxTableSyncWorkers: pipelineData?.config?.max_table_sync_workers ?? undefined,// BigQuery fieldsprojectId: isBigQueryConfig ? config.big_query.project_id : '',datasetId: isBigQueryConfig ? config.big_query.dataset_id : '',@@ -301,6 +303,8 @@ export const DestinationForm = ({sourceId,publicationName: data.publicationName,maxFillMs: data.maxFillMs,+ maxSize: data.maxSize,+ maxTableSyncWorkers: data.maxTableSyncWorkers,}),])@@ -398,9 +402,13 @@ export const DestinationForm = ({destinationConfig = { iceberg: icebergConfig }}- const batchConfig: BatchConfig | undefined = !!data.maxFillMs- ? { maxFillMs: data.maxFillMs }- : undefined+ const batchConfig: BatchConfig | undefined =+ data.maxFillMs !== undefined || data.maxSize !== undefined+ ? {+ ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),+ ...(data.maxSize !== undefined ? { maxSize: data.maxSize } : {}),+ }+ : undefinedconst hasBatchFields = batchConfig !== undefinedif (!destinationConfig) throw new Error('Destination configuration is missing')@@ -413,6 +421,7 @@ export const DestinationForm = ({destinationConfig,pipelineConfig: {publicationName: data.publicationName,+ maxTableSyncWorkers: data.maxTableSyncWorkers,...(hasBatchFields ? { batch: batchConfig } : {}),},sourceId,@@ -476,9 +485,13 @@ export const DestinationForm = ({}destinationConfig = { iceberg: icebergConfig }}- const batchConfig: BatchConfig | undefined = !!data.maxFillMs- ? { maxFillMs: data.maxFillMs }- : undefined+ const batchConfig: BatchConfig | undefined =+ data.maxFillMs !== undefined || data.maxSize !== undefined+ ? {+ ...(data.maxFillMs !== undefined ? { maxFillMs: data.maxFillMs } : {}),+ ...(data.maxSize !== undefined ? { maxSize: data.maxSize } : {}),+ }+ : undefinedconst hasBatchFields = batchConfig !== undefinedif (!destinationConfig) throw new Error('Destination configuration is missing')@@ -490,6 +503,7 @@ export const DestinationForm = ({sourceId,pipelineConfig: {publicationName: data.publicationName,+ maxTableSyncWorkers: data.maxTableSyncWorkers,...(hasBatchFields ? { batch: batchConfig } : {}),},})diff --git a/apps/studio/data/replication/create-destination-pipeline-mutation.ts b/apps/studio/data/replication/create-destination-pipeline-mutation.tsindex 8141dac26e6..4d900622284 100644--- a/apps/studio/data/replication/create-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/create-destination-pipeline-mutation.ts@@ -32,7 +32,8 @@ export type IcebergDestinationConfig = {}export type BatchConfig = {- maxFillMs: number+ maxFillMs?: number+ maxSize?: number}export type CreateDestinationPipelineParams = {@@ -43,6 +44,7 @@ export type CreateDestinationPipelineParams = {pipelineConfig: {publicationName: stringbatch?: BatchConfig+ maxTableSyncWorkers?: number}}@@ -51,7 +53,7 @@ async function createDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch },+ pipelineConfig: { publicationName, batch, maxTableSyncWorkers },sourceId,}: CreateDestinationPipelineParams,signal?: AbortSignal@@ -108,7 +110,17 @@ async function createDestinationPipeline(destination_config,pipeline_config: {publication_name: publicationName,- ...(!!batch ? { batch: { max_fill_ms: batch.maxFillMs } } : {}),+ ...(maxTableSyncWorkers !== undefined+ ? { max_table_sync_workers: maxTableSyncWorkers }+ : {}),+ ...(batch+ ? {+ batch: {+ ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),+ ...(batch.maxSize !== undefined ? { max_size: batch.maxSize } : {}),+ },+ }+ : {}),},},signal,diff --git a/apps/studio/data/replication/update-destination-pipeline-mutation.ts b/apps/studio/data/replication/update-destination-pipeline-mutation.tsindex 12da1fcd1db..8bdb50f0acc 100644--- a/apps/studio/data/replication/update-destination-pipeline-mutation.ts+++ b/apps/studio/data/replication/update-destination-pipeline-mutation.ts@@ -17,6 +17,7 @@ export type UpdateDestinationPipelineParams = {pipelineConfig: {publicationName: stringbatch?: BatchConfig+ maxTableSyncWorkers?: number}}@@ -27,7 +28,7 @@ async function updateDestinationPipeline(projectRef,destinationName: destinationName,destinationConfig,- pipelineConfig: { publicationName, batch },+ pipelineConfig: { publicationName, batch, maxTableSyncWorkers },sourceId,}: UpdateDestinationPipelineParams,signal?: AbortSignal@@ -84,7 +85,17 @@ async function updateDestinationPipeline(destination_name: destinationName,pipeline_config: {publication_name: publicationName,- ...(!!batch ? { batch: { max_fill_ms: batch.maxFillMs } } : {}),+ ...(maxTableSyncWorkers !== undefined+ ? { max_table_sync_workers: maxTableSyncWorkers }+ : {}),+ ...(batch+ ? {+ batch: {+ ...(batch.maxFillMs !== undefined ? { max_fill_ms: batch.maxFillMs } : {}),+ ...(batch.maxSize !== undefined ? { max_size: batch.maxSize } : {}),+ },+ }+ : {}),},},signal,diff --git a/apps/studio/data/replication/validate-pipeline-mutation.ts b/apps/studio/data/replication/validate-pipeline-mutation.tsindex 67d3be6acd2..9c85c9b2f12 100644--- a/apps/studio/data/replication/validate-pipeline-mutation.ts+++ b/apps/studio/data/replication/validate-pipeline-mutation.ts@@ -9,23 +9,38 @@ type ValidatePipelineParams = {sourceId: numberpublicationName: stringmaxFillMs?: number+ maxSize?: number+ maxTableSyncWorkers?: number}type ValidatePipelineResponse = components['schemas']['ValidatePipelineResponse']async function validatePipeline(- { projectRef, sourceId, publicationName, maxFillMs }: ValidatePipelineParams,+ {+ projectRef,+ sourceId,+ publicationName,+ maxFillMs,+ maxSize,+ maxTableSyncWorkers,+ }: ValidatePipelineParams,signal?: AbortSignal): Promise<ValidatePipelineResponse> {if (!projectRef) throw new Error('projectRef is required')if (!sourceId) throw new Error('sourceId is required')+ const batchConfig =+ maxFillMs !== undefined || maxSize !== undefined+ ? { max_fill_ms: maxFillMs, max_size: maxSize }+ : undefined+const { data, error } = await post('/platform/replication/{ref}/pipelines/validate', {params: { path: { ref: projectRef } },body: {source_id: sourceId,config: {publication_name: publicationName,- ...(maxFillMs !== undefined ? { batch: { max_fill_ms: maxFillMs } } : {}),+ max_table_sync_workers: maxTableSyncWorkers,+ batch: batchConfig,},},signal,diff --git a/packages/api-types/types/platform.d.ts b/packages/api-types/types/platform.d.tsindex 54861b411af..5831308e089 100644--- a/packages/api-types/types/platform.d.ts+++ b/packages/api-types/types/platform.d.ts@@ -5383,11 +5383,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -5411,11 +5413,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -8600,11 +8604,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -8660,11 +8666,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -10257,11 +10265,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -10285,11 +10295,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders@@ -10581,12 +10593,12 @@ export interface components {failure_type: 'critical' | 'warning'/*** @description Validation failure name- * @example BigQuery Dataset Not Found+ * @example Validation Failed*/name: string/*** @description Validation failure reason- * @example Dataset 'my_dataset' does not exist in project 'my_project'+ * @example The configuration is invalid*/reason: string}[]@@ -10602,12 +10614,12 @@ export interface components {failure_type: 'critical' | 'warning'/*** @description Validation failure name- * @example Publication Not Found+ * @example Validation Failed*/name: string/*** @description Validation failure reason- * @example Publication 'my_publication' does not exist in the source database+ * @example The configuration is invalid*/reason: string}[]@@ -10694,11 +10706,13 @@ export interface components {*/max_fill_ms?: number/**- * @description Maximum batch size+ * @description Maximum size of the batch* @example 200*/max_size?: number}+ /** @description Maximum number of table sync workers */+ max_table_sync_workers?: number/*** @description Publication name* @example pub_orders